Log-sum-exp

Author

Nipun Batra 

import torch
xs = torch.linspace(0.01, 1, 1000)
xs
tensor([0.0100, 0.0110, 0.0120, 0.0130, 0.0140, 0.0150, 0.0159, 0.0169, 0.0179,
        0.0189, 0.0199, 0.0209, 0.0219, 0.0229, 0.0239, 0.0249, 0.0259, 0.0268,
        0.0278, 0.0288, 0.0298, 0.0308, 0.0318, 0.0328, 0.0338, 0.0348, 0.0358,
        0.0368, 0.0377, 0.0387, 0.0397, 0.0407, 0.0417, 0.0427, 0.0437, 0.0447,
        0.0457, 0.0467, 0.0477, 0.0486, 0.0496, 0.0506, 0.0516, 0.0526, 0.0536,
        0.0546, 0.0556, 0.0566, 0.0576, 0.0586, 0.0595, 0.0605, 0.0615, 0.0625,
        0.0635, 0.0645, 0.0655, 0.0665, 0.0675, 0.0685, 0.0695, 0.0705, 0.0714,
        0.0724, 0.0734, 0.0744, 0.0754, 0.0764, 0.0774, 0.0784, 0.0794, 0.0804,
        0.0814, 0.0823, 0.0833, 0.0843, 0.0853, 0.0863, 0.0873, 0.0883, 0.0893,
        0.0903, 0.0913, 0.0923, 0.0932, 0.0942, 0.0952, 0.0962, 0.0972, 0.0982,
        0.0992, 0.1002, 0.1012, 0.1022, 0.1032, 0.1041, 0.1051, 0.1061, 0.1071,
        0.1081, 0.1091, 0.1101, 0.1111, 0.1121, 0.1131, 0.1141, 0.1150, 0.1160,
        0.1170, 0.1180, 0.1190, 0.1200, 0.1210, 0.1220, 0.1230, 0.1240, 0.1250,
        0.1259, 0.1269, 0.1279, 0.1289, 0.1299, 0.1309, 0.1319, 0.1329, 0.1339,
        0.1349, 0.1359, 0.1368, 0.1378, 0.1388, 0.1398, 0.1408, 0.1418, 0.1428,
        0.1438, 0.1448, 0.1458, 0.1468, 0.1477, 0.1487, 0.1497, 0.1507, 0.1517,
        0.1527, 0.1537, 0.1547, 0.1557, 0.1567, 0.1577, 0.1586, 0.1596, 0.1606,
        0.1616, 0.1626, 0.1636, 0.1646, 0.1656, 0.1666, 0.1676, 0.1686, 0.1695,
        0.1705, 0.1715, 0.1725, 0.1735, 0.1745, 0.1755, 0.1765, 0.1775, 0.1785,
        0.1795, 0.1805, 0.1814, 0.1824, 0.1834, 0.1844, 0.1854, 0.1864, 0.1874,
        0.1884, 0.1894, 0.1904, 0.1914, 0.1923, 0.1933, 0.1943, 0.1953, 0.1963,
        0.1973, 0.1983, 0.1993, 0.2003, 0.2013, 0.2023, 0.2032, 0.2042, 0.2052,
        0.2062, 0.2072, 0.2082, 0.2092, 0.2102, 0.2112, 0.2122, 0.2132, 0.2141,
        0.2151, 0.2161, 0.2171, 0.2181, 0.2191, 0.2201, 0.2211, 0.2221, 0.2231,
        0.2241, 0.2250, 0.2260, 0.2270, 0.2280, 0.2290, 0.2300, 0.2310, 0.2320,
        0.2330, 0.2340, 0.2350, 0.2359, 0.2369, 0.2379, 0.2389, 0.2399, 0.2409,
        0.2419, 0.2429, 0.2439, 0.2449, 0.2459, 0.2468, 0.2478, 0.2488, 0.2498,
        0.2508, 0.2518, 0.2528, 0.2538, 0.2548, 0.2558, 0.2568, 0.2577, 0.2587,
        0.2597, 0.2607, 0.2617, 0.2627, 0.2637, 0.2647, 0.2657, 0.2667, 0.2677,
        0.2686, 0.2696, 0.2706, 0.2716, 0.2726, 0.2736, 0.2746, 0.2756, 0.2766,
        0.2776, 0.2786, 0.2795, 0.2805, 0.2815, 0.2825, 0.2835, 0.2845, 0.2855,
        0.2865, 0.2875, 0.2885, 0.2895, 0.2905, 0.2914, 0.2924, 0.2934, 0.2944,
        0.2954, 0.2964, 0.2974, 0.2984, 0.2994, 0.3004, 0.3014, 0.3023, 0.3033,
        0.3043, 0.3053, 0.3063, 0.3073, 0.3083, 0.3093, 0.3103, 0.3113, 0.3123,
        0.3132, 0.3142, 0.3152, 0.3162, 0.3172, 0.3182, 0.3192, 0.3202, 0.3212,
        0.3222, 0.3232, 0.3241, 0.3251, 0.3261, 0.3271, 0.3281, 0.3291, 0.3301,
        0.3311, 0.3321, 0.3331, 0.3341, 0.3350, 0.3360, 0.3370, 0.3380, 0.3390,
        0.3400, 0.3410, 0.3420, 0.3430, 0.3440, 0.3450, 0.3459, 0.3469, 0.3479,
        0.3489, 0.3499, 0.3509, 0.3519, 0.3529, 0.3539, 0.3549, 0.3559, 0.3568,
        0.3578, 0.3588, 0.3598, 0.3608, 0.3618, 0.3628, 0.3638, 0.3648, 0.3658,
        0.3668, 0.3677, 0.3687, 0.3697, 0.3707, 0.3717, 0.3727, 0.3737, 0.3747,
        0.3757, 0.3767, 0.3777, 0.3786, 0.3796, 0.3806, 0.3816, 0.3826, 0.3836,
        0.3846, 0.3856, 0.3866, 0.3876, 0.3886, 0.3895, 0.3905, 0.3915, 0.3925,
        0.3935, 0.3945, 0.3955, 0.3965, 0.3975, 0.3985, 0.3995, 0.4005, 0.4014,
        0.4024, 0.4034, 0.4044, 0.4054, 0.4064, 0.4074, 0.4084, 0.4094, 0.4104,
        0.4114, 0.4123, 0.4133, 0.4143, 0.4153, 0.4163, 0.4173, 0.4183, 0.4193,
        0.4203, 0.4213, 0.4223, 0.4232, 0.4242, 0.4252, 0.4262, 0.4272, 0.4282,
        0.4292, 0.4302, 0.4312, 0.4322, 0.4332, 0.4341, 0.4351, 0.4361, 0.4371,
        0.4381, 0.4391, 0.4401, 0.4411, 0.4421, 0.4431, 0.4441, 0.4450, 0.4460,
        0.4470, 0.4480, 0.4490, 0.4500, 0.4510, 0.4520, 0.4530, 0.4540, 0.4550,
        0.4559, 0.4569, 0.4579, 0.4589, 0.4599, 0.4609, 0.4619, 0.4629, 0.4639,
        0.4649, 0.4659, 0.4668, 0.4678, 0.4688, 0.4698, 0.4708, 0.4718, 0.4728,
        0.4738, 0.4748, 0.4758, 0.4768, 0.4777, 0.4787, 0.4797, 0.4807, 0.4817,
        0.4827, 0.4837, 0.4847, 0.4857, 0.4867, 0.4877, 0.4886, 0.4896, 0.4906,
        0.4916, 0.4926, 0.4936, 0.4946, 0.4956, 0.4966, 0.4976, 0.4986, 0.4995,
        0.5005, 0.5015, 0.5025, 0.5035, 0.5045, 0.5055, 0.5065, 0.5075, 0.5085,
        0.5095, 0.5105, 0.5114, 0.5124, 0.5134, 0.5144, 0.5154, 0.5164, 0.5174,
        0.5184, 0.5194, 0.5204, 0.5214, 0.5223, 0.5233, 0.5243, 0.5253, 0.5263,
        0.5273, 0.5283, 0.5293, 0.5303, 0.5313, 0.5323, 0.5332, 0.5342, 0.5352,
        0.5362, 0.5372, 0.5382, 0.5392, 0.5402, 0.5412, 0.5422, 0.5432, 0.5441,
        0.5451, 0.5461, 0.5471, 0.5481, 0.5491, 0.5501, 0.5511, 0.5521, 0.5531,
        0.5541, 0.5550, 0.5560, 0.5570, 0.5580, 0.5590, 0.5600, 0.5610, 0.5620,
        0.5630, 0.5640, 0.5650, 0.5659, 0.5669, 0.5679, 0.5689, 0.5699, 0.5709,
        0.5719, 0.5729, 0.5739, 0.5749, 0.5759, 0.5768, 0.5778, 0.5788, 0.5798,
        0.5808, 0.5818, 0.5828, 0.5838, 0.5848, 0.5858, 0.5868, 0.5877, 0.5887,
        0.5897, 0.5907, 0.5917, 0.5927, 0.5937, 0.5947, 0.5957, 0.5967, 0.5977,
        0.5986, 0.5996, 0.6006, 0.6016, 0.6026, 0.6036, 0.6046, 0.6056, 0.6066,
        0.6076, 0.6086, 0.6095, 0.6105, 0.6115, 0.6125, 0.6135, 0.6145, 0.6155,
        0.6165, 0.6175, 0.6185, 0.6195, 0.6205, 0.6214, 0.6224, 0.6234, 0.6244,
        0.6254, 0.6264, 0.6274, 0.6284, 0.6294, 0.6304, 0.6314, 0.6323, 0.6333,
        0.6343, 0.6353, 0.6363, 0.6373, 0.6383, 0.6393, 0.6403, 0.6413, 0.6423,
        0.6432, 0.6442, 0.6452, 0.6462, 0.6472, 0.6482, 0.6492, 0.6502, 0.6512,
        0.6522, 0.6532, 0.6541, 0.6551, 0.6561, 0.6571, 0.6581, 0.6591, 0.6601,
        0.6611, 0.6621, 0.6631, 0.6641, 0.6650, 0.6660, 0.6670, 0.6680, 0.6690,
        0.6700, 0.6710, 0.6720, 0.6730, 0.6740, 0.6750, 0.6759, 0.6769, 0.6779,
        0.6789, 0.6799, 0.6809, 0.6819, 0.6829, 0.6839, 0.6849, 0.6859, 0.6868,
        0.6878, 0.6888, 0.6898, 0.6908, 0.6918, 0.6928, 0.6938, 0.6948, 0.6958,
        0.6968, 0.6977, 0.6987, 0.6997, 0.7007, 0.7017, 0.7027, 0.7037, 0.7047,
        0.7057, 0.7067, 0.7077, 0.7086, 0.7096, 0.7106, 0.7116, 0.7126, 0.7136,
        0.7146, 0.7156, 0.7166, 0.7176, 0.7186, 0.7195, 0.7205, 0.7215, 0.7225,
        0.7235, 0.7245, 0.7255, 0.7265, 0.7275, 0.7285, 0.7295, 0.7305, 0.7314,
        0.7324, 0.7334, 0.7344, 0.7354, 0.7364, 0.7374, 0.7384, 0.7394, 0.7404,
        0.7414, 0.7423, 0.7433, 0.7443, 0.7453, 0.7463, 0.7473, 0.7483, 0.7493,
        0.7503, 0.7513, 0.7523, 0.7532, 0.7542, 0.7552, 0.7562, 0.7572, 0.7582,
        0.7592, 0.7602, 0.7612, 0.7622, 0.7632, 0.7641, 0.7651, 0.7661, 0.7671,
        0.7681, 0.7691, 0.7701, 0.7711, 0.7721, 0.7731, 0.7741, 0.7750, 0.7760,
        0.7770, 0.7780, 0.7790, 0.7800, 0.7810, 0.7820, 0.7830, 0.7840, 0.7850,
        0.7859, 0.7869, 0.7879, 0.7889, 0.7899, 0.7909, 0.7919, 0.7929, 0.7939,
        0.7949, 0.7959, 0.7968, 0.7978, 0.7988, 0.7998, 0.8008, 0.8018, 0.8028,
        0.8038, 0.8048, 0.8058, 0.8068, 0.8077, 0.8087, 0.8097, 0.8107, 0.8117,
        0.8127, 0.8137, 0.8147, 0.8157, 0.8167, 0.8177, 0.8186, 0.8196, 0.8206,
        0.8216, 0.8226, 0.8236, 0.8246, 0.8256, 0.8266, 0.8276, 0.8286, 0.8295,
        0.8305, 0.8315, 0.8325, 0.8335, 0.8345, 0.8355, 0.8365, 0.8375, 0.8385,
        0.8395, 0.8405, 0.8414, 0.8424, 0.8434, 0.8444, 0.8454, 0.8464, 0.8474,
        0.8484, 0.8494, 0.8504, 0.8514, 0.8523, 0.8533, 0.8543, 0.8553, 0.8563,
        0.8573, 0.8583, 0.8593, 0.8603, 0.8613, 0.8623, 0.8632, 0.8642, 0.8652,
        0.8662, 0.8672, 0.8682, 0.8692, 0.8702, 0.8712, 0.8722, 0.8732, 0.8741,
        0.8751, 0.8761, 0.8771, 0.8781, 0.8791, 0.8801, 0.8811, 0.8821, 0.8831,
        0.8841, 0.8850, 0.8860, 0.8870, 0.8880, 0.8890, 0.8900, 0.8910, 0.8920,
        0.8930, 0.8940, 0.8950, 0.8959, 0.8969, 0.8979, 0.8989, 0.8999, 0.9009,
        0.9019, 0.9029, 0.9039, 0.9049, 0.9059, 0.9068, 0.9078, 0.9088, 0.9098,
        0.9108, 0.9118, 0.9128, 0.9138, 0.9148, 0.9158, 0.9168, 0.9177, 0.9187,
        0.9197, 0.9207, 0.9217, 0.9227, 0.9237, 0.9247, 0.9257, 0.9267, 0.9277,
        0.9286, 0.9296, 0.9306, 0.9316, 0.9326, 0.9336, 0.9346, 0.9356, 0.9366,
        0.9376, 0.9386, 0.9395, 0.9405, 0.9415, 0.9425, 0.9435, 0.9445, 0.9455,
        0.9465, 0.9475, 0.9485, 0.9495, 0.9505, 0.9514, 0.9524, 0.9534, 0.9544,
        0.9554, 0.9564, 0.9574, 0.9584, 0.9594, 0.9604, 0.9614, 0.9623, 0.9633,
        0.9643, 0.9653, 0.9663, 0.9673, 0.9683, 0.9693, 0.9703, 0.9713, 0.9723,
        0.9732, 0.9742, 0.9752, 0.9762, 0.9772, 0.9782, 0.9792, 0.9802, 0.9812,
        0.9822, 0.9832, 0.9841, 0.9851, 0.9861, 0.9871, 0.9881, 0.9891, 0.9901,
        0.9911, 0.9921, 0.9931, 0.9941, 0.9950, 0.9960, 0.9970, 0.9980, 0.9990,
        1.0000])
xs.prod()
tensor(0.)
xs = torch.linspace(0.01, 1, 100)
xs.prod()
tensor(9.3326e-43)
xs = torch.linspace(0.01, 1, 1000)
a = xs.log().sum()

b = torch.linspace(0.01, 1, 10000).log().sum()
print(a, b)
tensor(-954.8404) tensor(-9536.1816)
torch.linspace(0.01, 1, 1000).log().sum()
tensor(-954.8404)
torch.linspace(0.01, 1, 10000).log().sum()
tensor(-9536.1816)
# Trivial example to test the binary cross entropy loss

y_true = torch.tensor([0., 1., 1., 0.])
logits = torch.tensor([0.1, 0.9, 0.8, 0.2])

loss = torch.nn.BCEWithLogitsLoss()
loss(logits, y_true)
tensor(0.5637)

y_hat = torch.nn.Sigmoid()(logits)
y_hat
tensor([0.5250, 0.7109, 0.6900, 0.5498])
# BCE
-(y_true * torch.log(y_hat) + (1 - y_true) * torch.log(1 - y_hat)).mean()
tensor(0.5637)
def our_bce(y_true, logits):
    y_hat = torch.nn.Sigmoid()(logits)
    return -(y_true * torch.log(y_hat) + (1 - y_true) * torch.log(1 - y_hat)).mean()
# check that our implementation is correct
assert our_bce(y_true, logits) == loss(logits, y_true)
# Larger logits
logits = torch.tensor([100., 200., 300., 400.])

assert our_bce(y_true, logits) == loss(logits, y_true)

---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
/Users/nipun/git/ml2023/notebooks/posts/log-sum-exp.ipynb Cell 12 in <cell line: 4>()
      <a href='vscode-notebook-cell:/Users/nipun/git/ml2023/notebooks/posts/log-sum-exp.ipynb#X13sZmlsZQ%3D%3D?line=0'>1</a> # Larger logits
      <a href='vscode-notebook-cell:/Users/nipun/git/ml2023/notebooks/posts/log-sum-exp.ipynb#X13sZmlsZQ%3D%3D?line=1'>2</a> logits = torch.tensor([100., 200., 300., 400.])
----> <a href='vscode-notebook-cell:/Users/nipun/git/ml2023/notebooks/posts/log-sum-exp.ipynb#X13sZmlsZQ%3D%3D?line=3'>4</a> assert our_bce(y_true, logits) == loss(logits, y_true)

AssertionError: 
torch.nn.Sigmoid()(logits)
tensor([1., 1., 1., 1.])
torch.exp(logits)/(1 + torch.exp(logits))
tensor([nan, nan, nan, nan])
torch.exp(-logits)
tensor([3.7835e-44, 0.0000e+00, 0.0000e+00, 0.0000e+00])
1/(1+torch.exp(-logits))
tensor([1., 1., 1., 1.])