Why use logits

ML
Author

Nipun Batra

Published

February 27, 2024

Open In Colab
import numpy as np
import sklearn 
import torch
import torch.nn as nn
import matplotlib.pyplot as plt
from latexify import *
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
import torch
import torch.nn.functional as F
import ipywidgets as widgets
from ipywidgets import interactive
from IPython.display import display

# Example ground truth probabilities
ground_truth_probs = torch.tensor([0.3, 0.7])

# Example model predictions (logits)
model_logits = torch.tensor([-2.0, 2.0])

# Applying softmax to logits to get probabilities
model_probs = F.sigmoid(model_logits)

# Cross-entropy loss using probabilities
loss_probs = F.binary_cross_entropy(model_probs, ground_truth_probs)

# Cross-entropy loss using logits
loss_logits = F.binary_cross_entropy_with_logits(model_logits, ground_truth_probs)

print("Loss using probabilities:", loss_probs.item())
print("Loss using logits:", loss_logits.item())
Loss using probabilities: 0.7269279956817627
Loss using logits: 0.7269280552864075
l1 = widgets.FloatSlider(value=0.3, min=-300, max=300, step=0.01, description='Logits ex 1')
l2 = widgets.FloatSlider(value=0.7, min=-300, max=300, step=0.01, description='Logits ex 2')

box = widgets.VBox([l1, l2])
def print_loss_using_both_methods(l1, l2):
    logits = torch.tensor([l1, l2])
    probs = F.sigmoid(logits)
    loss_probs = F.binary_cross_entropy(probs, ground_truth_probs)
    loss_logits = F.binary_cross_entropy_with_logits(logits, ground_truth_probs)
    print("Loss using probabilities:", loss_probs.item())
    print("Loss using logits:", loss_logits.item())
# add interactivity
interactive(print_loss_using_both_methods, l1=l1, l2=l2)
print_loss_using_both_methods(l1.value, l2.value)
Loss using probabilities: 3.164093017578125
Loss using logits: 3.164093017578125
def our_sigmoid(z):
    return 1/(1+torch.exp(-z))
our_sigmoid(torch.tensor(-90.0))
tensor(0.)
our_sigmoid(torch.tensor(-92.0))
tensor(0.)
our_sigmoid(torch.tensor(9.0))
tensor(0.9999)
F.binary_cross_entropy_with_logits?
Signature:

F.binary_cross_entropy_with_logits(

    input: torch.Tensor,

    target: torch.Tensor,

    weight: Optional[torch.Tensor] = None,

    size_average: Optional[bool] = None,

    reduce: Optional[bool] = None,

    reduction: str = 'mean',

    pos_weight: Optional[torch.Tensor] = None,

) -> torch.Tensor

Docstring:

Function that measures Binary Cross Entropy between target and input

logits.



See :class:`~torch.nn.BCEWithLogitsLoss` for details.



Args:

    input: Tensor of arbitrary shape as unnormalized scores (often referred to as logits).

    target: Tensor of the same shape as input with values between 0 and 1

    weight (Tensor, optional): a manual rescaling weight

        if provided it's repeated to match input tensor shape

    size_average (bool, optional): Deprecated (see :attr:`reduction`). By default,

        the losses are averaged over each loss element in the batch. Note that for

        some losses, there multiple elements per sample. If the field :attr:`size_average`

        is set to ``False``, the losses are instead summed for each minibatch. Ignored

        when reduce is ``False``. Default: ``True``

    reduce (bool, optional): Deprecated (see :attr:`reduction`). By default, the

        losses are averaged or summed over observations for each minibatch depending

        on :attr:`size_average`. When :attr:`reduce` is ``False``, returns a loss per

        batch element instead and ignores :attr:`size_average`. Default: ``True``

    reduction (string, optional): Specifies the reduction to apply to the output:

        ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied,

        ``'mean'``: the sum of the output will be divided by the number of

        elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average`

        and :attr:`reduce` are in the process of being deprecated, and in the meantime,

        specifying either of those two args will override :attr:`reduction`. Default: ``'mean'``

    pos_weight (Tensor, optional): a weight of positive examples.

            Must be a vector with length equal to the number of classes.



Examples::



     >>> input = torch.randn(3, requires_grad=True)

     >>> target = torch.empty(3).random_(2)

     >>> loss = F.binary_cross_entropy_with_logits(input, target)

     >>> loss.backward()

File:      ~/miniconda3/lib/python3.9/site-packages/torch/nn/functional.py

Type:      function