feat: initial HSAP platform

Huaxu Sentinel Active Safety Platform with embedded algorithm code,
Docker Compose setup, and vendored dataset scaffolds for clone-and-run.

Co-authored-by: Cursor <cursoragent@cursor.com>

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/__init__.py

@@ -0,0 +1,9 @@
+# Implementation based on pytorch 1.6.0
+from .lane_seg_loss import LaneLoss, SADLoss
+from .hungarian_loss import HungarianLoss
+from .hungarian_bezier_loss import HungarianBezierLoss
+from .weighted_ce_loss import WeightedCrossEntropyLoss
+from .torch_loss import torch_loss
+from .focal_loss import _focal_loss, FocalLoss
+from .laneatt_loss import LaneAttLoss
+from .builder import LOSSES

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/_utils.py

@@ -0,0 +1,24 @@
+import torch
+from torch import Tensor
+from typing import Optional
+from torch.nn import _reduction as _Reduction
+
+
+class _Loss(torch.nn.Module):
+    reduction: str
+
+    def __init__(self, size_average=None, reduce=None, reduction: str = 'mean') -> None:
+        super(_Loss, self).__init__()
+        if size_average is not None or reduce is not None:
+            self.reduction = _Reduction.legacy_get_string(size_average, reduce)
+        else:
+            self.reduction = reduction
+
+
+class WeightedLoss(_Loss):
+    def __init__(self, weight: Optional[Tensor] = None, size_average=None, reduce=None,
+                 reduction: str = 'mean') -> None:
+        super(WeightedLoss, self).__init__(size_average, reduce, reduction)
+        if weight is not None and not isinstance(weight, Tensor):
+            weight = torch.tensor(weight).cuda()
+        self.register_buffer('weight', weight)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/builder.py

@@ -0,0 +1,3 @@
+from ..registry import SimpleRegistry
+
+LOSSES = SimpleRegistry()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/focal_loss.py

@@ -0,0 +1,151 @@
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# Source: https://github.com/kornia/kornia/blob/f4f70fefb63287f72bc80cd96df9c061b1cb60dd/kornia/losses/focal.py
+def one_hot(labels: torch.Tensor,
+            num_classes: int,
+            device: Optional[torch.device] = None,
+            dtype: Optional[torch.dtype] = None,
+            eps: Optional[float] = 1e-6) -> torch.Tensor:
+    r"""Converts an integer label x-D tensor to a one-hot (x+1)-D tensor.
+
+    Args:
+        labels (torch.Tensor) : tensor with labels of shape :math:`(N, *)`,
+                                where N is batch size. Each value is an integer
+                                representing correct classification.
+        num_classes (int): number of classes in labels.
+        device (Optional[torch.device]): the desired device of returned tensor.
+         Default: if None, uses the current device for the default tensor type
+         (see torch.set_default_tensor_type()). device will be the CPU for CPU
+         tensor types and the current CUDA device for CUDA tensor types.
+        dtype (Optional[torch.dtype]): the desired data type of returned
+         tensor. Default: if None, infers data type from values.
+
+    Returns:
+        torch.Tensor: the labels in one hot tensor of shape :math:`(N, C, *)`,
+
+    Examples::
+        >>> labels = torch.LongTensor([[[0, 1], [2, 0]]])
+        >>> kornia.losses.one_hot(labels, num_classes=3)
+        tensor([[[[1., 0.],
+                  [0., 1.]],
+                 [[0., 1.],
+                  [0., 0.]],
+                 [[0., 0.],
+                  [1., 0.]]]]
+    """
+    if not torch.is_tensor(labels):
+        raise TypeError("Input labels type is not a torch.Tensor. Got {}".format(type(labels)))
+    if not labels.dtype == torch.int64:
+        raise ValueError("labels must be of the same dtype torch.int64. Got: {}".format(labels.dtype))
+    if num_classes < 1:
+        raise ValueError("The number of classes must be bigger than one." " Got: {}".format(num_classes))
+    shape = labels.shape
+    one_hot = torch.zeros(shape[0], num_classes, *shape[1:], device=device, dtype=dtype)
+    return one_hot.scatter_(1, labels.unsqueeze(1), 1.0) + eps
+
+
+def _focal_loss(input: torch.Tensor,
+                target: torch.Tensor,
+                alpha: float,
+                gamma: float = 2.0,
+                reduction: str = 'none',
+                eps: float = 1e-8) -> torch.Tensor:
+    r"""Function that computes Focal loss.
+
+    See :class:`~kornia.losses.FocalLoss` for details.
+    """
+    if not torch.is_tensor(input):
+        raise TypeError("Input type is not a torch.Tensor. Got {}".format(type(input)))
+
+    if not len(input.shape) >= 2:
+        raise ValueError("Invalid input shape, we expect BxCx*. Got: {}".format(input.shape))
+
+    if input.size(0) != target.size(0):
+        raise ValueError('Expected input batch_size ({}) to match target batch_size ({}).'.format(
+            input.size(0), target.size(0)))
+
+    n = input.size(0)
+    out_size = (n, ) + input.size()[2:]
+    if target.size()[1:] != input.size()[2:]:
+        raise ValueError('Expected target size {}, got {}'.format(out_size, target.size()))
+
+    if not input.device == target.device:
+        raise ValueError("input and target must be in the same device. Got: {} and {}".format(
+            input.device, target.device))
+
+    # compute softmax over the classes axis
+    input_soft: torch.Tensor = F.softmax(input, dim=1) + eps
+
+    # create the labels one hot tensor
+    target_one_hot: torch.Tensor = one_hot(target, num_classes=input.shape[1], device=input.device, dtype=input.dtype)
+
+    # compute the actual focal loss
+    weight = torch.pow(-input_soft + 1., gamma)
+
+    focal = -alpha * weight * torch.log(input_soft)
+    loss_tmp = torch.sum(target_one_hot * focal, dim=1)
+
+    if reduction == 'none':
+        loss = loss_tmp
+    elif reduction == 'mean':
+        loss = torch.mean(loss_tmp)
+    elif reduction == 'sum':
+        loss = torch.sum(loss_tmp)
+    else:
+        raise NotImplementedError("Invalid reduction mode: {}".format(reduction))
+    return loss
+
+
+class FocalLoss(nn.Module):
+    r"""Criterion that computes Focal loss.
+
+    According to [1], the Focal loss is computed as follows:
+
+    .. math::
+
+        \text{FL}(p_t) = -\alpha_t (1 - p_t)^{\gamma} \, \text{log}(p_t)
+
+    where:
+       - :math:`p_t` is the model's estimated probability for each class.
+
+
+    Arguments:
+        alpha (float): Weighting factor :math:`\alpha \in [0, 1]`.
+        gamma (float): Focusing parameter :math:`\gamma >= 0`.
+        reduction (str, optional): Specifies the reduction to apply to the
+         output: ‘none’ | ‘mean’ | ‘sum’. ‘none’: no reduction will be applied,
+         ‘mean’: the sum of the output will be divided by the number of elements
+         in the output, ‘sum’: the output will be summed. Default: ‘none’.
+
+    Shape:
+        - Input: :math:`(N, C, *)` where C = number of classes.
+        - Target: :math:`(N, *)` where each value is
+          :math:`0 ≤ targets[i] ≤ C−1`.
+
+    Examples:
+        >>> N = 5  # num_classes
+        >>> kwargs = {"alpha": 0.5, "gamma": 2.0, "reduction": 'mean'}
+        >>> loss = kornia.losses.FocalLoss(**kwargs)
+        >>> input = torch.randn(1, N, 3, 5, requires_grad=True)
+        >>> target = torch.empty(1, 3, 5, dtype=torch.long).random_(N)
+        >>> output = loss(input, target)
+        >>> output.backward()
+
+    References:
+        [1] https://arxiv.org/abs/1708.02002
+    """
+    def __init__(self, alpha: float, gamma: float = 2.0, reduction: str = 'none') -> None:
+        super(FocalLoss, self).__init__()
+        self.alpha: float = alpha
+        self.gamma: float = gamma
+        self.reduction: str = reduction
+        self.eps: float = 1e-6
+
+    def forward(  # type: ignore
+            self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+        return _focal_loss(input, target, self.alpha, self.gamma, self.reduction, self.eps)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/hungarian_bezier_loss.py

@@ -0,0 +1,210 @@
+# Copied and modified from facebookresearch/detr
+# Refactored and added comments
+
+
+import torch
+from torch.nn import functional as F
+from scipy.optimize import linear_sum_assignment
+
+from ..ddp_utils import is_dist_avail_and_initialized, get_world_size
+from ..curve_utils import BezierSampler, cubic_bezier_curve_segment, get_valid_points
+from ._utils import WeightedLoss
+from .hungarian_loss import HungarianLoss
+from .builder import LOSSES
+
+
+# TODO: Speed-up Hungarian on GPU with tensors
+class _HungarianMatcher(torch.nn.Module):
+    """This class computes an assignment between the targets and the predictions of the network
+
+    For efficiency reasons, the targets don't include the no_object. Because of this, in general,
+    there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions,
+    while the others are un-matched (and thus treated as non-objects).
+    POTO matching, which maximizes the cost matrix.
+    """
+
+    def __init__(self, alpha=0.8, bezier_order=3, num_sample_points=100, k=7):
+        super().__init__()
+        self.k = k
+        self.alpha = alpha
+        self.num_sample_points = num_sample_points
+        self.bezier_sampler = BezierSampler(num_sample_points=num_sample_points, order=bezier_order)
+
+    @torch.no_grad()
+    def forward(self, outputs, targets):
+        # Compute the matrices for an entire batch (computation is all pairs, in a way includes the real loss function)
+        # targets: each target: ['keypoints': L x N x 2]
+        # B: batch size; Q: max lanes per-pred, G: total num ground-truth-lanes
+        B, Q = outputs["logits"].shape
+        target_keypoints = torch.cat([i['keypoints'] for i in targets], dim=0)  # G x N x 2
+        target_sample_points = torch.cat([i['sample_points'] for i in targets], dim=0)  # G x num_sample_points x 2
+
+        # Valid bezier segments
+        target_keypoints = cubic_bezier_curve_segment(target_keypoints, target_sample_points)
+        target_sample_points = self.bezier_sampler.get_sample_points(target_keypoints)
+
+        # target_valid_points = get_valid_points(target_sample_points)  # G x num_sample_points
+        G, N = target_keypoints.shape[:2]
+        out_prob = outputs["logits"].sigmoid()  # B x Q
+        out_lane = outputs['curves']  # B x Q x N x 2
+        sizes = [target['keypoints'].shape[0] for target in targets]
+
+        # 1. Local maxima prior
+        _, max_indices = torch.nn.functional.max_pool1d(out_prob.unsqueeze(1),
+                                                        kernel_size=self.k, stride=1,
+                                                        padding=(self.k - 1) // 2, return_indices=True)
+        max_indices = max_indices.squeeze(1)  # B x Q
+        indices = torch.arange(0, Q, dtype=out_prob.dtype, device=out_prob.device).unsqueeze(0).expand_as(max_indices)
+        local_maxima = (max_indices == indices).flatten().unsqueeze(-1).expand(-1, G)  # BQ x G
+
+        # Safe reshape
+        out_prob = out_prob.flatten()  # BQ
+        out_lane = out_lane.flatten(end_dim=1)  # BQ x N x 2
+
+        # 2. Compute the classification cost. Contrary to the loss, we don't use the NLL,
+        # but approximate it in 1 - prob[target class].
+        # Then 1 can be omitted due to it is only a constant.
+        # For binary classification, it is just prob (understand this prob as objectiveness in OD)
+        cost_label = out_prob.unsqueeze(-1).expand(-1, G)  # BQ x G
+
+        # 3. Compute the curve sampling cost
+        cost_curve = 1 - torch.cdist(self.bezier_sampler.get_sample_points(out_lane).flatten(start_dim=-2),
+                                     target_sample_points.flatten(start_dim=-2),
+                                     p=1) / self.num_sample_points  # BQ x G
+
+        # Bound the cost to [0, 1]
+        cost_curve = cost_curve.clamp(min=0, max=1)
+
+        # Final cost matrix (scipy uses min instead of max)
+        C = local_maxima * cost_label ** (1 - self.alpha) * cost_curve ** self.alpha
+        C = -C.view(B, Q, -1).cpu()
+
+        # Hungarian (weighted) on each image
+        indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))]
+
+        # Return (pred_indices, target_indices) for each image
+        return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices]
+
+
+@LOSSES.register()
+class HungarianBezierLoss(WeightedLoss):
+    def __init__(self, curve_weight=1, label_weight=0.1, seg_weight=0.75, alpha=0.8,
+                 num_sample_points=100, bezier_order=3, weight=None, size_average=None, reduce=None, reduction='mean',
+                 ignore_index=-100, weight_seg=None, k=9):
+        super().__init__(weight, size_average, reduce, reduction)
+        self.curve_weight = curve_weight  # Weight for sampled points' L1 distance error between curves
+        self.label_weight = label_weight  # Weight for classification error
+        self.seg_weight = seg_weight  # Weight for binary segmentation auxiliary task
+        self.weight_seg = weight_seg  # BCE loss weight
+        self.ignore_index = ignore_index
+        self.bezier_sampler = BezierSampler(num_sample_points=num_sample_points, order=bezier_order)
+        self.matcher = _HungarianMatcher(alpha=alpha, num_sample_points=num_sample_points, bezier_order=bezier_order,
+                                         k=k)
+        if self.weight is not None and not isinstance(self.weight, torch.Tensor):
+            self.weight = torch.tensor(self.weight).cuda()
+        if self.weight_seg is not None and not isinstance(self.weight_seg, torch.Tensor):
+            self.weight_seg = torch.tensor(self.weight_seg).cuda()
+        self.register_buffer('pos_weight', self.weight[1] / self.weight[0])
+        self.register_buffer('pos_weight_seg', self.weight_seg[1] / self.weight_seg[0])
+
+    def forward(self, inputs, targets, net):
+        outputs = net(inputs)
+        output_curves = outputs['curves']
+        target_labels = torch.zeros_like(outputs['logits'])
+        target_segmentations = torch.stack([target['segmentation_mask'] for target in targets])
+
+        total_targets = 0
+        for i in targets:
+            total_targets += i['keypoints'].numel()
+
+        # CULane actually can produce a whole batch of no-lane images,
+        # in which case, we just calculate the classification loss
+        if total_targets > 0:
+            # Match
+            indices = self.matcher(outputs=outputs, targets=targets)
+            idx = HungarianLoss.get_src_permutation_idx(indices)
+            output_curves = output_curves[idx]
+
+            # Targets (rearrange each lane in the whole batch)
+            # B x N x ... -> BN x ...
+            target_keypoints = torch.cat([t['keypoints'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+            target_sample_points = torch.cat([t['sample_points'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+
+            # Valid bezier segments
+            target_keypoints = cubic_bezier_curve_segment(target_keypoints, target_sample_points)
+            target_sample_points = self.bezier_sampler.get_sample_points(target_keypoints)
+
+            target_labels[idx] = 1  # Any matched lane has the same label 1
+
+        else:
+            # For DDP
+            target_sample_points = torch.tensor([], dtype=torch.float32, device=output_curves.device)
+
+        target_valid_points = get_valid_points(target_sample_points)
+        # Loss
+        loss_curve = self.point_loss(self.bezier_sampler.get_sample_points(output_curves),
+                                     target_sample_points)
+        loss_label = self.classification_loss(inputs=outputs['logits'], targets=target_labels)
+        loss_seg = self.binary_seg_loss(inputs=outputs['segmentations'], targets=target_segmentations)
+
+        loss = self.label_weight * loss_label + self.curve_weight * loss_curve + self.seg_weight * loss_seg
+
+        return loss, {'training loss': loss, 'loss label': loss_label, 'loss curve': loss_curve,
+                      'loss seg': loss_seg,
+                      'valid portion': target_valid_points.float().mean()}
+
+    def point_loss(self, inputs, targets, valid_points=None):
+        # L1 loss on sample points
+        # inputs/targets: L x N x 2
+        # valid points: L x N
+        if targets.numel() == 0:
+            targets = inputs.clone().detach()
+        loss = F.l1_loss(inputs, targets, reduction='none')
+        if valid_points is not None:
+            loss *= valid_points.unsqueeze(-1)
+            normalizer = valid_points.sum()
+        else:
+            normalizer = targets.shape[0] * targets.shape[1]
+            normalizer = torch.as_tensor([normalizer], dtype=inputs.dtype, device=inputs.device)
+        if self.reduction == 'mean':
+            if is_dist_avail_and_initialized():  # Global normalizer should be same across devices
+                torch.distributed.all_reduce(normalizer)
+            normalizer = torch.clamp(normalizer / get_world_size(), min=1).item()
+            loss = loss.sum() / normalizer
+        elif self.reduction == 'sum':  # Usually not needed, but let's have it anyway
+            loss = loss.sum()
+
+        return loss
+
+    def classification_loss(self, inputs, targets):
+        # Typical classification loss (cross entropy)
+        # No need for permutation, assume target is matched to inputs
+
+        # Negative weight as positive weight
+        return F.binary_cross_entropy_with_logits(inputs.unsqueeze(1), targets.unsqueeze(1), pos_weight=self.pos_weight,
+                                                  reduction=self.reduction) / self.pos_weight
+
+    def binary_seg_loss(self, inputs, targets):
+        # BCE segmentation loss with weighting and ignore index
+        # No relation whatever to matching
+
+        # Process inputs
+        inputs = torch.nn.functional.interpolate(inputs, size=targets.shape[-2:], mode='bilinear', align_corners=True)
+        inputs = inputs.squeeze(1)
+
+        # Process targets
+        valid_map = (targets != self.ignore_index)
+        targets[~valid_map] = 0
+        targets = targets.float()
+
+        # Negative weight as positive weight
+        loss = F.binary_cross_entropy_with_logits(inputs, targets, pos_weight=self.pos_weight_seg,
+                                                  reduction='none') / self.pos_weight_seg
+        loss *= valid_map
+
+        if self.reduction == 'mean':
+            return loss.mean()
+        elif self.reduction == 'sum':
+            return loss.sum()
+        else:
+            return loss

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/hungarian_loss.py

@@ -0,0 +1,187 @@
+# Copied and modified from facebookresearch/detr and liuruijin17/LSTR
+# Refactored and added comments
+# Hungarian loss for LSTR
+import torch
+from torch import Tensor
+from torch.nn import functional as F
+from scipy.optimize import linear_sum_assignment
+
+from ._utils import WeightedLoss
+from ..models.lane_detection import cubic_curve_with_projection
+from ..ddp_utils import is_dist_avail_and_initialized, get_world_size
+from .builder import LOSSES
+
+
+@torch.no_grad()
+def lane_normalize_in_batch(keypoints):
+    # Calculate normalization weights for lanes with different number of valid sample points,
+    # so they can produce loss in a similar scale: rather weird but it is what LSTR did
+    # https://github.com/liuruijin17/LSTR/blob/6044f7b2c5892dba7201c273ee632b4962350223/models/py_utils/matcher.py#L59
+    # keypoints: [..., N, 2], ... means arbitrary number of leading dimensions
+    # No gather/reduce is considered here as in the original implementation
+    valid_points = keypoints[..., 0] > 0
+    norm_weights = (valid_points.sum().float() / valid_points.sum(dim=-1).float()) ** 0.5
+    norm_weights /= norm_weights.max()
+
+    return norm_weights, valid_points  # [...], [..., N]
+
+
+# TODO: Speed-up Hungarian on GPU with tensors
+# Nothing will happen with DDP (for at last we use image-wise results)
+class HungarianMatcher(torch.nn.Module):
+    """This class computes an assignment between the targets and the predictions of the network
+
+    For efficiency reasons, the targets don't include the no_object. Because of this, in general,
+    there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions,
+    while the others are un-matched (and thus treated as non-objects).
+    """
+
+    def __init__(self, upper_weight=2, lower_weight=2, curve_weight=5, label_weight=3):
+        super().__init__()
+        self.lower_weight = lower_weight
+        self.upper_weight = upper_weight
+        self.curve_weight = curve_weight
+        self.label_weight = label_weight
+
+    @torch.no_grad()
+    def forward(self, outputs, targets):
+        # Compute the matrices for an entire batch (computation is all pairs, in a way includes the real loss function)
+        # targets: each target: ['keypoints': L x N x 2, 'padding_mask': H x W, 'uppers': L, 'lowers': L, 'labels': L]
+        # B: bs; Q: max lanes per-pred, L: num lanes, N: num keypoints per-lane, G: total num ground-truth-lanes
+        bs, num_queries = outputs["logits"].shape[:2]
+        out_prob = outputs["logits"].softmax(dim=-1)  # BQ x 2
+        out_lane = outputs['curves'].flatten(end_dim=-2)  # BQ x 8
+        target_uppers = torch.cat([i['uppers'] for i in targets])
+        target_lowers = torch.cat([i['lowers'] for i in targets])
+        sizes = [target['labels'].shape[0] for target in targets]
+        num_gt = sum(sizes)
+
+        # 1. Compute the classification cost. Contrary to the loss, we don't use the NLL,
+        # but approximate it in 1 - prob[target class].
+        # Then 1 can be omitted due to it is only a constant.
+        # For binary classification, it is just prob (understand this prob as objectiveness in OD)
+        cost_label = -out_prob[..., 1].unsqueeze(-1).flatten(end_dim=-2).repeat(1, num_gt)  # BQ x G
+
+        # 2. Compute the L1 cost between lowers and uppers
+        cost_upper = torch.cdist(out_lane[:, 0:1], target_uppers.unsqueeze(-1), p=1)  # BQ x G
+        cost_lower = torch.cdist(out_lane[:, 1:2], target_lowers.unsqueeze(-1), p=1)  # BQ x G
+
+        # 3. Compute the curve cost
+        target_keypoints = torch.cat([i['keypoints'] for i in targets], dim=0)  # G x N x 2
+        norm_weights, valid_points = lane_normalize_in_batch(target_keypoints)  # G, G x N
+
+        # Masked torch.cdist(p=1)
+        expand_shape = [bs * num_queries, num_gt, target_keypoints.shape[-2]]  # BQ x G x N
+        coefficients = out_lane[:, 2:].unsqueeze(1).expand(*expand_shape[:-1], -1)  # BQ x G x 6
+        out_x = cubic_curve_with_projection(y=target_keypoints[:, :, 1].unsqueeze(0).expand(expand_shape),
+                                            coefficients=coefficients)  # BQ x G x N
+        cost_curve = ((out_x - target_keypoints[:, :, 0].unsqueeze(0).expand(expand_shape)).abs() *
+                      valid_points.unsqueeze(0).expand(expand_shape)).sum(-1)  # BQ x G
+        cost_curve *= norm_weights  # BQ x G
+
+        # Final cost matrix
+        C = self.label_weight * cost_label + self.curve_weight * cost_curve + \
+            self.lower_weight * cost_lower + self.upper_weight * cost_upper
+        C = C.view(bs, num_queries, -1).cpu()
+
+        # Hungarian (weighted) on each image
+        indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))]
+
+        # Return (pred_indices, target_indices) for each image
+        return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices]
+
+
+# The Hungarian loss for LSTR
+@LOSSES.register()
+class HungarianLoss(WeightedLoss):
+    __constants__ = ['reduction']
+
+    def __init__(self, upper_weight=2, lower_weight=2, curve_weight=5, label_weight=3,
+                 weight=None, size_average=None, reduce=None, reduction='mean'):
+        super(HungarianLoss, self).__init__(weight, size_average, reduce, reduction)
+        self.lower_weight = lower_weight
+        self.upper_weight = upper_weight
+        self.curve_weight = curve_weight
+        self.label_weight = label_weight
+        self.matcher = HungarianMatcher(upper_weight, lower_weight, curve_weight, label_weight)
+
+    @staticmethod
+    def get_src_permutation_idx(indices):
+        # Permute predictions following indices
+        # 2-dim indices: (dim0 indices, dim1 indices)
+        batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)])
+        image_idx = torch.cat([src for (src, _) in indices])
+
+        return batch_idx, image_idx
+
+    def forward(self, inputs: Tensor, targets: Tensor, net):
+        # Support arbitrary auxiliary losses for transformer-based methods
+        if 'padding_mask' in targets[0].keys():  # For multi-scale training support
+            padding_masks = torch.stack([i['padding_mask'] for i in targets])
+            outputs = net(inputs, padding_masks)
+        else:
+            outputs = net(inputs)
+        loss, log_dict = self.calc_full_loss(outputs=outputs, targets=targets)
+        if 'aux' in outputs:
+            for i in range(len(outputs['aux'])):
+                aux_loss, aux_log_dict = self.calc_full_loss(outputs=outputs['aux'][i], targets=targets)
+                loss += aux_loss
+                for k in list(log_dict):  # list(dict) is needed for Python3, since .keys() does not copy like Python2
+                    log_dict[k + ' aux' + str(i)] = aux_log_dict[k]
+
+        return loss, log_dict
+
+    def calc_full_loss(self, outputs, targets):
+        # Match
+        indices = self.matcher(outputs=outputs, targets=targets)
+        idx = self.get_src_permutation_idx(indices)
+
+        # Targets (rearrange each lane in the whole batch)
+        # B x N x ... -> BN x ...
+        target_lowers = torch.cat([t['lowers'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+        target_uppers = torch.cat([t['uppers'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+        target_keypoints = torch.cat([t['keypoints'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+        target_labels = torch.zeros(outputs['logits'].shape[:-1], dtype=torch.int64, device=outputs['logits'].device)
+        target_labels[idx] = 1  # Any matched lane has the same label 1
+
+        # Loss
+        loss_label = self.classification_loss(inputs=outputs['logits'].permute(0, 2, 1), targets=target_labels)
+        output_curves = outputs['curves'][idx]
+        norm_weights, valid_points = lane_normalize_in_batch(target_keypoints)
+        out_x = cubic_curve_with_projection(coefficients=output_curves[:, 2:],
+                                            y=target_keypoints[:, :, 1].clone().detach())
+        loss_curve = self.point_loss(inputs=out_x, targets=target_keypoints[:, :, 0],
+                                     norm_weights=norm_weights, valid_points=valid_points)
+        loss_upper = self.point_loss(inputs=output_curves[:, 0], targets=target_uppers)
+        loss_lower = self.point_loss(inputs=output_curves[:, 1], targets=target_lowers)
+        loss = self.label_weight * loss_label + self.curve_weight * loss_curve + \
+            self.lower_weight * loss_lower + self.upper_weight * loss_upper
+
+        return loss, {'training loss': loss, 'loss label': loss_label, 'loss curve': loss_curve,
+                      'loss upper': loss_upper, 'loss lower': loss_lower}
+
+    def point_loss(self, inputs: Tensor, targets: Tensor, norm_weights=None, valid_points=None) -> Tensor:
+        # L1 loss on sample points, shouldn't it be direct regression?
+        # Also, loss_lowers and loss_uppers in original LSTR code can be done with this same function
+        # No need for permutation, assume target is matched to inputs
+        # inputs/targets: L x N
+        loss = F.l1_loss(inputs, targets, reduction='none')
+        if norm_weights is not None:  # Weights for each lane
+            loss *= norm_weights.unsqueeze(-1).expand_as(loss)
+        if valid_points is not None:  # Valid points
+            loss = loss[valid_points]
+        if self.reduction == 'mean':
+            normalizer = torch.as_tensor([targets.shape[0]], dtype=inputs.dtype, device=inputs.device)
+            if is_dist_avail_and_initialized():  # Global normalizer should be same across devices
+                torch.distributed.all_reduce(normalizer)
+            normalizer = torch.clamp(normalizer / get_world_size(), min=1).item()
+            loss = loss.sum() / normalizer  # Reduce only by number of curves (not number of points)
+        elif self.reduction == 'sum':  # Usually not needed, but let's have it anyway
+            loss = loss.sum()
+
+        return loss
+
+    def classification_loss(self, inputs: Tensor, targets: Tensor) -> Tensor:
+        # Typical classification loss (cross entropy)
+        # No need for permutation, assume target is matched to inputs
+        return F.cross_entropy(inputs, targets, reduction=self.reduction)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/lane_seg_loss.py

@@ -0,0 +1,50 @@
+import torch
+from torch import Tensor
+from typing import Optional
+from torch.nn import functional as F
+
+from ._utils import WeightedLoss
+from .builder import LOSSES
+
+
+# Typical lane detection loss by binary segmentation (e.g. SCNN)
+@LOSSES.register()
+class LaneLoss(WeightedLoss):
+    __constants__ = ['ignore_index', 'reduction']
+    ignore_index: int
+
+    def __init__(self, existence_weight: float = 0.1, weight: Optional[Tensor] = None, size_average=None,
+                 ignore_index: int = -100, reduce=None, reduction: str = 'mean'):
+        super(LaneLoss, self).__init__(weight, size_average, reduce, reduction)
+        self.ignore_index = ignore_index
+        self.existence_weight = existence_weight
+
+    def forward(self, inputs: Tensor, targets: Tensor, lane_existence: Tensor, net, interp_size):
+        outputs = net(inputs)
+        prob_maps = torch.nn.functional.interpolate(outputs['out'], size=interp_size, mode='bilinear',
+                                                    align_corners=True)
+        targets[targets > lane_existence.shape[-1]] = 255  # Ignore extra lanes
+        segmentation_loss = F.cross_entropy(prob_maps, targets, weight=self.weight,
+                                            ignore_index=self.ignore_index, reduction=self.reduction)
+        existence_loss = F.binary_cross_entropy_with_logits(outputs['lane'], lane_existence,
+                                                            weight=None, pos_weight=None, reduction=self.reduction)
+        total_loss = segmentation_loss + self.existence_weight * existence_loss
+
+        return total_loss, {'training loss': total_loss, 'loss seg': segmentation_loss,
+                            'loss exist': existence_loss}
+
+
+# Loss function for SAD
+@LOSSES.register()
+class SADLoss(WeightedLoss):
+    __constants__ = ['ignore_index', 'reduction']
+    ignore_index: int
+
+    def __init__(self, existence_weight: float = 0.1, weight: Optional[Tensor] = None, size_average=None,
+                 ignore_index: int = -100, reduce=None, reduction: str = 'mean'):
+        super(SADLoss, self).__init__(weight, size_average, reduce, reduction)
+        self.ignore_index = ignore_index
+        self.existence_weight = existence_weight
+
+    def forward(self, inputs: Tensor, targets: Tensor):
+        pass

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/laneatt_loss.py

@@ -0,0 +1,191 @@
+import torch
+from torch import Tensor
+from typing import Optional
+# from torch.nn import functional as F
+
+from ._utils import WeightedLoss
+from .builder import LOSSES
+from .focal_loss import FocalLoss
+from ..ddp_utils import get_world_size
+
+INFINITY = 987654.
+
+
+@LOSSES.register()
+class LaneAttLoss(WeightedLoss):
+    def __init__(self,
+                 cls_weight: float = 10.,
+                 reg_weight: float = 1.,
+                 alpha: float = 0.25,
+                 gamma: float = 2.,
+                 num_strips: int = 72 - 1,
+                 num_offsets: int = 72,
+                 t_pos: float = 15.,
+                 t_neg: float = 20.,
+                 weight: Optional[Tensor] = None,
+                 size_average=None,
+                 reduce=None,
+                 reduction: str = 'mean'):
+        super(LaneAttLoss, self).__init__(weight, size_average, reduce, reduction)
+        self.cls_weight = cls_weight
+        self.reg_weight = reg_weight
+        self.num_strips = num_strips
+        self.num_offsets = num_offsets
+        self.t_pos = t_pos
+        self.t_neg = t_neg
+        self.focal_loss = FocalLoss(alpha=alpha, gamma=gamma)
+        self.smooth_l1_loss = torch.nn.SmoothL1Loss()
+
+    def forward(self, inputs, targets, net):
+        # inputs: batchsize x 3 x img_h x img_w
+        # B: batch size, M: max lane
+        labels = {
+            'offsets': torch.stack([i['offsets'] for i in targets], dim=0),  # B x M x num_offsets
+            'starts': torch.stack([i['starts'] for i in targets], dim=0),  # B x M x 1
+            'lengths': torch.stack([i['lengths'] for i in targets], dim=0),  # B x M x 1
+            'flags': torch.stack([i['flags'] for i in targets], dim=0)  # B x M x 1
+        }
+        batch_size = inputs.shape[0]
+
+        outputs = net(inputs)
+        cls_loss = torch.tensor(0, dtype=torch.float32, device=inputs.device)
+        reg_loss = torch.tensor(0, dtype=torch.float32, device=inputs.device)
+        total_positives = 0
+
+        # TODO: Replace these ridiculous codes with batch computation
+        for i in range(batch_size):
+            # Filter lanes that do not exist (confidence == 0)
+            target = {k: v[i][labels['flags'][i]] for k, v in labels.items()}
+
+            # If there are no targets, all proposals have to be negatives (i.e., 0 confidence)
+            if len(target['offsets']) == 0:
+                cls_target = torch.zeros(outputs['logits'][i].shape[0],
+                                         dtype=torch.long, device=outputs['logits'].device)
+                cls_loss = cls_loss + self.focal_loss(outputs['logits'][i], cls_target).sum()
+                continue
+
+            # Match GT
+            positives_mask, invalid_offsets_mask, negatives_mask, target_positives_indices = \
+                self.match_proposals_with_targets(net.module.anchors.clone() if get_world_size() >= 2
+                                                  else net.anchors.clone(), target)
+            # Get positives & negatives
+            positives = {k: v[i][positives_mask] for k, v in outputs.items()}
+            num_positives = positives['logits'].shape[0]
+            total_positives += num_positives
+            negatives = {k: v[i][negatives_mask] for k, v in outputs.items()}
+            num_negatives = negatives['logits'].shape[0]
+
+            # Handle edge case of no positives found
+            if num_positives == 0:
+                cls_target = torch.zeros(outputs['logits'][i].shape[0],
+                                         dtype=torch.long, device=outputs['logits'].device)
+                cls_loss = cls_loss + self.focal_loss(outputs['logits'][i], cls_target).sum()
+                continue
+
+            # Get classification targets (normal cases)
+            cls_target = torch.zeros(num_positives + num_negatives,
+                                     dtype=torch.long, device=outputs['logits'].device)
+            cls_target[:num_positives] = 1
+            cls_pred = torch.cat([positives['logits'], negatives['logits']], dim=0)
+
+            # Regression loss (including length)
+            reg_pred = torch.cat([positives['lengths'][..., None], positives['offsets']], dim=1)
+            with torch.no_grad():
+                target = {k: v[target_positives_indices] for k, v in target.items()}
+                positive_starts = (positives['starts'] * self.num_strips).round().long()
+                targets_starts = (target['starts'] * self.num_strips).round().long()
+                target['lengths'] -= (positive_starts - targets_starts)
+                all_indices = torch.arange(num_positives, dtype=torch.long)
+                ends = (positive_starts + target['lengths'] - 1).round().long()
+                # length + num_offsets + pad (assignment trick ?)
+                invalid_offsets_mask = torch.zeros((num_positives, 1 + self.num_offsets + 1), dtype=torch.int)
+                invalid_offsets_mask[all_indices, 1 + positive_starts] = 1
+                invalid_offsets_mask[all_indices, 1 + ends + 1] -= 1
+                invalid_offsets_mask = invalid_offsets_mask.cumsum(dim=1) == 0
+                invalid_offsets_mask = invalid_offsets_mask[:, :-1]
+                invalid_offsets_mask[:, 0] = False
+                reg_target = torch.cat([target['lengths'][..., None], target['offsets']], dim=1)
+                reg_target[invalid_offsets_mask] = reg_pred[invalid_offsets_mask]  # apply invalids
+
+            # loss
+            reg_loss += self.smooth_l1_loss(reg_pred, reg_target)
+            cls_loss += self.focal_loss(cls_pred, cls_target).sum() / num_positives
+
+        # Batch mean
+        if self.reduction == 'mean':
+            reg_loss /= batch_size
+            cls_loss /= batch_size
+        elif self.reduction != 'sum':
+            raise NotImplementedError
+
+        total_loss = self.cls_weight * cls_loss + self.reg_weight * reg_loss
+
+        # print(torch.tensor(total_positives))
+        return total_loss, {'total loss': total_loss,
+                            'cls loss': cls_loss,
+                            'reg loss': reg_loss,
+                            'all positives': torch.tensor(total_positives).to(reg_loss.device)}
+
+    @torch.no_grad()
+    def match_proposals_with_targets(self, proposals, labels):
+        # Note: matching is between GT and anchors, not predictions
+        # repeat proposals and targets to generate all combinations
+        num_proposals = proposals.shape[0]
+
+        # Match targets with anchors data format (start len offsets)
+        targets = torch.cat([labels['starts'][..., None], labels['lengths'][..., None], labels['offsets']], dim=1)
+        num_targets = targets.shape[0]
+
+        # pad proposals and target for the valid_offset_mask's trick
+        proposals_pad = proposals.new_zeros(proposals.shape[0], proposals.shape[1] + 1)
+        proposals_pad[:, :-1] = proposals
+        proposals = proposals_pad
+        targets_pad = targets.new_zeros(targets.shape[0], targets.shape[1] + 1)
+        targets_pad[:, :-1] = targets
+        targets = targets_pad
+
+        # repeat interleave [a, b] 2 times gives [a, a, b, b]
+        proposals = torch.repeat_interleave(proposals, num_targets, dim=0)
+        # applying this 2 times on [c, d] gives [c, d, c, d]
+        targets = torch.cat(num_proposals * [targets])
+
+        # get start and the intersection of offsets
+        targets_starts = targets[:, 0] * self.num_strips
+        proposals_starts = proposals[:, 0] * self.num_strips
+        starts = torch.max(targets_starts.float(), proposals_starts).round().long()
+        ends = (targets_starts + targets[:, 1].float() - 1.).round().long()
+        lengths = ends - starts + 1
+        ends[lengths < 0] = starts[lengths < 0] - 1
+        lengths[lengths < 0] = 0  # a negative number here means no intersection, thus no length
+
+        # generate valid offsets mask, which works like this:
+        # start with mask [0, 0, 0, 0, 0]
+        # suppose start = 1
+        # length = 2
+        valid_offsets_mask = targets.new_zeros(targets.shape)
+        all_indices = torch.arange(valid_offsets_mask.shape[0], dtype=torch.long, device=targets.device)
+        #  put a one on index `start`, giving [0, 1, 0, 0, 0]
+        valid_offsets_mask[all_indices, 2 + starts] = 1
+        #  put a -1 on the `end` index, giving [0, 1, 0, -1, 0]
+        valid_offsets_mask[all_indices, 2 + ends + 1] -= 1
+        valid_offsets_mask = valid_offsets_mask.cumsum(dim=1) != 0
+        invalid_offsets_mask = ~valid_offsets_mask
+
+        # compute distance
+        # this compares [ac, ad, bc, bd], i.e., all combinations
+        distances = torch.abs((targets - proposals) * valid_offsets_mask.float()).sum(dim=1) / \
+                    (lengths.float() + 1e-9)  # avoid division by zero
+        distances[lengths == 0] = INFINITY
+        invalid_offsets_mask = invalid_offsets_mask.view(num_proposals, num_targets, invalid_offsets_mask.shape[1])
+        distances = distances.view(num_proposals, num_targets)  # d[i,j] = distance from proposal i to target j
+
+        positives = distances.min(dim=1)[0] < self.t_pos
+        negatives = distances.min(dim=1)[0] > self.t_neg
+
+        if positives.sum() == 0:
+            target_positives_indices = torch.tensor([], device=positives.device, dtype=torch.long)
+        else:
+            target_positives_indices = distances[positives].argmin(dim=1)
+
+        invalid_offsets_mask = invalid_offsets_mask[positives, target_positives_indices]
+        return positives, invalid_offsets_mask[:, :-1], negatives, target_positives_indices

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/torch_loss.py

@@ -0,0 +1,10 @@
+from torch import nn
+
+from .builder import LOSSES
+
+
+@LOSSES.register()
+def torch_loss(torch_loss_class, *args, **kwargs):
+    # A direct mapping
+
+    return getattr(nn, torch_loss_class)(*args, **kwargs)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/weighted_ce_loss.py

@@ -0,0 +1,19 @@
+import torch.nn.functional as F
+
+from ._utils import WeightedLoss
+from .builder import LOSSES
+
+
+# Use a base class that can take list weight instead of Tensor
+@LOSSES.register()
+class WeightedCrossEntropyLoss(WeightedLoss):
+    __constants__ = ['ignore_index', 'reduction']
+
+    def __init__(self, weight=None, size_average=None, ignore_index=-100,
+                 reduce=None, reduction='mean'):
+        super().__init__(weight, size_average, reduce, reduction)
+        self.ignore_index = ignore_index
+
+    def forward(self, inputs, targets):
+        return F.cross_entropy(inputs, targets, weight=self.weight,
+                               ignore_index=self.ignore_index, reduction=self.reduction)