feat: initial HSAP platform
Huaxu Sentinel Active Safety Platform with embedded algorithm code, Docker Compose setup, and vendored dataset scaffolds for clone-and-run. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
172
algorithms/lane_ufld/code/UFLD/utils/common.py
Executable file
172
algorithms/lane_ufld/code/UFLD/utils/common.py
Executable file
@@ -0,0 +1,172 @@
|
||||
import os, argparse
|
||||
from utils.dist_utils import is_main_process, dist_print, DistSummaryWriter
|
||||
from utils.config import Config
|
||||
import torch
|
||||
import time
|
||||
|
||||
def load_checkpoint(path, map_location='cpu'):
|
||||
"""Load legacy UFLD checkpoints (may contain DataParallel); PyTorch 2.6+ needs weights_only=False."""
|
||||
return torch.load(path, map_location=map_location, weights_only=False)
|
||||
|
||||
|
||||
def checkpoint_state_dict(path, map_location='cpu'):
|
||||
"""Return state_dict from a .pth file (handles DataParallel module or raw state_dict)."""
|
||||
ckpt = load_checkpoint(path, map_location=map_location)
|
||||
model = ckpt['model'] if isinstance(ckpt, dict) and 'model' in ckpt else ckpt
|
||||
if isinstance(model, torch.nn.Module):
|
||||
model = model.state_dict()
|
||||
compatible = {}
|
||||
for k, v in model.items():
|
||||
if 'module.' in k:
|
||||
compatible[k[7:]] = v
|
||||
else:
|
||||
compatible[k] = v
|
||||
return compatible
|
||||
|
||||
|
||||
def str2bool(v):
|
||||
if isinstance(v, bool):
|
||||
return v
|
||||
if v.lower() in ('yes', 'true', 't', 'y', '1'):
|
||||
return True
|
||||
elif v.lower() in ('no', 'false', 'f', 'n', '0'):
|
||||
return False
|
||||
else:
|
||||
raise argparse.ArgumentTypeError('Boolean value expected.')
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('config', help = 'path to config file')
|
||||
parser.add_argument('--local_rank', type=int, default=0)
|
||||
|
||||
parser.add_argument('--dataset', default = None, type = str)
|
||||
parser.add_argument('--data_root', default = None, type = str)
|
||||
parser.add_argument('--epoch', default = None, type = int)
|
||||
parser.add_argument('--batch_size', default = None, type = int)
|
||||
parser.add_argument('--optimizer', default = None, type = str)
|
||||
parser.add_argument('--learning_rate', default = None, type = float)
|
||||
parser.add_argument('--weight_decay', default = None, type = float)
|
||||
parser.add_argument('--momentum', default = None, type = float)
|
||||
parser.add_argument('--scheduler', default = None, type = str)
|
||||
parser.add_argument('--steps', default = None, type = int, nargs='+')
|
||||
parser.add_argument('--gamma', default = None, type = float)
|
||||
parser.add_argument('--warmup', default = None, type = str)
|
||||
parser.add_argument('--warmup_iters', default = None, type = int)
|
||||
parser.add_argument('--backbone', default = None, type = str)
|
||||
parser.add_argument('--griding_num', default = None, type = int)
|
||||
parser.add_argument('--use_aux', default = None, type = str2bool)
|
||||
parser.add_argument('--sim_loss_w', default = None, type = float)
|
||||
parser.add_argument('--shp_loss_w', default = None, type = float)
|
||||
parser.add_argument('--note', default = None, type = str)
|
||||
parser.add_argument('--log_path', default = None, type = str)
|
||||
parser.add_argument('--finetune', default = None, type = str)
|
||||
parser.add_argument('--resume', default = None, type = str)
|
||||
parser.add_argument('--test_model', default = None, type = str)
|
||||
parser.add_argument('--test_work_dir', default = None, type = str)
|
||||
parser.add_argument('--num_lanes', default = None, type = int)
|
||||
parser.add_argument('--test_list', default = None, type = str,
|
||||
help='relative path under data_root, e.g. list/test.txt')
|
||||
parser.add_argument('--skip_eval', default = None, type = str2bool,
|
||||
help='only run inference, skip metric evaluation')
|
||||
parser.add_argument('--auto_backup', action='store_true', help='automatically backup current code in the log path')
|
||||
|
||||
return parser
|
||||
|
||||
def merge_config():
|
||||
args = get_args().parse_args()
|
||||
cfg = Config.fromfile(args.config)
|
||||
|
||||
items = ['dataset','data_root','epoch','batch_size','optimizer','learning_rate',
|
||||
'weight_decay','momentum','scheduler','steps','gamma','warmup','warmup_iters',
|
||||
'use_aux','griding_num','backbone','sim_loss_w','shp_loss_w','note','log_path',
|
||||
'finetune','resume', 'test_model','test_work_dir', 'num_lanes', 'test_list', 'skip_eval']
|
||||
for item in items:
|
||||
if getattr(args, item) is not None:
|
||||
dist_print('merge ', item, ' config')
|
||||
setattr(cfg, item, getattr(args, item))
|
||||
return args, cfg
|
||||
|
||||
|
||||
def save_model(net, optimizer, epoch, save_path, distributed):
|
||||
if is_main_process():
|
||||
# model_state_dict = net.state_dict()
|
||||
# state = {'model': model_state_dict, 'optimizer': optimizer.state_dict()}
|
||||
# # state = {'model': net, 'optimizer': optimizer}
|
||||
# assert os.path.exists(save_path)
|
||||
# #model_path = os.path.join(save_path, 'ep%03d.pth' % epoch)
|
||||
# model_path = os.path.join(save_path, 'best.pth')
|
||||
# torch.save(state, model_path)
|
||||
model_state_all = net
|
||||
state = {'model': model_state_all, 'optimizer': optimizer.state_dict(),'epoch':epoch}
|
||||
assert os.path.exists(save_path)
|
||||
model_path = os.path.join(save_path, 'best.pth')
|
||||
torch.save(state, model_path)
|
||||
|
||||
def save_model_pruned_model(model, optimizer, epoch, save_path, distributed):
|
||||
state = {'model': model.state_dict(), 'optimizer': optimizer.state_dict(),'epoch':epoch}
|
||||
assert os.path.exists(save_path)
|
||||
model_path = os.path.join(save_path, 'pruned_model.pth')
|
||||
torch.save(state, model_path)
|
||||
|
||||
def save_model_fine_tune(net, optimizer, epoch, save_path, distributed):
|
||||
if is_main_process():
|
||||
# model_state_dict = net.state_dict()
|
||||
# state = {'model': model_state_dict, 'optimizer': optimizer.state_dict()}
|
||||
# # state = {'model': net, 'optimizer': optimizer}
|
||||
# assert os.path.exists(save_path)
|
||||
# #model_path = os.path.join(save_path, 'ep%03d.pth' % epoch)
|
||||
# model_path = os.path.join(save_path, 'best.pth')
|
||||
# torch.save(state, model_path)
|
||||
model_state_dict = net.state_dict() ##注意此处
|
||||
state = {'model': model_state_dict, 'optimizer': optimizer.state_dict(),'epoch':epoch}
|
||||
assert os.path.exists(save_path)
|
||||
model_path = os.path.join(save_path, 'fine_tune_model.pth')
|
||||
torch.save(state, model_path)
|
||||
|
||||
import pathspec
|
||||
|
||||
|
||||
def cp_projects(auto_backup, to_path):
|
||||
if is_main_process() and auto_backup:
|
||||
with open('./.gitignore','r') as fp:
|
||||
ign = fp.read()
|
||||
ign += '\n.git'
|
||||
spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ign.splitlines())
|
||||
all_files = {os.path.join(root,name) for root,dirs,files in os.walk('./') for name in files}
|
||||
matches = spec.match_files(all_files)
|
||||
matches = set(matches)
|
||||
to_cp_files = all_files - matches
|
||||
dist_print('Copying projects to '+ to_path + ' for backup')
|
||||
t0 = time.time()
|
||||
warning_flag = True
|
||||
for f in to_cp_files:
|
||||
dirs = os.path.join(to_path,'code',os.path.split(f[2:])[0])
|
||||
if not os.path.exists(dirs):
|
||||
os.makedirs(dirs)
|
||||
os.system('cp %s %s'%(f,os.path.join(to_path,'code',f[2:])))
|
||||
elapsed_time = time.time() - t0
|
||||
if elapsed_time > 5 and warning_flag:
|
||||
dist_print('If the program is stuck, it might be copying large files in this directory. please don\'t set --auto_backup. Or please make you working directory clean, i.e, don\'t place large files like dataset, log results under this directory.')
|
||||
warning_flag = False
|
||||
|
||||
|
||||
import datetime,os
|
||||
|
||||
|
||||
def get_work_dir(cfg):
|
||||
now = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
hyper_param_str = '_lr_%1.0e_b_%d' % (cfg.learning_rate, cfg.batch_size)
|
||||
log_path = cfg.log_path or './log'
|
||||
print(log_path, now + hyper_param_str + cfg.note)
|
||||
work_dir = os.path.join(log_path, now + hyper_param_str + cfg.note)
|
||||
return work_dir
|
||||
|
||||
|
||||
def get_logger(work_dir, cfg):
|
||||
logger = DistSummaryWriter(work_dir)
|
||||
config_txt = os.path.join(work_dir, 'cfg.txt')
|
||||
if is_main_process():
|
||||
with open(config_txt, 'w') as fp:
|
||||
fp.write(str(cfg))
|
||||
|
||||
return logger
|
||||
121
algorithms/lane_ufld/code/UFLD/utils/common_bk.py
Executable file
121
algorithms/lane_ufld/code/UFLD/utils/common_bk.py
Executable file
@@ -0,0 +1,121 @@
|
||||
import os, argparse
|
||||
from utils.dist_utils import is_main_process, dist_print, DistSummaryWriter
|
||||
from utils.config import Config
|
||||
import torch
|
||||
import time
|
||||
|
||||
def str2bool(v):
|
||||
if isinstance(v, bool):
|
||||
return v
|
||||
if v.lower() in ('yes', 'true', 't', 'y', '1'):
|
||||
return True
|
||||
elif v.lower() in ('no', 'false', 'f', 'n', '0'):
|
||||
return False
|
||||
else:
|
||||
raise argparse.ArgumentTypeError('Boolean value expected.')
|
||||
|
||||
def get_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('config', help = 'path to config file')
|
||||
parser.add_argument('--local_rank', type=int, default=0)
|
||||
|
||||
parser.add_argument('--dataset', default = None, type = str)
|
||||
parser.add_argument('--data_root', default = None, type = str)
|
||||
parser.add_argument('--epoch', default = None, type = int)
|
||||
parser.add_argument('--batch_size', default = None, type = int)
|
||||
parser.add_argument('--optimizer', default = None, type = str)
|
||||
parser.add_argument('--learning_rate', default = None, type = float)
|
||||
parser.add_argument('--weight_decay', default = None, type = float)
|
||||
parser.add_argument('--momentum', default = None, type = float)
|
||||
parser.add_argument('--scheduler', default = None, type = str)
|
||||
parser.add_argument('--steps', default = None, type = int, nargs='+')
|
||||
parser.add_argument('--gamma', default = None, type = float)
|
||||
parser.add_argument('--warmup', default = None, type = str)
|
||||
parser.add_argument('--warmup_iters', default = None, type = int)
|
||||
parser.add_argument('--backbone', default = None, type = str)
|
||||
parser.add_argument('--griding_num', default = None, type = int)
|
||||
parser.add_argument('--use_aux', default = None, type = str2bool)
|
||||
parser.add_argument('--sim_loss_w', default = None, type = float)
|
||||
parser.add_argument('--shp_loss_w', default = None, type = float)
|
||||
parser.add_argument('--note', default = None, type = str)
|
||||
parser.add_argument('--log_path', default = None, type = str)
|
||||
parser.add_argument('--finetune', default = None, type = str)
|
||||
parser.add_argument('--resume', default = None, type = str)
|
||||
parser.add_argument('--test_model', default = None, type = str)
|
||||
parser.add_argument('--test_work_dir', default = None, type = str)
|
||||
parser.add_argument('--num_lanes', default = None, type = int)
|
||||
parser.add_argument('--auto_backup', action='store_true', help='automatically backup current code in the log path')
|
||||
|
||||
return parser
|
||||
|
||||
def merge_config():
|
||||
args = get_args().parse_args()
|
||||
cfg = Config.fromfile(args.config)
|
||||
|
||||
items = ['dataset','data_root','epoch','batch_size','optimizer','learning_rate',
|
||||
'weight_decay','momentum','scheduler','steps','gamma','warmup','warmup_iters',
|
||||
'use_aux','griding_num','backbone','sim_loss_w','shp_loss_w','note','log_path',
|
||||
'finetune','resume', 'test_model','test_work_dir', 'num_lanes']
|
||||
for item in items:
|
||||
if getattr(args, item) is not None:
|
||||
dist_print('merge ', item, ' config')
|
||||
setattr(cfg, item, getattr(args, item))
|
||||
return args, cfg
|
||||
|
||||
|
||||
def save_model(net, optimizer, epoch, save_path, distributed):
|
||||
if is_main_process():
|
||||
model_state_dict = net.state_dict()
|
||||
state = {'model': model_state_dict, 'optimizer': optimizer.state_dict()}
|
||||
# state = {'model': net, 'optimizer': optimizer}
|
||||
assert os.path.exists(save_path)
|
||||
model_path = os.path.join(save_path, 'ep%03d.pth' % epoch)
|
||||
torch.save(state, model_path)
|
||||
|
||||
|
||||
import pathspec
|
||||
|
||||
|
||||
def cp_projects(auto_backup, to_path):
|
||||
if is_main_process() and auto_backup:
|
||||
with open('./.gitignore','r') as fp:
|
||||
ign = fp.read()
|
||||
ign += '\n.git'
|
||||
spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, ign.splitlines())
|
||||
all_files = {os.path.join(root,name) for root,dirs,files in os.walk('./') for name in files}
|
||||
matches = spec.match_files(all_files)
|
||||
matches = set(matches)
|
||||
to_cp_files = all_files - matches
|
||||
dist_print('Copying projects to '+ to_path + ' for backup')
|
||||
t0 = time.time()
|
||||
warning_flag = True
|
||||
for f in to_cp_files:
|
||||
dirs = os.path.join(to_path,'code',os.path.split(f[2:])[0])
|
||||
if not os.path.exists(dirs):
|
||||
os.makedirs(dirs)
|
||||
os.system('cp %s %s'%(f,os.path.join(to_path,'code',f[2:])))
|
||||
elapsed_time = time.time() - t0
|
||||
if elapsed_time > 5 and warning_flag:
|
||||
dist_print('If the program is stuck, it might be copying large files in this directory. please don\'t set --auto_backup. Or please make you working directory clean, i.e, don\'t place large files like dataset, log results under this directory.')
|
||||
warning_flag = False
|
||||
|
||||
|
||||
import datetime,os
|
||||
|
||||
|
||||
def get_work_dir(cfg):
|
||||
now = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
hyper_param_str = '_lr_%1.0e_b_%d' % (cfg.learning_rate, cfg.batch_size)
|
||||
print(cfg.log_path, now + hyper_param_str + cfg.note)
|
||||
work_dir = os.path.join(cfg.log_path, now + hyper_param_str + cfg.note)
|
||||
return work_dir
|
||||
|
||||
|
||||
def get_logger(work_dir, cfg):
|
||||
logger = DistSummaryWriter(work_dir)
|
||||
config_txt = os.path.join(work_dir, 'cfg.txt')
|
||||
if is_main_process():
|
||||
with open(config_txt, 'w') as fp:
|
||||
fp.write(str(cfg))
|
||||
|
||||
return logger
|
||||
352
algorithms/lane_ufld/code/UFLD/utils/config.py
Executable file
352
algorithms/lane_ufld/code/UFLD/utils/config.py
Executable file
@@ -0,0 +1,352 @@
|
||||
import json
|
||||
import os.path as osp
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from argparse import Action, ArgumentParser
|
||||
from collections import abc
|
||||
from importlib import import_module
|
||||
|
||||
from addict import Dict
|
||||
|
||||
|
||||
BASE_KEY = '_base_'
|
||||
DELETE_KEY = '_delete_'
|
||||
|
||||
|
||||
class ConfigDict(Dict):
|
||||
|
||||
def __missing__(self, name):
|
||||
raise KeyError(name)
|
||||
|
||||
def __getattr__(self, name):
|
||||
try:
|
||||
value = super(ConfigDict, self).__getattr__(name)
|
||||
except KeyError:
|
||||
ex = AttributeError(f"'{self.__class__.__name__}' object has no "
|
||||
f"attribute '{name}'")
|
||||
except Exception as e:
|
||||
ex = e
|
||||
else:
|
||||
return value
|
||||
raise ex
|
||||
|
||||
|
||||
def add_args(parser, cfg, prefix=''):
|
||||
for k, v in cfg.items():
|
||||
if isinstance(v, str):
|
||||
parser.add_argument('--' + prefix + k)
|
||||
elif isinstance(v, int):
|
||||
parser.add_argument('--' + prefix + k, type=int)
|
||||
elif isinstance(v, float):
|
||||
parser.add_argument('--' + prefix + k, type=float)
|
||||
elif isinstance(v, bool):
|
||||
parser.add_argument('--' + prefix + k, action='store_true')
|
||||
elif isinstance(v, dict):
|
||||
add_args(parser, v, prefix + k + '.')
|
||||
elif isinstance(v, abc.Iterable):
|
||||
parser.add_argument('--' + prefix + k, type=type(v[0]), nargs='+')
|
||||
else:
|
||||
print(f'cannot parse key {prefix + k} of type {type(v)}')
|
||||
return parser
|
||||
|
||||
|
||||
class Config(object):
|
||||
"""A facility for config and config files.
|
||||
It supports common file formats as configs: python/json/yaml. The interface
|
||||
is the same as a dict object and also allows access config values as
|
||||
attributes.
|
||||
Example:
|
||||
>>> cfg = Config(dict(a=1, b=dict(b1=[0, 1])))
|
||||
>>> cfg.a
|
||||
1
|
||||
>>> cfg.b
|
||||
{'b1': [0, 1]}
|
||||
>>> cfg.b.b1
|
||||
[0, 1]
|
||||
>>> cfg = Config.fromfile('tests/data/config/a.py')
|
||||
>>> cfg.filename
|
||||
"/home/kchen/projects/mmcv/tests/data/config/a.py"
|
||||
>>> cfg.item4
|
||||
'test'
|
||||
>>> cfg
|
||||
"Config [path: /home/kchen/projects/mmcv/tests/data/config/a.py]: "
|
||||
"{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}"
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _file2dict(filename):
|
||||
filename = osp.abspath(osp.expanduser(filename))
|
||||
if filename.endswith('.py'):
|
||||
with tempfile.TemporaryDirectory() as temp_config_dir:
|
||||
temp_config_file = tempfile.NamedTemporaryFile(
|
||||
dir=temp_config_dir, suffix='.py')
|
||||
temp_config_name = osp.basename(temp_config_file.name)
|
||||
# close temp file
|
||||
temp_config_file.close()
|
||||
shutil.copyfile(filename,
|
||||
osp.join(temp_config_dir, temp_config_name))
|
||||
temp_module_name = osp.splitext(temp_config_name)[0]
|
||||
sys.path.insert(0, temp_config_dir)
|
||||
mod = import_module(temp_module_name)
|
||||
sys.path.pop(0)
|
||||
cfg_dict = {
|
||||
name: value
|
||||
for name, value in mod.__dict__.items()
|
||||
if not name.startswith('__')
|
||||
}
|
||||
# delete imported module
|
||||
del sys.modules[temp_module_name]
|
||||
|
||||
elif filename.endswith(('.yml', '.yaml', '.json')):
|
||||
import mmcv
|
||||
cfg_dict = mmcv.load(filename)
|
||||
else:
|
||||
raise IOError('Only py/yml/yaml/json type are supported now!')
|
||||
|
||||
cfg_text = filename + '\n'
|
||||
with open(filename, 'r') as f:
|
||||
cfg_text += f.read()
|
||||
|
||||
if BASE_KEY in cfg_dict:
|
||||
cfg_dir = osp.dirname(filename)
|
||||
base_filename = cfg_dict.pop(BASE_KEY)
|
||||
base_filename = base_filename if isinstance(
|
||||
base_filename, list) else [base_filename]
|
||||
|
||||
cfg_dict_list = list()
|
||||
cfg_text_list = list()
|
||||
for f in base_filename:
|
||||
_cfg_dict, _cfg_text = Config._file2dict(osp.join(cfg_dir, f))
|
||||
cfg_dict_list.append(_cfg_dict)
|
||||
cfg_text_list.append(_cfg_text)
|
||||
|
||||
base_cfg_dict = dict()
|
||||
for c in cfg_dict_list:
|
||||
if len(base_cfg_dict.keys() & c.keys()) > 0:
|
||||
raise KeyError('Duplicate key is not allowed among bases')
|
||||
base_cfg_dict.update(c)
|
||||
|
||||
base_cfg_dict = Config._merge_a_into_b(cfg_dict, base_cfg_dict)
|
||||
cfg_dict = base_cfg_dict
|
||||
|
||||
# merge cfg_text
|
||||
cfg_text_list.append(cfg_text)
|
||||
cfg_text = '\n'.join(cfg_text_list)
|
||||
|
||||
return cfg_dict, cfg_text
|
||||
|
||||
@staticmethod
|
||||
def _merge_a_into_b(a, b):
|
||||
# merge dict `a` into dict `b` (non-inplace). values in `a` will
|
||||
# overwrite `b`.
|
||||
# copy first to avoid inplace modification
|
||||
b = b.copy()
|
||||
for k, v in a.items():
|
||||
if isinstance(v, dict) and k in b and not v.pop(DELETE_KEY, False):
|
||||
if not isinstance(b[k], dict):
|
||||
raise TypeError(
|
||||
f'{k}={v} in child config cannot inherit from base '
|
||||
f'because {k} is a dict in the child config but is of '
|
||||
f'type {type(b[k])} in base config. You may set '
|
||||
f'`{DELETE_KEY}=True` to ignore the base config')
|
||||
b[k] = Config._merge_a_into_b(v, b[k])
|
||||
else:
|
||||
b[k] = v
|
||||
return b
|
||||
|
||||
@staticmethod
|
||||
def fromfile(filename):
|
||||
cfg_dict, cfg_text = Config._file2dict(filename)
|
||||
return Config(cfg_dict, cfg_text=cfg_text, filename=filename)
|
||||
|
||||
@staticmethod
|
||||
def auto_argparser(description=None):
|
||||
"""Generate argparser from config file automatically (experimental)
|
||||
"""
|
||||
partial_parser = ArgumentParser(description=description)
|
||||
partial_parser.add_argument('config', help='config file path')
|
||||
cfg_file = partial_parser.parse_known_args()[0].config
|
||||
cfg = Config.fromfile(cfg_file)
|
||||
parser = ArgumentParser(description=description)
|
||||
parser.add_argument('config', help='config file path')
|
||||
add_args(parser, cfg)
|
||||
return parser, cfg
|
||||
|
||||
def __init__(self, cfg_dict=None, cfg_text=None, filename=None):
|
||||
if cfg_dict is None:
|
||||
cfg_dict = dict()
|
||||
elif not isinstance(cfg_dict, dict):
|
||||
raise TypeError('cfg_dict must be a dict, but '
|
||||
f'got {type(cfg_dict)}')
|
||||
|
||||
super(Config, self).__setattr__('_cfg_dict', ConfigDict(cfg_dict))
|
||||
super(Config, self).__setattr__('_filename', filename)
|
||||
if cfg_text:
|
||||
text = cfg_text
|
||||
elif filename:
|
||||
with open(filename, 'r') as f:
|
||||
text = f.read()
|
||||
else:
|
||||
text = ''
|
||||
super(Config, self).__setattr__('_text', text)
|
||||
|
||||
@property
|
||||
def filename(self):
|
||||
return self._filename
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
return self._text
|
||||
|
||||
@property
|
||||
def pretty_text(self):
|
||||
|
||||
indent = 4
|
||||
|
||||
def _indent(s_, num_spaces):
|
||||
s = s_.split('\n')
|
||||
if len(s) == 1:
|
||||
return s_
|
||||
first = s.pop(0)
|
||||
s = [(num_spaces * ' ') + line for line in s]
|
||||
s = '\n'.join(s)
|
||||
s = first + '\n' + s
|
||||
return s
|
||||
|
||||
def _format_basic_types(k, v):
|
||||
if isinstance(v, str):
|
||||
v_str = f"'{v}'"
|
||||
else:
|
||||
v_str = str(v)
|
||||
attr_str = f'{str(k)}={v_str}'
|
||||
attr_str = _indent(attr_str, indent)
|
||||
|
||||
return attr_str
|
||||
|
||||
def _format_list(k, v):
|
||||
# check if all items in the list are dict
|
||||
if all(isinstance(_, dict) for _ in v):
|
||||
v_str = '[\n'
|
||||
v_str += '\n'.join(
|
||||
f'dict({_indent(_format_dict(v_), indent)}),'
|
||||
for v_ in v).rstrip(',')
|
||||
attr_str = f'{str(k)}={v_str}'
|
||||
attr_str = _indent(attr_str, indent) + ']'
|
||||
else:
|
||||
attr_str = _format_basic_types(k, v)
|
||||
return attr_str
|
||||
|
||||
def _format_dict(d, outest_level=False):
|
||||
r = ''
|
||||
s = []
|
||||
for idx, (k, v) in enumerate(d.items()):
|
||||
is_last = idx >= len(d) - 1
|
||||
end = '' if outest_level or is_last else ','
|
||||
if isinstance(v, dict):
|
||||
v_str = '\n' + _format_dict(v)
|
||||
attr_str = f'{str(k)}=dict({v_str}'
|
||||
attr_str = _indent(attr_str, indent) + ')' + end
|
||||
elif isinstance(v, list):
|
||||
attr_str = _format_list(k, v) + end
|
||||
else:
|
||||
attr_str = _format_basic_types(k, v) + end
|
||||
|
||||
s.append(attr_str)
|
||||
r += '\n'.join(s)
|
||||
return r
|
||||
|
||||
cfg_dict = self._cfg_dict.to_dict()
|
||||
text = _format_dict(cfg_dict, outest_level=True)
|
||||
|
||||
return text
|
||||
|
||||
def __repr__(self):
|
||||
return f'Config (path: {self.filename}): {self._cfg_dict.__repr__()}'
|
||||
|
||||
def __len__(self):
|
||||
return len(self._cfg_dict)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._cfg_dict, name)
|
||||
|
||||
def __getitem__(self, name):
|
||||
return self._cfg_dict.__getitem__(name)
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
if isinstance(value, dict):
|
||||
value = ConfigDict(value)
|
||||
self._cfg_dict.__setattr__(name, value)
|
||||
|
||||
def __setitem__(self, name, value):
|
||||
if isinstance(value, dict):
|
||||
value = ConfigDict(value)
|
||||
self._cfg_dict.__setitem__(name, value)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._cfg_dict)
|
||||
|
||||
def dump(self):
|
||||
cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
|
||||
format_text = json.dumps(cfg_dict, indent=2)
|
||||
return format_text
|
||||
|
||||
def merge_from_dict(self, options):
|
||||
"""Merge list into cfg_dict
|
||||
Merge the dict parsed by MultipleKVAction into this cfg.
|
||||
Examples:
|
||||
>>> options = {'model.backbone.depth': 50,
|
||||
... 'model.backbone.with_cp':True}
|
||||
>>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet'))))
|
||||
>>> cfg.merge_from_dict(options)
|
||||
>>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
|
||||
>>> assert cfg_dict == dict(
|
||||
... model=dict(backbone=dict(depth=50, with_cp=True)))
|
||||
Args:
|
||||
options (dict): dict of configs to merge from.
|
||||
"""
|
||||
option_cfg_dict = {}
|
||||
for full_key, v in options.items():
|
||||
d = option_cfg_dict
|
||||
key_list = full_key.split('.')
|
||||
for subkey in key_list[:-1]:
|
||||
d.setdefault(subkey, ConfigDict())
|
||||
d = d[subkey]
|
||||
subkey = key_list[-1]
|
||||
d[subkey] = v
|
||||
|
||||
cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
|
||||
super(Config, self).__setattr__(
|
||||
'_cfg_dict', Config._merge_a_into_b(option_cfg_dict, cfg_dict))
|
||||
|
||||
|
||||
class DictAction(Action):
|
||||
"""
|
||||
argparse action to split an argument into KEY=VALUE form
|
||||
on the first = and append to a dictionary. List options should
|
||||
be passed as comma separated values, i.e KEY=V1,V2,V3
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _parse_int_float_bool(val):
|
||||
try:
|
||||
return int(val)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return float(val)
|
||||
except ValueError:
|
||||
pass
|
||||
if val.lower() in ['true', 'false']:
|
||||
return True if val.lower() == 'true' else False
|
||||
return val
|
||||
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
options = {}
|
||||
for kv in values:
|
||||
key, val = kv.split('=', maxsplit=1)
|
||||
val = [self._parse_int_float_bool(v) for v in val.split(',')]
|
||||
if len(val) == 1:
|
||||
val = val[0]
|
||||
options[key] = val
|
||||
setattr(namespace, self.dest, options)
|
||||
163
algorithms/lane_ufld/code/UFLD/utils/dataset_packs.py
Normal file
163
algorithms/lane_ufld/code/UFLD/utils/dataset_packs.py
Normal file
@@ -0,0 +1,163 @@
|
||||
"""Resolve train_list from config train_packs (DATASET, DATASET-A, ...)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from utils.dist_utils import dist_print, is_main_process
|
||||
|
||||
|
||||
def parse_gt_line(line: str) -> tuple[str, str] | None:
|
||||
parts = line.strip().split()
|
||||
if len(parts) < 2:
|
||||
return None
|
||||
return parts[0].lstrip("/"), parts[1].lstrip("/")
|
||||
|
||||
|
||||
def apply_pack_prefix(img: str, msk: str, prefix: str) -> tuple[str, str]:
|
||||
if not prefix:
|
||||
return img, msk
|
||||
if not img.startswith(prefix):
|
||||
img = prefix + img
|
||||
if not msk.startswith(prefix):
|
||||
msk = prefix + msk
|
||||
return img, msk
|
||||
|
||||
|
||||
def load_registry(data_root: Path) -> dict:
|
||||
path = data_root / "datasets_registry.json"
|
||||
if not path.is_file():
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def resolve_pack_dir(pack: str, data_root: Path, registry: dict) -> str:
|
||||
"""Map config name (e.g. DATASET-A) to directory name under data_root."""
|
||||
aliases = registry.get("aliases", {})
|
||||
if pack in aliases:
|
||||
pack = aliases[pack]
|
||||
pack_dirs = registry.get("pack_dirs", {})
|
||||
if pack in pack_dirs:
|
||||
pack = pack_dirs[pack]
|
||||
pack_path = data_root / pack
|
||||
if not pack_path.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"pack directory not found: {pack_path} (config train_packs entry: {pack!r})"
|
||||
)
|
||||
return pack
|
||||
|
||||
|
||||
def pack_list_path(data_root: Path, pack_dir: str, list_name: str) -> Path:
|
||||
p = data_root / pack_dir / list_name
|
||||
if not p.is_file():
|
||||
raise FileNotFoundError(f"pack list not found: {p}")
|
||||
return p
|
||||
|
||||
|
||||
def merge_pack_lists(
|
||||
data_root: Path,
|
||||
pack_dirs: list[str],
|
||||
list_name: str,
|
||||
out_path: Path,
|
||||
*,
|
||||
validate: bool = True,
|
||||
) -> int:
|
||||
merged: list[tuple[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for pack_dir in pack_dirs:
|
||||
prefix = f"{pack_dir}/"
|
||||
list_path = pack_list_path(data_root, pack_dir, list_name)
|
||||
for line in list_path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
parsed = parse_gt_line(line)
|
||||
if not parsed:
|
||||
continue
|
||||
img, msk = apply_pack_prefix(parsed[0], parsed[1], prefix)
|
||||
if img in seen:
|
||||
continue
|
||||
seen.add(img)
|
||||
if validate:
|
||||
if not (data_root / img).is_file():
|
||||
raise FileNotFoundError(f"missing image: {data_root / img}")
|
||||
if not (data_root / msk).is_file():
|
||||
raise FileNotFoundError(f"missing mask: {data_root / msk}")
|
||||
merged.append((img, msk))
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text("\n".join(f"{a} {b}" for a, b in merged) + "\n", encoding="utf-8")
|
||||
return len(merged)
|
||||
|
||||
|
||||
def merged_list_basename(pack_dirs: list[str]) -> str:
|
||||
safe = "__".join(p.replace("/", "_") for p in pack_dirs)
|
||||
if len(safe) > 180:
|
||||
safe = safe[:180]
|
||||
return f"train__{safe}.txt"
|
||||
|
||||
|
||||
def resolve_train_list(cfg) -> str:
|
||||
"""
|
||||
Return train list path relative to cfg.data_root.
|
||||
|
||||
- If cfg.train_packs is set: merge packs and return lists_merged/... path.
|
||||
- Else: use cfg.train_list (default list/train_gt.txt).
|
||||
"""
|
||||
train_packs = getattr(cfg, "train_packs", None)
|
||||
if not train_packs:
|
||||
return getattr(cfg, "train_list", "list/train_gt.txt")
|
||||
|
||||
per_pack_lists: dict = {}
|
||||
if isinstance(train_packs, str):
|
||||
train_packs = [p.strip() for p in train_packs.split(",") if p.strip()]
|
||||
elif isinstance(train_packs, dict):
|
||||
per_pack_lists = dict(train_packs)
|
||||
train_packs = list(per_pack_lists.keys())
|
||||
else:
|
||||
train_packs = list(train_packs)
|
||||
|
||||
data_root = Path(cfg.data_root).resolve()
|
||||
registry = load_registry(data_root)
|
||||
pack_dirs = [resolve_pack_dir(p, data_root, registry) for p in train_packs]
|
||||
|
||||
list_name = getattr(cfg, "pack_list_name", "list/train_gt.txt")
|
||||
if per_pack_lists:
|
||||
# merge with different list per pack — sequential merge
|
||||
merged_dir = Path(getattr(cfg, "merged_list_dir", "lists_merged"))
|
||||
out_name = getattr(cfg, "merged_train_list", None) or merged_list_basename(pack_dirs)
|
||||
out_path = data_root / merged_dir / out_name
|
||||
if is_main_process():
|
||||
merged: list[tuple[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for pack, pack_dir in zip(train_packs, pack_dirs):
|
||||
prefix = f"{pack_dir}/"
|
||||
rel_list = per_pack_lists.get(pack, list_name)
|
||||
list_path = pack_list_path(data_root, pack_dir, rel_list)
|
||||
for line in list_path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
parsed = parse_gt_line(line)
|
||||
if not parsed:
|
||||
continue
|
||||
img, msk = apply_pack_prefix(parsed[0], parsed[1], prefix)
|
||||
if img in seen:
|
||||
continue
|
||||
seen.add(img)
|
||||
merged.append((img, msk))
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text("\n".join(f"{a} {b}" for a, b in merged) + "\n", encoding="utf-8")
|
||||
dist_print(f"merged {len(merged)} samples -> {out_path}")
|
||||
return str((merged_dir / out_name).as_posix())
|
||||
|
||||
merged_dir = Path(getattr(cfg, "merged_list_dir", "lists_merged"))
|
||||
out_name = getattr(cfg, "merged_train_list", None) or merged_list_basename(pack_dirs)
|
||||
out_rel = (merged_dir / out_name).as_posix()
|
||||
out_path = data_root / merged_dir / out_name
|
||||
|
||||
force = getattr(cfg, "remerge_train_list", False)
|
||||
if is_main_process():
|
||||
if force or not out_path.is_file():
|
||||
n = merge_pack_lists(data_root, pack_dirs, list_name, out_path, validate=True)
|
||||
dist_print(f"train_packs {train_packs} -> {n} samples, list={out_rel}")
|
||||
else:
|
||||
dist_print(f"reuse merged list: {out_rel}")
|
||||
return out_rel
|
||||
173
algorithms/lane_ufld/code/UFLD/utils/dist_utils.py
Executable file
173
algorithms/lane_ufld/code/UFLD/utils/dist_utils.py
Executable file
@@ -0,0 +1,173 @@
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import pickle
|
||||
|
||||
|
||||
def get_world_size():
|
||||
if not dist.is_available():
|
||||
return 1
|
||||
if not dist.is_initialized():
|
||||
return 1
|
||||
return dist.get_world_size()
|
||||
|
||||
|
||||
def to_python_float(t):
|
||||
if hasattr(t, 'item'):
|
||||
return t.item()
|
||||
else:
|
||||
return t[0]
|
||||
|
||||
|
||||
def get_rank():
|
||||
if not dist.is_available():
|
||||
return 0
|
||||
if not dist.is_initialized():
|
||||
return 0
|
||||
return dist.get_rank()
|
||||
|
||||
|
||||
def is_main_process():
|
||||
return get_rank() == 0
|
||||
|
||||
|
||||
def can_log():
|
||||
return is_main_process()
|
||||
|
||||
|
||||
def dist_print(*args, **kwargs):
|
||||
if can_log():
|
||||
print(*args, **kwargs)
|
||||
|
||||
|
||||
def synchronize():
|
||||
"""
|
||||
Helper function to synchronize (barrier) among all processes when
|
||||
using distributed training
|
||||
"""
|
||||
if not dist.is_available():
|
||||
return
|
||||
if not dist.is_initialized():
|
||||
return
|
||||
world_size = dist.get_world_size()
|
||||
if world_size == 1:
|
||||
return
|
||||
dist.barrier()
|
||||
|
||||
def dist_cat_reduce_tensor(tensor):
|
||||
if not dist.is_available():
|
||||
return tensor
|
||||
if not dist.is_initialized():
|
||||
return tensor
|
||||
# dist_print(tensor)
|
||||
rt = tensor.clone()
|
||||
all_list = [torch.zeros_like(tensor) for _ in range(get_world_size())]
|
||||
dist.all_gather(all_list,rt)
|
||||
# dist_print(all_list[0][1],all_list[1][1],all_list[2][1],all_list[3][1])
|
||||
# dist_print(all_list[0][2],all_list[1][2],all_list[2][2],all_list[3][2])
|
||||
# dist_print(all_list[0][3],all_list[1][3],all_list[2][3],all_list[3][3])
|
||||
# dist_print(all_list[0].shape)
|
||||
return torch.cat(all_list,dim = 0)
|
||||
|
||||
def dist_sum_reduce_tensor(tensor):
|
||||
if not dist.is_available():
|
||||
return tensor
|
||||
if not dist.is_initialized():
|
||||
return tensor
|
||||
if not isinstance(tensor, torch.Tensor):
|
||||
return tensor
|
||||
rt = tensor.clone()
|
||||
dist.all_reduce(rt, op=dist.reduce_op.SUM)
|
||||
return rt
|
||||
|
||||
|
||||
def dist_mean_reduce_tensor(tensor):
|
||||
rt = dist_sum_reduce_tensor(tensor)
|
||||
rt /= get_world_size()
|
||||
return rt
|
||||
|
||||
|
||||
def all_gather(data):
|
||||
"""
|
||||
Run all_gather on arbitrary picklable data (not necessarily tensors)
|
||||
Args:
|
||||
data: any picklable object
|
||||
Returns:
|
||||
list[data]: list of data gathered from each rank
|
||||
"""
|
||||
world_size = get_world_size()
|
||||
if world_size == 1:
|
||||
return [data]
|
||||
|
||||
# serialized to a Tensor
|
||||
buffer = pickle.dumps(data)
|
||||
storage = torch.ByteStorage.from_buffer(buffer)
|
||||
tensor = torch.ByteTensor(storage).to("cuda")
|
||||
|
||||
# obtain Tensor size of each rank
|
||||
local_size = torch.LongTensor([tensor.numel()]).to("cuda")
|
||||
size_list = [torch.LongTensor([0]).to("cuda") for _ in range(world_size)]
|
||||
dist.all_gather(size_list, local_size)
|
||||
size_list = [int(size.item()) for size in size_list]
|
||||
max_size = max(size_list)
|
||||
|
||||
# receiving Tensor from all ranks
|
||||
# we pad the tensor because torch all_gather does not support
|
||||
# gathering tensors of different shapes
|
||||
tensor_list = []
|
||||
for _ in size_list:
|
||||
tensor_list.append(torch.ByteTensor(size=(max_size,)).to("cuda"))
|
||||
if local_size != max_size:
|
||||
padding = torch.ByteTensor(size=(max_size - local_size,)).to("cuda")
|
||||
tensor = torch.cat((tensor, padding), dim=0)
|
||||
dist.all_gather(tensor_list, tensor)
|
||||
|
||||
data_list = []
|
||||
for size, tensor in zip(size_list, tensor_list):
|
||||
buffer = tensor.cpu().numpy().tobytes()[:size]
|
||||
data_list.append(pickle.loads(buffer))
|
||||
|
||||
return data_list
|
||||
|
||||
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
|
||||
|
||||
class DistSummaryWriter(SummaryWriter):
|
||||
def __init__(self, *args, **kwargs):
|
||||
if can_log():
|
||||
super(DistSummaryWriter, self).__init__(*args, **kwargs)
|
||||
|
||||
def add_scalar(self, *args, **kwargs):
|
||||
if can_log():
|
||||
super(DistSummaryWriter, self).add_scalar(*args, **kwargs)
|
||||
|
||||
def add_figure(self, *args, **kwargs):
|
||||
if can_log():
|
||||
super(DistSummaryWriter, self).add_figure(*args, **kwargs)
|
||||
|
||||
def add_graph(self, *args, **kwargs):
|
||||
if can_log():
|
||||
super(DistSummaryWriter, self).add_graph(*args, **kwargs)
|
||||
|
||||
def add_histogram(self, *args, **kwargs):
|
||||
if can_log():
|
||||
super(DistSummaryWriter, self).add_histogram(*args, **kwargs)
|
||||
|
||||
def add_image(self, *args, **kwargs):
|
||||
if can_log():
|
||||
super(DistSummaryWriter, self).add_image(*args, **kwargs)
|
||||
|
||||
def close(self):
|
||||
if can_log():
|
||||
super(DistSummaryWriter, self).close()
|
||||
|
||||
|
||||
import tqdm
|
||||
|
||||
|
||||
def dist_tqdm(obj, *args, **kwargs):
|
||||
if can_log():
|
||||
return tqdm.tqdm(obj, *args, **kwargs)
|
||||
else:
|
||||
return obj
|
||||
|
||||
129
algorithms/lane_ufld/code/UFLD/utils/factory.py
Executable file
129
algorithms/lane_ufld/code/UFLD/utils/factory.py
Executable file
@@ -0,0 +1,129 @@
|
||||
from utils.loss import SoftmaxFocalLoss, ParsingRelationLoss, ParsingRelationDis
|
||||
from utils.metrics import MultiLabelAcc, AccTopk, Metric_mIoU
|
||||
from utils.dist_utils import DistSummaryWriter
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def get_optimizer(net,cfg):
|
||||
training_params = filter(lambda p: p.requires_grad, net.parameters())
|
||||
if cfg.optimizer == 'Adam':
|
||||
optimizer = torch.optim.AdamW(training_params, lr=cfg.learning_rate, weight_decay=cfg.weight_decay)
|
||||
elif cfg.optimizer == 'SGD':
|
||||
optimizer = torch.optim.SGD(training_params, lr=cfg.learning_rate, momentum=cfg.momentum,
|
||||
weight_decay=cfg.weight_decay)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
return optimizer
|
||||
|
||||
def get_scheduler(optimizer, cfg, iters_per_epoch):
|
||||
if cfg.scheduler == 'multi':
|
||||
scheduler = MultiStepLR(optimizer, cfg.steps, cfg.gamma, iters_per_epoch, cfg.warmup, iters_per_epoch if cfg.warmup_iters is None else cfg.warmup_iters)
|
||||
elif cfg.scheduler == 'cos':
|
||||
scheduler = CosineAnnealingLR(optimizer, cfg.epoch * iters_per_epoch, eta_min = 0, warmup = cfg.warmup, warmup_iters = cfg.warmup_iters)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
return scheduler
|
||||
|
||||
def get_loss_dict(cfg):
|
||||
|
||||
if cfg.use_aux:
|
||||
loss_dict = {
|
||||
'name': ['cls_loss', 'relation_loss', 'aux_loss', 'relation_dis'],
|
||||
'op': [SoftmaxFocalLoss(2), ParsingRelationLoss(), torch.nn.CrossEntropyLoss(), ParsingRelationDis()],
|
||||
'weight': [1.0, cfg.sim_loss_w, 1.0, cfg.shp_loss_w],
|
||||
'data_src': [('cls_out', 'cls_label'), ('cls_out',), ('seg_out', 'seg_label'), ('cls_out',)]
|
||||
}
|
||||
else:
|
||||
loss_dict = {
|
||||
'name': ['cls_loss', 'relation_loss', 'relation_dis'],
|
||||
'op': [SoftmaxFocalLoss(2), ParsingRelationLoss(), ParsingRelationDis()],
|
||||
'weight': [1.0, cfg.sim_loss_w, cfg.shp_loss_w],
|
||||
'data_src': [('cls_out', 'cls_label'), ('cls_out',), ('cls_out',)]
|
||||
}
|
||||
|
||||
return loss_dict
|
||||
|
||||
def get_metric_dict(cfg):
|
||||
|
||||
if cfg.use_aux:
|
||||
metric_dict = {
|
||||
'name': ['top1', 'top2', 'top3', 'iou'],
|
||||
'op': [MultiLabelAcc(), AccTopk(cfg.griding_num, 2), AccTopk(cfg.griding_num, 3), Metric_mIoU(cfg.num_lanes+1)],
|
||||
'data_src': [('cls_out', 'cls_label'), ('cls_out', 'cls_label'), ('cls_out', 'cls_label'), ('seg_out', 'seg_label')]
|
||||
}
|
||||
else:
|
||||
metric_dict = {
|
||||
'name': ['top1', 'top2', 'top3'],
|
||||
'op': [MultiLabelAcc(), AccTopk(cfg.griding_num, 2), AccTopk(cfg.griding_num, 3)],
|
||||
'data_src': [('cls_out', 'cls_label'), ('cls_out', 'cls_label'), ('cls_out', 'cls_label')]
|
||||
}
|
||||
|
||||
|
||||
return metric_dict
|
||||
|
||||
|
||||
class MultiStepLR:
|
||||
def __init__(self, optimizer, steps, gamma = 0.1, iters_per_epoch = None, warmup = None, warmup_iters = None):
|
||||
self.warmup = warmup
|
||||
self.warmup_iters = warmup_iters
|
||||
self.optimizer = optimizer
|
||||
self.steps = steps
|
||||
self.steps.sort()
|
||||
self.gamma = gamma
|
||||
self.iters_per_epoch = iters_per_epoch
|
||||
self.iters = 0
|
||||
self.base_lr = [group['lr'] for group in optimizer.param_groups]
|
||||
|
||||
def step(self, external_iter = None):
|
||||
self.iters += 1
|
||||
if external_iter is not None:
|
||||
self.iters = external_iter
|
||||
if self.warmup == 'linear' and self.iters < self.warmup_iters:
|
||||
rate = self.iters / self.warmup_iters
|
||||
for group, lr in zip(self.optimizer.param_groups, self.base_lr):
|
||||
group['lr'] = lr * rate
|
||||
return
|
||||
|
||||
# multi policy
|
||||
if self.iters % self.iters_per_epoch == 0:
|
||||
epoch = int(self.iters / self.iters_per_epoch)
|
||||
power = -1
|
||||
for i, st in enumerate(self.steps):
|
||||
if epoch < st:
|
||||
power = i
|
||||
break
|
||||
if power == -1:
|
||||
power = len(self.steps)
|
||||
# print(self.iters, self.iters_per_epoch, self.steps, power)
|
||||
|
||||
for group, lr in zip(self.optimizer.param_groups, self.base_lr):
|
||||
group['lr'] = lr * (self.gamma ** power)
|
||||
import math
|
||||
class CosineAnnealingLR:
|
||||
def __init__(self, optimizer, T_max , eta_min = 0, warmup = None, warmup_iters = None):
|
||||
self.warmup = warmup
|
||||
self.warmup_iters = warmup_iters
|
||||
self.optimizer = optimizer
|
||||
self.T_max = T_max
|
||||
self.eta_min = eta_min
|
||||
|
||||
self.iters = 0
|
||||
self.base_lr = [group['lr'] for group in optimizer.param_groups]
|
||||
|
||||
def step(self, external_iter = None):
|
||||
self.iters += 1
|
||||
if external_iter is not None:
|
||||
self.iters = external_iter
|
||||
if self.warmup == 'linear' and self.iters < self.warmup_iters:
|
||||
rate = self.iters / self.warmup_iters
|
||||
for group, lr in zip(self.optimizer.param_groups, self.base_lr):
|
||||
group['lr'] = lr * rate
|
||||
return
|
||||
|
||||
# cos policy
|
||||
|
||||
for group, lr in zip(self.optimizer.param_groups, self.base_lr):
|
||||
group['lr'] = self.eta_min + (lr - self.eta_min) * (1 + math.cos(math.pi * self.iters / self.T_max)) / 2
|
||||
|
||||
|
||||
74
algorithms/lane_ufld/code/UFLD/utils/loss.py
Executable file
74
algorithms/lane_ufld/code/UFLD/utils/loss.py
Executable file
@@ -0,0 +1,74 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
import numpy as np
|
||||
|
||||
class OhemCELoss(nn.Module):
|
||||
def __init__(self, thresh, n_min, ignore_lb=255, *args, **kwargs):
|
||||
super(OhemCELoss, self).__init__()
|
||||
self.thresh = -torch.log(torch.tensor(thresh, dtype=torch.float)).cuda()
|
||||
self.n_min = n_min
|
||||
self.ignore_lb = ignore_lb
|
||||
self.criteria = nn.CrossEntropyLoss(ignore_index=ignore_lb, reduction='none')
|
||||
|
||||
def forward(self, logits, labels):
|
||||
N, C, H, W = logits.size()
|
||||
loss = self.criteria(logits, labels).view(-1)
|
||||
loss, _ = torch.sort(loss, descending=True)
|
||||
if loss[self.n_min] > self.thresh:
|
||||
loss = loss[loss>self.thresh]
|
||||
else:
|
||||
loss = loss[:self.n_min]
|
||||
return torch.mean(loss)
|
||||
|
||||
|
||||
class SoftmaxFocalLoss(nn.Module):
|
||||
def __init__(self, gamma, ignore_lb=255, *args, **kwargs):
|
||||
super(SoftmaxFocalLoss, self).__init__()
|
||||
self.gamma = gamma
|
||||
self.nll = nn.NLLLoss(ignore_index=ignore_lb)
|
||||
|
||||
def forward(self, logits, labels):
|
||||
scores = F.softmax(logits, dim=1)
|
||||
factor = torch.pow(1.-scores, self.gamma)
|
||||
log_score = F.log_softmax(logits, dim=1)
|
||||
log_score = factor * log_score
|
||||
loss = self.nll(log_score, labels)
|
||||
return loss
|
||||
|
||||
class ParsingRelationLoss(nn.Module):
|
||||
def __init__(self):
|
||||
super(ParsingRelationLoss, self).__init__()
|
||||
def forward(self,logits):
|
||||
n,c,h,w = logits.shape
|
||||
loss_all = []
|
||||
for i in range(0,h-1):
|
||||
loss_all.append(logits[:,:,i,:] - logits[:,:,i+1,:])
|
||||
#loss0 : n,c,w
|
||||
loss = torch.cat(loss_all)
|
||||
return torch.nn.functional.smooth_l1_loss(loss,torch.zeros_like(loss))
|
||||
|
||||
|
||||
|
||||
class ParsingRelationDis(nn.Module):
|
||||
def __init__(self):
|
||||
super(ParsingRelationDis, self).__init__()
|
||||
self.l1 = torch.nn.L1Loss()
|
||||
# self.l1 = torch.nn.MSELoss()
|
||||
def forward(self, x):
|
||||
n,dim,num_rows,num_cols = x.shape
|
||||
x = torch.nn.functional.softmax(x[:,:dim-1,:,:],dim=1)
|
||||
embedding = torch.Tensor(np.arange(dim-1)).float().to(x.device).view(1,-1,1,1)
|
||||
pos = torch.sum(x*embedding,dim = 1)
|
||||
|
||||
diff_list1 = []
|
||||
for i in range(0,num_rows // 2):
|
||||
diff_list1.append(pos[:,i,:] - pos[:,i+1,:])
|
||||
|
||||
loss = 0
|
||||
for i in range(len(diff_list1)-1):
|
||||
loss += self.l1(diff_list1[i],diff_list1[i+1])
|
||||
loss /= len(diff_list1) - 1
|
||||
return loss
|
||||
|
||||
103
algorithms/lane_ufld/code/UFLD/utils/metrics.py
Executable file
103
algorithms/lane_ufld/code/UFLD/utils/metrics.py
Executable file
@@ -0,0 +1,103 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
import time,pdb
|
||||
|
||||
def converter(data):
|
||||
if isinstance(data,torch.Tensor):
|
||||
data = data.cpu().data.numpy().flatten()
|
||||
return data.flatten()
|
||||
def fast_hist(label_pred, label_true,num_classes):
|
||||
#pdb.set_trace()
|
||||
hist = np.bincount(num_classes * label_true.astype(int) + label_pred, minlength=num_classes ** 2)
|
||||
hist = hist.reshape(num_classes, num_classes)
|
||||
return hist
|
||||
|
||||
class Metric_mIoU():
|
||||
def __init__(self,class_num):
|
||||
self.class_num = class_num
|
||||
self.hist = np.zeros((self.class_num,self.class_num))
|
||||
def update(self,predict,target):
|
||||
predict,target = converter(predict),converter(target)
|
||||
|
||||
self.hist += fast_hist(predict,target,self.class_num)
|
||||
|
||||
def reset(self):
|
||||
self.hist = np.zeros((self.class_num,self.class_num))
|
||||
def get_miou(self):
|
||||
miou = np.diag(self.hist) / (
|
||||
np.sum(self.hist, axis=1) + np.sum(self.hist, axis=0) -
|
||||
np.diag(self.hist))
|
||||
miou = np.nanmean(miou)
|
||||
return miou
|
||||
|
||||
def get_acc(self):
|
||||
acc = np.diag(self.hist) / self.hist.sum(axis=1)
|
||||
acc = np.nanmean(acc)
|
||||
return acc
|
||||
def get(self):
|
||||
return self.get_miou()
|
||||
class MultiLabelAcc():
|
||||
def __init__(self):
|
||||
self.cnt = 0
|
||||
self.correct = 0
|
||||
def reset(self):
|
||||
self.cnt = 0
|
||||
self.correct = 0
|
||||
def update(self,predict,target):
|
||||
predict,target = converter(predict),converter(target)
|
||||
self.cnt += len(predict)
|
||||
self.correct += np.sum(predict==target)
|
||||
def get_acc(self):
|
||||
return self.correct * 1.0 / self.cnt
|
||||
def get(self):
|
||||
return self.get_acc()
|
||||
class AccTopk():
|
||||
def __init__(self,background_classes,k):
|
||||
self.background_classes = background_classes
|
||||
self.k = k
|
||||
self.cnt = 0
|
||||
self.top5_correct = 0
|
||||
def reset(self):
|
||||
self.cnt = 0
|
||||
self.top5_correct = 0
|
||||
def update(self,predict,target):
|
||||
predict,target = converter(predict),converter(target)
|
||||
self.cnt += len(predict)
|
||||
background_idx = (predict == self.background_classes) + (target == self.background_classes)
|
||||
self.top5_correct += np.sum(predict[background_idx] == target[background_idx])
|
||||
not_background_idx = np.logical_not(background_idx)
|
||||
self.top5_correct += np.sum(np.absolute(predict[not_background_idx]-target[not_background_idx])<self.k)
|
||||
def get(self):
|
||||
return self.top5_correct * 1.0 / self.cnt
|
||||
|
||||
|
||||
|
||||
def update_metrics(metric_dict, pair_data):
|
||||
for i in range(len(metric_dict['name'])):
|
||||
metric_op = metric_dict['op'][i]
|
||||
data_src = metric_dict['data_src'][i]
|
||||
metric_op.update(pair_data[data_src[0]], pair_data[data_src[1]])
|
||||
|
||||
|
||||
def reset_metrics(metric_dict):
|
||||
for op in metric_dict['op']:
|
||||
op.reset()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# p = np.random.randint(5, size=(800, 800))
|
||||
# t = np.zeros((800, 800))
|
||||
# me = Metric_mIoU(5)
|
||||
# me.update(p,p)
|
||||
# me.update(p,t)
|
||||
# me.update(p,p)
|
||||
# me.update(p,t)
|
||||
# print(me.get_miou())
|
||||
# print(me.get_acc())
|
||||
|
||||
a = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0])
|
||||
b = np.array([1, 1, 2, 2, 2, 3, 3, 4, 4, 0])
|
||||
me = AccTopk(0,5)
|
||||
me.update(b,a)
|
||||
print(me.get())
|
||||
Reference in New Issue
Block a user