54 lines
1.5 KiB
Bash
Executable File
54 lines
1.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
|
|
PYTHON_BIN="${PYTHON_BIN:-/deeplearning_team/ydong/dongying/miniconda/envs/dev/bin/python}"
|
|
|
|
detect_cpu_count() {
|
|
local cpu_count
|
|
cpu_count="$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || echo 1)"
|
|
if [[ ! "${cpu_count}" =~ ^[0-9]+$ ]] || (( cpu_count <= 0 )); then
|
|
cpu_count=1
|
|
fi
|
|
echo "${cpu_count}"
|
|
}
|
|
|
|
calc_scan_workers() {
|
|
local cpu_count="$1"
|
|
if (( cpu_count >= 8 )); then
|
|
echo 8
|
|
else
|
|
echo "${cpu_count}"
|
|
fi
|
|
}
|
|
|
|
calc_part_workers() {
|
|
local cpu_count="$1"
|
|
if (( cpu_count >= 32 )); then
|
|
echo 4
|
|
elif (( cpu_count >= 16 )); then
|
|
echo 3
|
|
elif (( cpu_count >= 8 )); then
|
|
echo 2
|
|
else
|
|
echo 1
|
|
fi
|
|
}
|
|
|
|
CPU_COUNT="$(detect_cpu_count)"
|
|
|
|
# 打包是 I/O + gzip 压缩混合负载，这里默认给一套偏稳妥的并发配置；
|
|
# 如需手动调节，可在命令前覆盖这些环境变量。
|
|
VIS_PACK_SCAN_WORKERS="${VIS_PACK_SCAN_WORKERS:-$(calc_scan_workers "${CPU_COUNT}")}"
|
|
VIS_PACK_ARCHIVE_WORKERS="${VIS_PACK_ARCHIVE_WORKERS:-2}"
|
|
VIS_PACK_PART_WORKERS="${VIS_PACK_PART_WORKERS:-$(calc_part_workers "${CPU_COUNT}")}"
|
|
|
|
exec "${PYTHON_BIN}" \
|
|
"${PROJECT_ROOT}/tools/convert_gt_to_label/package_visualization_batches.py" \
|
|
--scan-workers "${VIS_PACK_SCAN_WORKERS}" \
|
|
--archive-workers "${VIS_PACK_ARCHIVE_WORKERS}" \
|
|
--part-workers "${VIS_PACK_PART_WORKERS}" \
|
|
"$@"
|