59 lines
1.4 KiB
Python
59 lines
1.4 KiB
Python
|
# Copyright (c) 2022 the ObjectPerceptionTool authors. All right reserved.
|
||
|
|
#
|
||
|
|
# This is a script to convert *.index.json and its data into JsonLines.
|
||
|
|
#
|
||
|
|
# This requires protobuf and yj_pyproto_versions. See:
|
||
|
|
#
|
||
|
|
# https://git.minieye.tech/pcview/yj_pyproto_versions
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
#
|
||
|
|
# make_jl.py xx.data.json -d xx.bin -o xx.jl
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import json
|
||
|
|
import math
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
from google.protobuf import json_format
|
||
|
|
|
||
|
|
import proto_reader.reader as pb_reader
|
||
|
|
|
||
|
|
sys.path.append(os.path.join(sys.path[0], "pyproto"))
|
||
|
|
import object_pb2_new as object_pb2
|
||
|
|
|
||
|
|
|
||
|
|
def make_jsonl(data_json_path, writer):
|
||
|
|
reader = pb_reader.Reader(data_json_path)
|
||
|
|
|
||
|
|
i = 0
|
||
|
|
for chunk in reader.each():
|
||
|
|
# objs = json.loads(chunk)
|
||
|
|
objs = object_pb2.ObjectList.FromString(chunk)
|
||
|
|
objs = json_format.MessageToDict(objs, including_default_value_fields=True)
|
||
|
|
|
||
|
|
writer.write(json.dumps(objs, indent=2) + "\n")
|
||
|
|
print("Convert message", i, end="\r", file=sys.stderr)
|
||
|
|
i += 1
|
||
|
|
|
||
|
|
print("\ndone", file=sys.stderr)
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
ap = argparse.ArgumentParser()
|
||
|
|
ap.add_argument("data_json")
|
||
|
|
ap.add_argument("-o", "--output")
|
||
|
|
opt = ap.parse_args()
|
||
|
|
|
||
|
|
try:
|
||
|
|
if opt.output is None:
|
||
|
|
make_jsonl(opt.data_json, sys.stdout)
|
||
|
|
else:
|
||
|
|
with open(opt.output, "w") as writer:
|
||
|
|
make_jsonl(opt.data_json, writer)
|
||
|
|
except KeyboardInterrupt:
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
main()
|