diff --git a/scripts/eval_full/gen_results.py b/scripts/eval_full/gen_results.py index efdc378..71ea656 100755 --- a/scripts/eval_full/gen_results.py +++ b/scripts/eval_full/gen_results.py @@ -5,63 +5,78 @@ import sys import json -datasets = ['Sequence', 'MH_01_easy', 'MH_02_easy', 'MH_03_medium', 'MH_04_difficult', +datasets = ['Seq.', 'MH_01_easy', 'MH_02_easy', 'MH_03_medium', 'MH_04_difficult', 'MH_05_difficult', 'V1_01_easy', 'V1_02_medium', 'V1_03_difficult', 'V2_01_easy', 'V2_02_medium'] # Other results. -results_vio = ['VIO RMS ATE [m]'] -time_vio = ['VIO Time [s]'] -num_frames_vio = ['VIO Num. Frames'] -results_mapping = ['MAP RMS ATE [m]'] -time_mapping = ['MAP Time [s]'] -num_frames_mapping = ['MAP Num. KFs'] + +vio = { +'ate' : ['VIO RMS ATE [m]'], +'time' : ['VIO Time [s]'], +'num_frames' : ['VIO Num. Frames'] +} + +mapping = { +'ate' : ['MAP RMS ATE [m]'], +'time' : ['MAP Time [s]'], +'num_frames' : ['MAP Num. KFs'] +} + +pose_graph = { +'ate' : ['PG RMS ATE [m]'], +'time' : ['PG Time [s]'], +'num_frames' : ['PG Num. KFs'] +} out_dir = sys.argv[1] +def load_data(x, prefix, key): + fname = out_dir + '/' + prefix + '_' + key + if os.path.isfile(fname): + with open(fname, 'r') as f: + j = json.load(f) + res = round(j['rms_ate'], 3) + x['ate'].append(float(res)) + x['time'].append(round(j['exec_time_ns']*1e-9, 3)) + x['num_frames'].append(j['num_frames']) + else: + x['ate'].append(float('Inf')) + x['time'].append(float('Inf')) + x['num_frames'].append(float('Inf')) + for key in datasets[1:]: - fname = out_dir + '/vio_' + key - if os.path.isfile(fname): - with open(fname, 'r') as f: - j = json.load(f) - res = round(j['rms_ate'], 3) - results_vio.append(float(res)) - time_vio.append(round(j['exec_time_ns']*1e-9, 3)) - num_frames_vio.append(j['num_frames']) - else: - results_vio.append(float('Inf')) - time_vio.append(float('Inf')) - num_frames_vio.append(float('Inf')) - - fname = out_dir + '/mapper_' + key - if os.path.isfile(fname): - with open(fname, 'r') as f: - j = json.load(f) - res = round(j['rms_ate'], 3) - results_mapping.append(float(res)) - time_mapping.append(round(j['exec_time_ns']*1e-9, 3)) - num_frames_mapping.append(j['num_frames']) - else: - results_mapping.append(float('Inf')) - time_mapping.append(float('Inf')) - num_frames_mapping.append(float('Inf')) + load_data(vio, 'vio', key) + load_data(mapping, 'mapper', key) + load_data(pose_graph, 'mapper_no_weights', key) -row_format ="{:>17}" * (len(datasets)) -print 'Visual-Inertial Odometry' -print row_format.format(*datasets) +row_format ="{:>17}" + "{:>13}" * (len(datasets)-1) -print row_format.format(*results_vio) -print row_format.format(*time_vio) -print row_format.format(*num_frames_vio) +datasets_short = [x[:5] for x in datasets] -print 'Visual-Inertial Mapping' -print row_format.format(*datasets) +print '\nVisual-Inertial Odometry' +print row_format.format(*datasets_short) -print row_format.format(*results_mapping) -print row_format.format(*time_mapping) -print row_format.format(*num_frames_mapping) +print row_format.format(*vio['ate']) +print row_format.format(*vio['time']) +print row_format.format(*vio['num_frames']) + +print '\nVisual-Inertial Mapping' +print row_format.format(*datasets_short) + +print row_format.format(*mapping['ate']) +print row_format.format(*mapping['time']) +print row_format.format(*mapping['num_frames']) + + +print '\nPose-Graph optimization (Identity weights for all factors)' +print row_format.format(*datasets_short) + +print row_format.format(*pose_graph['ate']) +print row_format.format(*pose_graph['time']) +print row_format.format(*pose_graph['num_frames']) diff --git a/scripts/eval_full/run_evaluations.sh b/scripts/eval_full/run_evaluations.sh index f78e430..833f849 100755 --- a/scripts/eval_full/run_evaluations.sh +++ b/scripts/eval_full/run_evaluations.sh @@ -20,6 +20,10 @@ for d in ${DATASETS[$CI_NODE_INDEX-1]}; do basalt_mapper --show-gui 0 --cam-calib /usr/etc/basalt/euroc_eucm_calib.json --config-path /usr/etc/basalt/euroc_config.json --marg-data eval_tmp_marg_data \ --result-path $folder_name/mapper_$d + + basalt_mapper --show-gui 0 --cam-calib /usr/etc/basalt/euroc_eucm_calib.json --config-path /usr/etc/basalt/euroc_config_no_weights.json --marg-data eval_tmp_marg_data \ + --result-path $folder_name/mapper_no_weights_$d + rm -rf eval_tmp_marg_data done