Added evaluation for pose-graph optimization

scripts/eval_full/gen_results.py

@@ -5,63 +5,78 @@ import sys
 import json
 
 
-datasets = ['Sequence', 'MH_01_easy', 'MH_02_easy', 'MH_03_medium', 'MH_04_difficult',
+datasets = ['Seq.', 'MH_01_easy', 'MH_02_easy', 'MH_03_medium', 'MH_04_difficult',
 'MH_05_difficult', 'V1_01_easy', 'V1_02_medium',
 'V1_03_difficult', 'V2_01_easy', 'V2_02_medium']
 
 
 # Other results.
-results_vio = ['VIO RMS ATE [m]']
-time_vio = ['VIO Time [s]']
-num_frames_vio = ['VIO Num. Frames'] 
 
-results_mapping = ['MAP RMS ATE [m]']
-time_mapping = ['MAP Time [s]']
-num_frames_mapping = ['MAP Num. KFs'] 
+
+vio = {
+'ate' : ['VIO RMS ATE [m]'],
+'time' : ['VIO Time [s]'],
+'num_frames' : ['VIO Num. Frames']
+}
+
+mapping = {
+'ate' : ['MAP RMS ATE [m]'],
+'time' : ['MAP Time [s]'],
+'num_frames' : ['MAP Num. KFs']
+}
+
+pose_graph = {
+'ate' : ['PG RMS ATE [m]'],
+'time' : ['PG Time [s]'],
+'num_frames' : ['PG Num. KFs']
+}
 
 out_dir = sys.argv[1]
 
+def load_data(x, prefix, key):
+    fname = out_dir + '/' + prefix + '_' + key
+    if os.path.isfile(fname): 
+        with open(fname, 'r') as f:
+            j = json.load(f)
+            res = round(j['rms_ate'], 3)
+            x['ate'].append(float(res))
+            x['time'].append(round(j['exec_time_ns']*1e-9, 3))
+            x['num_frames'].append(j['num_frames'])
+    else:
+        x['ate'].append(float('Inf'))
+        x['time'].append(float('Inf'))
+        x['num_frames'].append(float('Inf'))
+
 
 for key in datasets[1:]:
-    fname = out_dir + '/vio_' + key
-    if os.path.isfile(fname): 
-        with open(fname, 'r') as f:
-            j = json.load(f)
-            res = round(j['rms_ate'], 3)
-            results_vio.append(float(res))
-            time_vio.append(round(j['exec_time_ns']*1e-9, 3))
-            num_frames_vio.append(j['num_frames'])
-    else:
-        results_vio.append(float('Inf'))
-        time_vio.append(float('Inf'))
-        num_frames_vio.append(float('Inf'))
-    
-    fname = out_dir + '/mapper_' + key
-    if os.path.isfile(fname): 
-        with open(fname, 'r') as f:
-            j = json.load(f)
-            res = round(j['rms_ate'], 3)
-            results_mapping.append(float(res))
-            time_mapping.append(round(j['exec_time_ns']*1e-9, 3))
-            num_frames_mapping.append(j['num_frames'])
-    else:
-        results_mapping.append(float('Inf'))
-        time_mapping.append(float('Inf'))
-        num_frames_mapping.append(float('Inf'))
+    load_data(vio, 'vio', key)
+    load_data(mapping, 'mapper', key)
+    load_data(pose_graph, 'mapper_no_weights', key)
 
-row_format ="{:>17}" * (len(datasets))
 
-print 'Visual-Inertial Odometry'
-print row_format.format(*datasets)
+row_format ="{:>17}" + "{:>13}" * (len(datasets)-1)
 
-print row_format.format(*results_vio)
-print row_format.format(*time_vio)
-print row_format.format(*num_frames_vio)
+datasets_short = [x[:5] for x in datasets]
 
-print 'Visual-Inertial Mapping'
-print row_format.format(*datasets)
+print '\nVisual-Inertial Odometry'
+print row_format.format(*datasets_short)
 
-print row_format.format(*results_mapping)
-print row_format.format(*time_mapping)
-print row_format.format(*num_frames_mapping)
+print row_format.format(*vio['ate'])
+print row_format.format(*vio['time'])
+print row_format.format(*vio['num_frames'])
+
+print '\nVisual-Inertial Mapping'
+print row_format.format(*datasets_short)
+
+print row_format.format(*mapping['ate'])
+print row_format.format(*mapping['time'])
+print row_format.format(*mapping['num_frames'])
+
+
+print '\nPose-Graph optimization (Identity weights for all factors)'
+print row_format.format(*datasets_short)
+
+print row_format.format(*pose_graph['ate'])
+print row_format.format(*pose_graph['time'])
+print row_format.format(*pose_graph['num_frames'])
 

scripts/eval_full/run_evaluations.sh

@@ -20,6 +20,10 @@ for d in ${DATASETS[$CI_NODE_INDEX-1]}; do
 
     basalt_mapper --show-gui 0 --cam-calib /usr/etc/basalt/euroc_eucm_calib.json --config-path /usr/etc/basalt/euroc_config.json --marg-data eval_tmp_marg_data \
         --result-path $folder_name/mapper_$d
+
+    basalt_mapper --show-gui 0 --cam-calib /usr/etc/basalt/euroc_eucm_calib.json --config-path /usr/etc/basalt/euroc_config_no_weights.json --marg-data eval_tmp_marg_data \
+        --result-path $folder_name/mapper_no_weights_$d
+    
     rm -rf eval_tmp_marg_data
 done