Better evaluation output

scripts/eval_full/gen_results.py

@@ -2,39 +2,65 @@
 
 import os
 import sys
+import json
 
 
-datasets = ['MH_01_easy', 'MH_02_easy', 'MH_03_medium', 'MH_04_difficult',
+datasets = ['Sequence', 'MH_01_easy', 'MH_02_easy', 'MH_03_medium', 'MH_04_difficult',
 'MH_05_difficult', 'V1_01_easy', 'V1_02_medium',
 'V1_03_difficult', 'V2_01_easy', 'V2_02_medium']
 
 
 # Other results.
-results_vio = []
+results_vio = ['VIO RMS ATE [m]']
-results_mapping = []
+time_vio = ['VIO Time [s]']
+num_frames_vio = ['VIO Num. Frames'] 
+
+results_mapping = ['MAP RMS ATE [m]']
+time_mapping = ['MAP Time [s]']
+num_frames_mapping = ['MAP Num. KFs'] 
 
 out_dir = sys.argv[1]
 
 
-for key in datasets:
+for key in datasets[1:]:
     fname = out_dir + '/vio_' + key
     if os.path.isfile(fname): 
         with open(fname, 'r') as f:
-            res = round(float(f.read()), 3)
+            j = json.load(f)
-        results_vio.append(float(res))
+            res = round(j['rms_ate'], 3)
+            results_vio.append(float(res))
+            time_vio.append(round(j['exec_time_ns']*1e-9, 3))
+            num_frames_vio.append(j['num_frames'])
     else:
         results_vio.append(float('Inf'))
+        time_vio.append(float('Inf'))
+        num_frames_vio.append(float('Inf'))
     
     fname = out_dir + '/mapper_' + key
     if os.path.isfile(fname): 
         with open(fname, 'r') as f:
-            res = round(float(f.read()), 3)
+            j = json.load(f)
-        results_mapping.append(float(res))
+            res = round(j['rms_ate'], 3)
+            results_mapping.append(float(res))
+            time_mapping.append(round(j['exec_time_ns']*1e-9, 3))
+            num_frames_mapping.append(j['num_frames'])
     else:
         results_mapping.append(float('Inf'))
+        time_mapping.append(float('Inf'))
+        num_frames_mapping.append(float('Inf'))
 
 row_format ="{:>17}" * (len(datasets))
 print row_format.format(*datasets)
-print row_format.format(*results_vio)
+
-print row_format.format(*results_mapping)
+print row_format.format(*results_vio)
+print row_format.format(*time_vio)
+print row_format.format(*num_frames_vio)
+
+print '\n'
+
+print row_format.format(*datasets)
+
+print row_format.format(*results_mapping)
+print row_format.format(*time_mapping)
+print row_format.format(*num_frames_mapping)
 

src/mapper.cpp

@@ -325,6 +325,7 @@ int main(int argc, char** argv) {
       std::this_thread::sleep_for(std::chrono::milliseconds(50));
     }
   } else {
+    auto time_start = std::chrono::high_resolution_clock::now();
     // optimize();
     detect();
     match();
@@ -333,11 +334,21 @@ int main(int argc, char** argv) {
     filter();
     optimize();
 
+    auto time_end = std::chrono::high_resolution_clock::now();
+
     if (!result_path.empty()) {
       double error = alignButton();
 
+      auto exec_time_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
+          time_end - time_start);
+
       std::ofstream os(result_path);
-      os << error << std::endl;
+      {
+        cereal::JSONOutputArchive ar(os);
+        ar(cereal::make_nvp("rms_ate", error));
+        ar(cereal::make_nvp("num_frames", nrf_mapper->getFramePoses().size()));
+        ar(cereal::make_nvp("exec_time_ns", exec_time_ns.count()));
+      }
       os.close();
     }
   }

src/vio.cpp

@@ -346,6 +346,8 @@ int main(int argc, char** argv) {
     }));
   }
 
+  auto time_start = std::chrono::high_resolution_clock::now();
+
   if (show_gui) {
     pangolin::CreateWindowAndBind("Main", 1800, 1000);
 
@@ -469,11 +471,22 @@ int main(int argc, char** argv) {
   if (t3.get()) t3->join();
   t4.join();
 
+  auto time_end = std::chrono::high_resolution_clock::now();
+
   if (!result_path.empty()) {
     double error = basalt::alignSVD(vio_t_ns, vio_t_w_i, gt_t_ns, gt_t_w_i);
 
+    auto exec_time_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
+        time_end - time_start);
+
     std::ofstream os(result_path);
-    os << error << std::endl;
+    {
+      cereal::JSONOutputArchive ar(os);
+      ar(cereal::make_nvp("rms_ate", error));
+      ar(cereal::make_nvp("num_frames",
+                          vio_dataset->get_image_timestamps().size()));
+      ar(cereal::make_nvp("exec_time_ns", exec_time_ns.count()));
+    }
     os.close();
   }