From 2926020df7feae6423656ffb5c065772a64dc48c Mon Sep 17 00:00:00 2001
From: Vladyslav Usenko <vlad.usenko@tum.de>
Date: Fri, 6 Mar 2020 12:51:28 +0100
Subject: [PATCH] added tumvi eval

---
 .gitlab-ci.yml                             | 35 ++++++++++++++
 scripts/eval_full/gen_results_tumvi.py     | 53 ++++++++++++++++++++++
 scripts/eval_full/run_evaluations_tumvi.sh | 30 ++++++++++++
 3 files changed, 118 insertions(+)
 create mode 100755 scripts/eval_full/gen_results_tumvi.py
 create mode 100755 scripts/eval_full/run_evaluations_tumvi.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 48d6272..2f2359a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -106,6 +106,23 @@ stages:
     - cd scripts/eval_full
     - ./run_evaluations.sh
 
+# template to evaluate on TUM-VI sequences
+.eval_tumvi_template: &eval_tumvi_definition
+  stage: eval
+  parallel: 4
+  tags: [docker, dataset-eval]
+  variables:
+    GIT_STRATEGY: none
+  artifacts:
+    paths:
+    - scripts/eval_full/eval_results_tumvi/*
+    expire_in: 1 week
+  script:
+    - dpkg -i deb_bionic/*.deb
+    - cd scripts/eval_full
+    - ./run_evaluations_tumvi.sh
+
+
 .eval_euroc_template: &eval_kitti_definition
   stage: eval
   parallel: 10
@@ -222,6 +239,20 @@ eval_euroc:
     - master
   allow_failure: false
 
+# evaluate on TUM-VI sequences
+eval_tumvi_master:
+  <<: *eval_tumvi_definition
+  only:
+    - master
+
+# evaluate on TUM-VI sequences
+eval_tumvi:
+  <<: *eval_tumvi_definition
+  when: manual
+  except:
+    - master
+  allow_failure: false
+
 # evaluate on KITTI sequences
 eval_kitti_master:
   <<: *eval_kitti_definition
@@ -248,6 +279,7 @@ gen_results:
     paths:
     - euroc_results.txt
     - kitti_results.txt
+    - tumvi_results.txt
     - scripts/eval_full/eval_results/*
   script:
     - cd scripts/eval_full
@@ -255,8 +287,11 @@ gen_results:
     - cat euroc_results.txt
     - ./gen_results_kitti.py eval_results_kitti > kitti_results.txt
     - cat kitti_results.txt
+    - ./gen_results_tumvi.py eval_results_tumvi > tumvi_results.txt
+    - cat kitti_results.txt
     - mv euroc_results.txt ../../
     - mv kitti_results.txt ../../
+    - mv tumvi_results.txt ../../
 
 # deploy deb packages
 deploy:
diff --git a/scripts/eval_full/gen_results_tumvi.py b/scripts/eval_full/gen_results_tumvi.py
new file mode 100755
index 0000000..7f4c25e
--- /dev/null
+++ b/scripts/eval_full/gen_results_tumvi.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import json
+
+
+datasets = ['Seq.', 'dataset-corridor1_512_16', 'dataset-magistrale1_512_16', 'dataset-room1_512_16', 'dataset-slides1_512_16']
+
+# Other results.
+
+
+vio = {
+'ate' : ['VIO RMS ATE [m]'],
+'time' : ['VIO Time [s]'],
+'num_frames' : ['VIO Num. Frames']
+}
+
+out_dir = sys.argv[1]
+
+def load_data(x, prefix, key):
+    fname = out_dir + '/' + prefix + '_' + key
+    if os.path.isfile(fname): 
+        with open(fname, 'r') as f:
+            j = json.load(f)
+            res = round(j['rms_ate'], 3)
+            x['ate'].append(float(res))
+            x['time'].append(round(j['exec_time_ns']*1e-9, 3))
+            x['num_frames'].append(j['num_frames'])
+    else:
+        x['ate'].append(float('Inf'))
+        x['time'].append(float('Inf'))
+        x['num_frames'].append(float('Inf'))
+
+
+for key in datasets[1:]:
+    load_data(vio, 'vio', key)
+
+
+row_format ="{:>17}" + "{:>13}" * (len(datasets)-1)
+
+datasets_short = [x[:5] for x in datasets]
+
+print '\nVisual-Inertial Odometry'
+print row_format.format(*datasets_short)
+
+print row_format.format(*vio['ate'])
+#print row_format.format(*vio['time'])
+print row_format.format(*vio['num_frames'])
+
+
+
+
diff --git a/scripts/eval_full/run_evaluations_tumvi.sh b/scripts/eval_full/run_evaluations_tumvi.sh
new file mode 100755
index 0000000..321cc2c
--- /dev/null
+++ b/scripts/eval_full/run_evaluations_tumvi.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+set -x
+
+DATASET_PATH=/data/tumvi/512_16/
+
+DATASETS=(
+dataset-corridor1_512_16
+dataset-magistrale1_512_16
+dataset-room1_512_16
+dataset-slides1_512_16
+)
+
+
+folder_name=eval_results_tumvi
+mkdir $folder_name
+
+
+
+for d in ${DATASETS[$CI_NODE_INDEX-1]}; do
+   ../../build/basalt_vio --dataset-path  $DATASET_PATH/$d --cam-calib /usr/etc/basalt/tumvi_512_eucm_calib.json \
+        --dataset-type euroc --show-gui 1 --config-path /usr/etc/basalt/tumvi_512_config.json \
+        --result-path $folder_name/vio_$d --save-trajectory tum
+
+   mv trajectory.txt $folder_name/${d}_basalt_poses.txt
+
+done
+
+#./gen_results_tumvi.py $folder_name > euroc_tumvi.txt
\ No newline at end of file