From e15bb2241005424ea6dfe10f571dac7744a153c2 Mon Sep 17 00:00:00 2001
From: Vladyslav Usenko <usenko.vladyslav@gmail.com>
Date: Mon, 15 Apr 2019 15:38:58 +0000
Subject: [PATCH] Added euroc evaluation to CI

---
 .gitlab-ci.yml                       | 45 ++++++++++++++++++++++++++++
 scripts/eval_full/gen_results.py     |  7 +++--
 scripts/eval_full/run_evaluations.sh | 20 ++++++-------
 3 files changed, 59 insertions(+), 13 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d38d87c..7b0a7f0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,11 @@
 image: vladyslavusenko/b_image:latest
 
+stages:
+  - build
+  - test
+  - test_results
+  - deploy
+
 compile:
   stage: build
   variables:
@@ -26,10 +32,13 @@ compile:
   artifacts:
     paths:
     - deb_bionic/*.deb
+    - scripts/eval_full/*
     expire_in: 1 week
 
 compile_16_04:
   stage: build
+  variables:
+    CXX_MARCH: 'corei7-avx'
   before_script:
     - mkdir -p ccache
     - export CCACHE_BASEDIR=${PWD}
@@ -57,6 +66,42 @@ compile_16_04:
     - master
   image: vladyslavusenko/b_image_xenial:latest
 
+
+eval_euroc:
+  stage: test
+  parallel: 10
+  tags: 
+    - euroc-eval
+  variables:
+    GIT_STRATEGY: none
+  only:
+    - master
+  artifacts:
+    paths:
+    - scripts/eval_full/eval_results/*
+    expire_in: 1 week
+  script:
+    - dpkg -i deb_bionic/*.deb
+    - cd scripts/eval_full
+    - ./run_evaluations.sh
+
+gen_results_euroc:
+  stage: test_results
+  variables:
+    GIT_STRATEGY: none
+  when: on_success
+  only:
+    - master
+  artifacts:
+    paths:
+    - euroc_results.txt
+  script:
+    - cd scripts/eval_full
+    - ./gen_results.py eval_results > euroc_results.txt
+    - cat euroc_results.txt
+    - mv euroc_results.txt ../../
+
+
 deploy:
   stage: deploy
   before_script:
diff --git a/scripts/eval_full/gen_results.py b/scripts/eval_full/gen_results.py
index a1a551a..c62ffa2 100755
--- a/scripts/eval_full/gen_results.py
+++ b/scripts/eval_full/gen_results.py
@@ -2,7 +2,6 @@
 
 import os
 import sys
-import numpy as np
 
 
 datasets = ['MH_01_easy', 'MH_02_easy', 'MH_03_medium', 'MH_04_difficult',
@@ -20,14 +19,16 @@ out_dir = sys.argv[1]
 for key in datasets:
     fname = out_dir + '/vio_' + key
     if os.path.isfile(fname): 
-        res = round(float(np.loadtxt(fname)), 3)
+        with open(fname, 'r') as f:
+            res = round(float(f.read()), 3)
         results_vio.append(float(res))
     else:
         results_vio.append(float('Inf'))
     
     fname = out_dir + '/mapper_' + key
     if os.path.isfile(fname): 
-        res = round(float(np.loadtxt(fname)), 3)
+        with open(fname, 'r') as f:
+            res = round(float(f.read()), 3)
         results_mapping.append(float(res))
     else:
         results_mapping.append(float('Inf'))
diff --git a/scripts/eval_full/run_evaluations.sh b/scripts/eval_full/run_evaluations.sh
index 8b4e5e5..61b5435 100755
--- a/scripts/eval_full/run_evaluations.sh
+++ b/scripts/eval_full/run_evaluations.sh
@@ -5,22 +5,22 @@ set -x
 
 DATASET_PATH=/data/euroc
 
-DATASETS=(MH_01_easy MH_02_easy MH_03_medium MH_04_difficult MH_05_difficult V1_01_easy V1_02_medium V1_03_difficult V2_01_easy V2_02_medium V2_03_difficult)
+DATASETS=(MH_01_easy MH_02_easy MH_03_medium MH_04_difficult MH_05_difficult V1_01_easy V1_02_medium V1_03_difficult V2_01_easy V2_02_medium)
 
 
-folder_name=eval_$(date +%Y%m%d_%H%M%S)
+folder_name=eval_results
 mkdir $folder_name
 
-cp ../../data/euroc_config.json $folder_name/config.json
 
 
-for d in ${DATASETS[@]}; do
-    ../../build/basalt_vio --dataset-path  $DATASET_PATH/$d --cam-calib ../../data/euroc_ds_calib.json \
-        --dataset-type euroc --show-gui 0 --config-path $folder_name/config.json \
-        --result-path $folder_name/vio_$d --marg-data $folder_name/marg_$d/
+for d in ${DATASETS[$CI_NODE_INDEX-1]}; do
+   basalt_vio --dataset-path  $DATASET_PATH/$d --cam-calib /usr/etc/basalt/euroc_ds_calib.json \
+        --dataset-type euroc --show-gui 0 --config-path /usr/etc/basalt/euroc_config.json \
+        --result-path $folder_name/vio_$d --marg-data eval_tmp_marg_data
 
-    ../../build/basalt_mapper --show-gui 0 --cam-calib ../../data/euroc_ds_calib.json --marg-data $folder_name/marg_$d/ \
-        --vocabulary ../../thirdparty/DBoW3/vocab/orbvoc.dbow3 --result-path $folder_name/mapper_$d
+    basalt_mapper --show-gui 0 --cam-calib /usr/etc/basalt/euroc_ds_calib.json --marg-data eval_tmp_marg_data \
+        --vocabulary /usr/etc/basalt/orbvoc.dbow3 --result-path $folder_name/mapper_$d
+    rm -rf eval_tmp_marg_data
 done
 
-./gen_results.py $folder_name
+#./gen_results.py $folder_name > euroc_results.txt