From 2926020df7feae6423656ffb5c065772a64dc48c Mon Sep 17 00:00:00 2001 From: Vladyslav Usenko Date: Fri, 6 Mar 2020 12:51:28 +0100 Subject: [PATCH] added tumvi eval --- .gitlab-ci.yml | 35 ++++++++++++++ scripts/eval_full/gen_results_tumvi.py | 53 ++++++++++++++++++++++ scripts/eval_full/run_evaluations_tumvi.sh | 30 ++++++++++++ 3 files changed, 118 insertions(+) create mode 100755 scripts/eval_full/gen_results_tumvi.py create mode 100755 scripts/eval_full/run_evaluations_tumvi.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 48d6272..2f2359a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,6 +106,23 @@ stages: - cd scripts/eval_full - ./run_evaluations.sh +# template to evaluate on TUM-VI sequences +.eval_tumvi_template: &eval_tumvi_definition + stage: eval + parallel: 4 + tags: [docker, dataset-eval] + variables: + GIT_STRATEGY: none + artifacts: + paths: + - scripts/eval_full/eval_results_tumvi/* + expire_in: 1 week + script: + - dpkg -i deb_bionic/*.deb + - cd scripts/eval_full + - ./run_evaluations_tumvi.sh + + .eval_euroc_template: &eval_kitti_definition stage: eval parallel: 10 @@ -222,6 +239,20 @@ eval_euroc: - master allow_failure: false +# evaluate on TUM-VI sequences +eval_tumvi_master: + <<: *eval_tumvi_definition + only: + - master + +# evaluate on TUM-VI sequences +eval_tumvi: + <<: *eval_tumvi_definition + when: manual + except: + - master + allow_failure: false + # evaluate on KITTI sequences eval_kitti_master: <<: *eval_kitti_definition @@ -248,6 +279,7 @@ gen_results: paths: - euroc_results.txt - kitti_results.txt + - tumvi_results.txt - scripts/eval_full/eval_results/* script: - cd scripts/eval_full @@ -255,8 +287,11 @@ gen_results: - cat euroc_results.txt - ./gen_results_kitti.py eval_results_kitti > kitti_results.txt - cat kitti_results.txt + - ./gen_results_tumvi.py eval_results_tumvi > tumvi_results.txt + - cat kitti_results.txt - mv euroc_results.txt ../../ - mv kitti_results.txt ../../ + - mv tumvi_results.txt ../../ # deploy deb packages deploy: diff --git a/scripts/eval_full/gen_results_tumvi.py b/scripts/eval_full/gen_results_tumvi.py new file mode 100755 index 0000000..7f4c25e --- /dev/null +++ b/scripts/eval_full/gen_results_tumvi.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import os +import sys +import json + + +datasets = ['Seq.', 'dataset-corridor1_512_16', 'dataset-magistrale1_512_16', 'dataset-room1_512_16', 'dataset-slides1_512_16'] + +# Other results. + + +vio = { +'ate' : ['VIO RMS ATE [m]'], +'time' : ['VIO Time [s]'], +'num_frames' : ['VIO Num. Frames'] +} + +out_dir = sys.argv[1] + +def load_data(x, prefix, key): + fname = out_dir + '/' + prefix + '_' + key + if os.path.isfile(fname): + with open(fname, 'r') as f: + j = json.load(f) + res = round(j['rms_ate'], 3) + x['ate'].append(float(res)) + x['time'].append(round(j['exec_time_ns']*1e-9, 3)) + x['num_frames'].append(j['num_frames']) + else: + x['ate'].append(float('Inf')) + x['time'].append(float('Inf')) + x['num_frames'].append(float('Inf')) + + +for key in datasets[1:]: + load_data(vio, 'vio', key) + + +row_format ="{:>17}" + "{:>13}" * (len(datasets)-1) + +datasets_short = [x[:5] for x in datasets] + +print '\nVisual-Inertial Odometry' +print row_format.format(*datasets_short) + +print row_format.format(*vio['ate']) +#print row_format.format(*vio['time']) +print row_format.format(*vio['num_frames']) + + + + diff --git a/scripts/eval_full/run_evaluations_tumvi.sh b/scripts/eval_full/run_evaluations_tumvi.sh new file mode 100755 index 0000000..321cc2c --- /dev/null +++ b/scripts/eval_full/run_evaluations_tumvi.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e +set -x + +DATASET_PATH=/data/tumvi/512_16/ + +DATASETS=( +dataset-corridor1_512_16 +dataset-magistrale1_512_16 +dataset-room1_512_16 +dataset-slides1_512_16 +) + + +folder_name=eval_results_tumvi +mkdir $folder_name + + + +for d in ${DATASETS[$CI_NODE_INDEX-1]}; do + ../../build/basalt_vio --dataset-path $DATASET_PATH/$d --cam-calib /usr/etc/basalt/tumvi_512_eucm_calib.json \ + --dataset-type euroc --show-gui 1 --config-path /usr/etc/basalt/tumvi_512_config.json \ + --result-path $folder_name/vio_$d --save-trajectory tum + + mv trajectory.txt $folder_name/${d}_basalt_poses.txt + +done + +#./gen_results_tumvi.py $folder_name > euroc_tumvi.txt \ No newline at end of file