added tumvi eval

This commit is contained in:
Vladyslav Usenko 2020-03-06 12:51:28 +01:00
parent f8227d2e22
commit 2926020df7
3 changed files with 118 additions and 0 deletions

View File

@ -106,6 +106,23 @@ stages:
- cd scripts/eval_full - cd scripts/eval_full
- ./run_evaluations.sh - ./run_evaluations.sh
# template to evaluate on TUM-VI sequences
.eval_tumvi_template: &eval_tumvi_definition
stage: eval
parallel: 4
tags: [docker, dataset-eval]
variables:
GIT_STRATEGY: none
artifacts:
paths:
- scripts/eval_full/eval_results_tumvi/*
expire_in: 1 week
script:
- dpkg -i deb_bionic/*.deb
- cd scripts/eval_full
- ./run_evaluations_tumvi.sh
.eval_euroc_template: &eval_kitti_definition .eval_euroc_template: &eval_kitti_definition
stage: eval stage: eval
parallel: 10 parallel: 10
@ -222,6 +239,20 @@ eval_euroc:
- master - master
allow_failure: false allow_failure: false
# evaluate on TUM-VI sequences
eval_tumvi_master:
<<: *eval_tumvi_definition
only:
- master
# evaluate on TUM-VI sequences
eval_tumvi:
<<: *eval_tumvi_definition
when: manual
except:
- master
allow_failure: false
# evaluate on KITTI sequences # evaluate on KITTI sequences
eval_kitti_master: eval_kitti_master:
<<: *eval_kitti_definition <<: *eval_kitti_definition
@ -248,6 +279,7 @@ gen_results:
paths: paths:
- euroc_results.txt - euroc_results.txt
- kitti_results.txt - kitti_results.txt
- tumvi_results.txt
- scripts/eval_full/eval_results/* - scripts/eval_full/eval_results/*
script: script:
- cd scripts/eval_full - cd scripts/eval_full
@ -255,8 +287,11 @@ gen_results:
- cat euroc_results.txt - cat euroc_results.txt
- ./gen_results_kitti.py eval_results_kitti > kitti_results.txt - ./gen_results_kitti.py eval_results_kitti > kitti_results.txt
- cat kitti_results.txt - cat kitti_results.txt
- ./gen_results_tumvi.py eval_results_tumvi > tumvi_results.txt
- cat kitti_results.txt
- mv euroc_results.txt ../../ - mv euroc_results.txt ../../
- mv kitti_results.txt ../../ - mv kitti_results.txt ../../
- mv tumvi_results.txt ../../
# deploy deb packages # deploy deb packages
deploy: deploy:

View File

@ -0,0 +1,53 @@
#!/usr/bin/env python
import os
import sys
import json
datasets = ['Seq.', 'dataset-corridor1_512_16', 'dataset-magistrale1_512_16', 'dataset-room1_512_16', 'dataset-slides1_512_16']
# Other results.
vio = {
'ate' : ['VIO RMS ATE [m]'],
'time' : ['VIO Time [s]'],
'num_frames' : ['VIO Num. Frames']
}
out_dir = sys.argv[1]
def load_data(x, prefix, key):
fname = out_dir + '/' + prefix + '_' + key
if os.path.isfile(fname):
with open(fname, 'r') as f:
j = json.load(f)
res = round(j['rms_ate'], 3)
x['ate'].append(float(res))
x['time'].append(round(j['exec_time_ns']*1e-9, 3))
x['num_frames'].append(j['num_frames'])
else:
x['ate'].append(float('Inf'))
x['time'].append(float('Inf'))
x['num_frames'].append(float('Inf'))
for key in datasets[1:]:
load_data(vio, 'vio', key)
row_format ="{:>17}" + "{:>13}" * (len(datasets)-1)
datasets_short = [x[:5] for x in datasets]
print '\nVisual-Inertial Odometry'
print row_format.format(*datasets_short)
print row_format.format(*vio['ate'])
#print row_format.format(*vio['time'])
print row_format.format(*vio['num_frames'])

View File

@ -0,0 +1,30 @@
#!/bin/bash
set -e
set -x
DATASET_PATH=/data/tumvi/512_16/
DATASETS=(
dataset-corridor1_512_16
dataset-magistrale1_512_16
dataset-room1_512_16
dataset-slides1_512_16
)
folder_name=eval_results_tumvi
mkdir $folder_name
for d in ${DATASETS[$CI_NODE_INDEX-1]}; do
../../build/basalt_vio --dataset-path $DATASET_PATH/$d --cam-calib /usr/etc/basalt/tumvi_512_eucm_calib.json \
--dataset-type euroc --show-gui 1 --config-path /usr/etc/basalt/tumvi_512_config.json \
--result-path $folder_name/vio_$d --save-trajectory tum
mv trajectory.txt $folder_name/${d}_basalt_poses.txt
done
#./gen_results_tumvi.py $folder_name > euroc_tumvi.txt