Better evaluation output

This commit is contained in:
Vladyslav Usenko 2019-07-29 16:14:02 +02:00
parent 547ed06a13
commit 92955209b8
3 changed files with 62 additions and 12 deletions

View File

@ -2,39 +2,65 @@
import os import os
import sys import sys
import json
datasets = ['MH_01_easy', 'MH_02_easy', 'MH_03_medium', 'MH_04_difficult', datasets = ['Sequence', 'MH_01_easy', 'MH_02_easy', 'MH_03_medium', 'MH_04_difficult',
'MH_05_difficult', 'V1_01_easy', 'V1_02_medium', 'MH_05_difficult', 'V1_01_easy', 'V1_02_medium',
'V1_03_difficult', 'V2_01_easy', 'V2_02_medium'] 'V1_03_difficult', 'V2_01_easy', 'V2_02_medium']
# Other results. # Other results.
results_vio = [] results_vio = ['VIO RMS ATE [m]']
results_mapping = [] time_vio = ['VIO Time [s]']
num_frames_vio = ['VIO Num. Frames']
results_mapping = ['MAP RMS ATE [m]']
time_mapping = ['MAP Time [s]']
num_frames_mapping = ['MAP Num. KFs']
out_dir = sys.argv[1] out_dir = sys.argv[1]
for key in datasets: for key in datasets[1:]:
fname = out_dir + '/vio_' + key fname = out_dir + '/vio_' + key
if os.path.isfile(fname): if os.path.isfile(fname):
with open(fname, 'r') as f: with open(fname, 'r') as f:
res = round(float(f.read()), 3) j = json.load(f)
res = round(j['rms_ate'], 3)
results_vio.append(float(res)) results_vio.append(float(res))
time_vio.append(round(j['exec_time_ns']*1e-9, 3))
num_frames_vio.append(j['num_frames'])
else: else:
results_vio.append(float('Inf')) results_vio.append(float('Inf'))
time_vio.append(float('Inf'))
num_frames_vio.append(float('Inf'))
fname = out_dir + '/mapper_' + key fname = out_dir + '/mapper_' + key
if os.path.isfile(fname): if os.path.isfile(fname):
with open(fname, 'r') as f: with open(fname, 'r') as f:
res = round(float(f.read()), 3) j = json.load(f)
res = round(j['rms_ate'], 3)
results_mapping.append(float(res)) results_mapping.append(float(res))
time_mapping.append(round(j['exec_time_ns']*1e-9, 3))
num_frames_mapping.append(j['num_frames'])
else: else:
results_mapping.append(float('Inf')) results_mapping.append(float('Inf'))
time_mapping.append(float('Inf'))
num_frames_mapping.append(float('Inf'))
row_format ="{:>17}" * (len(datasets)) row_format ="{:>17}" * (len(datasets))
print row_format.format(*datasets) print row_format.format(*datasets)
print row_format.format(*results_vio)
print row_format.format(*results_mapping) print row_format.format(*results_vio)
print row_format.format(*time_vio)
print row_format.format(*num_frames_vio)
print '\n'
print row_format.format(*datasets)
print row_format.format(*results_mapping)
print row_format.format(*time_mapping)
print row_format.format(*num_frames_mapping)

View File

@ -325,6 +325,7 @@ int main(int argc, char** argv) {
std::this_thread::sleep_for(std::chrono::milliseconds(50)); std::this_thread::sleep_for(std::chrono::milliseconds(50));
} }
} else { } else {
auto time_start = std::chrono::high_resolution_clock::now();
// optimize(); // optimize();
detect(); detect();
match(); match();
@ -333,11 +334,21 @@ int main(int argc, char** argv) {
filter(); filter();
optimize(); optimize();
auto time_end = std::chrono::high_resolution_clock::now();
if (!result_path.empty()) { if (!result_path.empty()) {
double error = alignButton(); double error = alignButton();
auto exec_time_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
time_end - time_start);
std::ofstream os(result_path); std::ofstream os(result_path);
os << error << std::endl; {
cereal::JSONOutputArchive ar(os);
ar(cereal::make_nvp("rms_ate", error));
ar(cereal::make_nvp("num_frames", nrf_mapper->getFramePoses().size()));
ar(cereal::make_nvp("exec_time_ns", exec_time_ns.count()));
}
os.close(); os.close();
} }
} }

View File

@ -346,6 +346,8 @@ int main(int argc, char** argv) {
})); }));
} }
auto time_start = std::chrono::high_resolution_clock::now();
if (show_gui) { if (show_gui) {
pangolin::CreateWindowAndBind("Main", 1800, 1000); pangolin::CreateWindowAndBind("Main", 1800, 1000);
@ -469,11 +471,22 @@ int main(int argc, char** argv) {
if (t3.get()) t3->join(); if (t3.get()) t3->join();
t4.join(); t4.join();
auto time_end = std::chrono::high_resolution_clock::now();
if (!result_path.empty()) { if (!result_path.empty()) {
double error = basalt::alignSVD(vio_t_ns, vio_t_w_i, gt_t_ns, gt_t_w_i); double error = basalt::alignSVD(vio_t_ns, vio_t_w_i, gt_t_ns, gt_t_w_i);
auto exec_time_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
time_end - time_start);
std::ofstream os(result_path); std::ofstream os(result_path);
os << error << std::endl; {
cereal::JSONOutputArchive ar(os);
ar(cereal::make_nvp("rms_ate", error));
ar(cereal::make_nvp("num_frames",
vio_dataset->get_image_timestamps().size()));
ar(cereal::make_nvp("exec_time_ns", exec_time_ns.count()));
}
os.close(); os.close();
} }