basalt/data/iccv21/experiments-iccv.toml

[options]
base_path = "$config_dir"
output_path = "./tables/experiments-iccv"
cache_dir = "./tables/cache"
show_values_failed_runs = false
screenread = true
#overwrite_cache = true


###################
## abbreviate some sequence names for more compact tables / legends
[seq_displayname_mapping]
"tumvi-corridor1" = "tumvi-corr1"
"tumvi-corridor2" = "tumvi-corr2"
"tumvi-magistrale1" = "tumvi-mag1"
"tumvi-magistrale2" = "tumvi-mag2"


###################
## define method names in table headers / legends globally
[[substitutions]]
DISPLAY_NAME_SQRTVIO64 = "$\\sqrt{VIO}$-64"
DISPLAY_NAME_SQRTVIO32 = "$\\sqrt{VIO}$-32"
DISPLAY_NAME_SQRTVO64 = "$\\sqrt{VO}$-64"
DISPLAY_NAME_SQRTVO32 = "$\\sqrt{VO}$-32"
DISPLAY_NAME_SQVIO64 = "$VIO$-64"
DISPLAY_NAME_SQVIO32 = "$VIO$-32"
DISPLAY_NAME_SQVO64 = "$VO$-64"
DISPLAY_NAME_SQVO32 = "$VO$-32"


###################
## where to find experimental runs
[[substitutions]]

EXP_PATTERN_VIO = "01_iccv_all/*-*/vio_*/"
EXP_PATTERN_VO = "01_iccv_all/*-*/vo_*/"

#EXP_PATTERN_VIO = "02_iccv_runtime/*-*/vio_*/"
#EXP_PATTERN_VO = "02_iccv_runtime/*-*/vo_*/"


###################
## which kind of plots to show
[[substitutions]]
SHOW_TRAJECTORY_PLOTS = true
SHOW_EIGENVALUE_PLOTS = true
SHOW_NULLSPACE_PLOTS = true


###################
## define which sequences to show in plots
[[substitutions]]
SEQUENCES_EUROC = [
    "eurocMH01",
    "eurocMH02",
    "eurocMH03",
    "eurocMH04",
    "eurocMH05",
    "eurocV101",
    "eurocV102",
    "eurocV103",
    "eurocV201",
    "eurocV202",
]
SEQUENCES_TUMVI = [
    "tumvi-corridor1",
    "tumvi-corridor2",
    "tumvi-magistrale1",
    "tumvi-magistrale2",
    "tumvi-room1",
    "tumvi-room2",
    "tumvi-slides1",
    "tumvi-slides2",
]
SEQUENCES_KITTI = [
    "kitti00",
    "kitti02",
    "kitti03",
    "kitti04",
    "kitti05",
    "kitti06",
    "kitti07",
    "kitti08",
    "kitti09",
    "kitti10",
]


###################
## VIO experiments
[[experiments]]
name = "vio_sqrt-nsllt-qr-64"
display_name = "${DISPLAY_NAME_SQRTVIO64}${}^{\\textrm{NS}}_{\\textrm{QR}}$"
pattern = "${EXP_PATTERN_VIO}/*_sqrt-nsllt-qr_double_*"
[[experiments]]
name = "vio_sqrt-nsllt-qr-32"
display_name = "${DISPLAY_NAME_SQRTVIO32}${}^{\\textrm{NS}}_{\\textrm{QR}}$"
pattern = "${EXP_PATTERN_VIO}/*_sqrt-nsllt-qr_float_*"
[[experiments]]
name = "vio_sqrt-sc-sc-64"
display_name = "${DISPLAY_NAME_SQRTVIO64}${}^{\\textrm{SC}}_{\\textrm{SC}}$"
pattern = "${EXP_PATTERN_VIO}/*_sqrt-sc-sc_double_*"
[[experiments]]
name = "vio_sqrt-sc-sc-32"
display_name = "${DISPLAY_NAME_SQRTVIO32}${}^{\\textrm{SC}}_{\\textrm{SC}}$"
pattern = "${EXP_PATTERN_VIO}/*_sqrt-sc-sc_float_*"
[[experiments]]
name = "vio_sq-sc-sc-64"
display_name = "${DISPLAY_NAME_SQVIO64}${}^{\\textrm{SC}}_{\\textrm{SC}}$"
pattern = "${EXP_PATTERN_VIO}/*_sq-sc-sc_double_*"
[[experiments]]
name = "vio_sq-sc-sc-32"
display_name = "${DISPLAY_NAME_SQVIO32}${}^{\\textrm{SC}}_{\\textrm{SC}}$"
pattern = "${EXP_PATTERN_VIO}/*_sq-sc-sc_float_*"

###################
## VIO experiments -- aliases with shorter display names for plots
[[experiments]]
name = "vio_sqrt-64"
display_name = "${DISPLAY_NAME_SQRTVIO64}"
extend = "vio_sqrt-nsllt-qr-64"
[[experiments]]
name = "vio_sqrt-32"
display_name = "${DISPLAY_NAME_SQRTVIO32}"
extend = "vio_sqrt-nsllt-qr-32"
[[experiments]]
name = "vio_sq-64"
display_name = "${DISPLAY_NAME_SQVIO64}"
extend = "vio_sq-sc-sc-64"
[[experiments]]
name = "vio_sq-32"
display_name = "${DISPLAY_NAME_SQVIO32}"
extend = "vio_sq-sc-sc-32"



###################
## VO experiments
[[experiments]]
name = "vo_sqrt-nsllt-qr-64"
display_name = "${DISPLAY_NAME_SQRTVO64}${}^{\\textrm{NS}}_{\\textrm{QR}}$"
pattern = "${EXP_PATTERN_VO}/*_sqrt-nsllt-qr_double_*"
[[experiments]]
name = "vo_sqrt-nsllt-qr-32"
display_name = "${DISPLAY_NAME_SQRTVO32}${}^{\\textrm{NS}}_{\\textrm{QR}}$"
pattern = "${EXP_PATTERN_VO}/*_sqrt-nsllt-qr_float_*"
[[experiments]]
name = "vo_sqrt-sc-sc-64"
display_name = "${DISPLAY_NAME_SQRTVO64}${}^{\\textrm{SC}}_{\\textrm{SC}}$"
pattern = "${EXP_PATTERN_VO}/*_sqrt-sc-sc_double_*"
[[experiments]]
name = "vo_sqrt-sc-sc-32"
display_name = "${DISPLAY_NAME_SQRTVO32}${}^{\\textrm{SC}}_{\\textrm{SC}}$"
pattern = "${EXP_PATTERN_VO}/*_sqrt-sc-sc_float_*"
[[experiments]]
name = "vo_sq-sc-sc-64"
display_name = "${DISPLAY_NAME_SQVO64}${}^{\\textrm{SC}}_{\\textrm{SC}}$"
pattern = "${EXP_PATTERN_VO}/*_sq-sc-sc_double_*"
[[experiments]]
name = "vo_sq-sc-sc-32"
display_name = "${DISPLAY_NAME_SQVO32}${}^{\\textrm{SC}}_{\\textrm{SC}}$"
pattern = "${EXP_PATTERN_VO}/*_sq-sc-sc_float_*"

###################
## VIO experiments -- aliases with shorter display names for plots
[[experiments]]
name = "vo_sqrt-64"
display_name = "${DISPLAY_NAME_SQRTVO64}"
extend = "vo_sqrt-nsllt-qr-64"
[[experiments]]
name = "vo_sqrt-32"
display_name = "${DISPLAY_NAME_SQRTVO32}"
extend = "vo_sqrt-nsllt-qr-32"
[[experiments]]
name = "vo_sq-64"
display_name = "${DISPLAY_NAME_SQVO64}"
extend = "vo_sq-sc-sc-64"
[[experiments]]
name = "vo_sq-32"
display_name = "${DISPLAY_NAME_SQVO32}"
extend = "vo_sq-sc-sc-32"



################################################################################
################################################################################
[[results]]
class = "section"
name = "Accuracy VIO & VO"
show = true

###################
## accuracy results table for all VIO sequences
[[results]]
class = "results_table"
name = "vio ate"
show = true
export_latex = "vio_ate"
metrics_legend = false
rotate_header = true
escape_latex_header = false
vertical_bars = false
color_failed = ""
filter_regex = "euroc|tumvi.*1|tumvi.*2"
experiments = [
    "vio_sqrt-nsllt-qr-64",
    "vio_sqrt-nsllt-qr-32",
    "vio_sq-sc-sc-64",
    "vio_sq-sc-sc-32",
]
metrics = [
    "ate_rmse",
]

###################
## accuracy results table for all VO sequences
[[results]]
class = "results_table"
name = "vo ate"
show = true
export_latex = "vo_ate"
metrics_legend = false
rotate_header = true
escape_latex_header = false
vertical_bars = false
color_failed = ""
#filter_regex = "kitti00|kitti0[2-9]|kitti10"
#filter_regex = "kitti"
override_as_failed = ["kitti01"]
experiments = [
    "vo_sqrt-nsllt-qr-64",
    "vo_sqrt-nsllt-qr-32",
    "vo_sq-sc-sc-64",
    "vo_sq-sc-sc-32",
]
metrics = [
    {name = "ate_rmse", failed_threshold = 100, decimals = 2},
]



################################################################################
################################################################################
[[results]]
class = "section"
name = "Runtime VIO & VO"
show = true

###################
## runtime results table for all VIO sequences
[[results]]
class = "results_table"
name = "vio time optimize / marginalize"
show = true
export_latex = "vio_runtime"
metrics_legend = false
rotate_header = true
escape_latex_header = false
vertical_bars = false
color_failed = ""
multirow = false
filter_regex = "euroc|tumvi.*1|tumvi.*2"
experiments = [
    "vio_sqrt-nsllt-qr-64",
    "vio_sqrt-nsllt-qr-32",
    "vio_sq-sc-sc-64",
    "vio_sq-sc-sc-32",
]
metrics = [
    {name = "time_opt", decimals = 1},
    {name = "time_marg", decimals = 1},
]

###################
## runtime results table for all VO sequences
[[results]]
class = "results_table"
name = "vo time optimize / marginalize"
show = true
export_latex = "vo_runtime"
metrics_legend = false
rotate_header = true
escape_latex_header = false
vertical_bars = false
color_failed = ""
multirow = false
#filter_regex = "kitti00|kitti0[2-9]|kitti10"
#filter_regex = "kitti"
override_as_failed = ["kitti01"]
experiments = [
    "vo_sqrt-nsllt-qr-64",
    "vo_sqrt-nsllt-qr-32",
    "vo_sq-sc-sc-64",
    "vo_sq-sc-sc-32",
]
metrics = [
    "time_opt",
    "time_marg",
]



################################################################################
################################################################################
[[results]]
class = "section"
name = "Ablation VIO & VO"
show = true

###################
## ablation study table for VIO summarized over all EuRoC sequences
[[results]]
class = "summarize_sequences_table"
name = "summary euroc ablation"
export_latex = "vio_euroc_ablation"
escape_latex_header = false
filter_regex = "euroc"
experiments =  [
    "vio_sqrt-nsllt-qr-64",
    "vio_sqrt-nsllt-qr-32",
    "vio_sqrt-sc-sc-64",
    "vio_sqrt-sc-sc-32",
]
metrics = [
    {name = "ate_rmse", display_name = "ATE [m]", geometric_mean = false,  decimals = 3},
    {name = "time_exec_realtimefactor", display_name = "real-time", geometric_mean = true, format_string = "{:.1f}x"},
    {name = "time_exec", display_name = "t total [s]", geometric_mean = false, decimals = 1},
    {name = "time_opt", display_name = "t opt [s]", geometric_mean = false, decimals = 1},
    {name = "time_marg", display_name = "t marg [s]", geometric_mean = false,  decimals = 1},
#    {name = "time_marg", display_name = "t marg (frac. of meas)", relative_to_metric = "time_measure", geometric_mean = true,  format_string = "{:.2f}"},
#    {name = "time_opt", display_name = "t opt (frac. of meas)", relative_to_metric = "time_measure", geometric_mean = true, format_string = "{:.2f}"},
#    {name = "time_marg", display_name = "t marg (frac. of total)", relative_to_metric = "time_exec", geometric_mean = true,  format_string = "{:.2f}"},
#    {name = "time_opt", display_name = "t opt (frac. of total)", relative_to_metric = "time_exec", geometric_mean = true, format_string = "{:.2f}"},
    {name = "avg_num_it", display_name = "avg #it", geometric_mean = false, decimals = 1},
    {name = "avg_num_it_failed", display_name = "avg #it-failed", geometric_mean = false, decimals = 1},
]

###################
## ablation study table for VO summarized over all Kitti sequences (w/o kitti01)
[[results]]
class = "summarize_sequences_table"
name = "summary kitti (w/o kitti01) ablation"
show = true
export_latex = "vo_kitti_ablation"
escape_latex_header = false
filter_regex = "kitti00|kitti0[2-9]|kitti10"
experiments =  [
    "vo_sqrt-nsllt-qr-64",
    "vo_sqrt-nsllt-qr-32",
    "vo_sqrt-sc-sc-64",
    "vo_sqrt-sc-sc-32",
]
metrics = [
    {name = "ate_rmse", display_name = "ATE [m]", geometric_mean = false,  decimals = 3},
    {name = "time_exec_realtimefactor", display_name = "real-time", geometric_mean = true, format_string = "{:.1f}x"},
    {name = "time_exec", display_name = "t total [s]", geometric_mean = false, decimals = 1},
    {name = "time_opt", display_name = "t opt [s]", geometric_mean = false, decimals = 1},
    {name = "time_marg", display_name = "t marg [s]", geometric_mean = false,  decimals = 1},    
    {name = "avg_num_it", display_name = "avg #it", geometric_mean = false, decimals = 1},
    {name = "avg_num_it_failed", display_name = "avg #it-failed", geometric_mean = false, decimals = 1},
]



################################################################################
################################################################################
[[results]]
class = "section"
name = "Trajectories EuRoC"
show = "<SHOW_TRAJECTORY_PLOTS>"

###################
## template for trajectory plots for a set of sequences
[[templates]]
_name = "TRAJECTORY"
_arguments = ["SEQUENCE", "EXPERIMENTS", "AXES"]
class = "plot"
name = "trajectory"
type = "trajectory"
figsize = [4.0,2.5]
trajectory_axes = "<AXES>"
sequence = "<SEQUENCE>"
experiments = "<EXPERIMENTS>"

###################
## instantiate trajectory plot template for EuRoC
[[results]]
  [results._template]
  _name = "TRAJECTORY"
  AXES = "xy"
  SEQUENCE = {_argument = "product", _value = "<SEQUENCES_EUROC>"}
  EXPERIMENTS = [
    "vio_sqrt-32",
    "vio_sq-32",
  ]



################################################################################
################################################################################
[[results]]
class = "section"
name = "Eigenvalues EuRoC"
show = "<SHOW_EIGENVALUE_PLOTS>"

###################
## template for eigenvalue plots for a set of sequences
[[templates]]
_name = "EIGENVALUES"
_arguments = ["SEQUENCE", "EXPERIMENTS"]
class = "plot"
type = "eigenvalues"
name = "vio"
#legend_loc = "upper left"
#ylim.top = 1e6
#ylim.bottom = -1e6
sequence = "<SEQUENCE>"
experiments = "<EXPERIMENTS>"

###################
## instantiate eigenvalue plot template for EuRoC
[[results]]
  [results._template]
  _name = "EIGENVALUES"
  SEQUENCE = {_argument = "product", _value = "<SEQUENCES_EUROC>"}
  EXPERIMENTS = [
    "vio_sqrt-64",
    "vio_sqrt-32",
    "vio_sq-64",
    "vio_sq-32",
  ]



################################################################################
################################################################################
[[results]]
class = "section"
name = "Nullspace EuRoC"
show = "<SHOW_NULLSPACE_PLOTS>"

###################
## template for nullspace plots for a set of sequences
[[templates]]
_name = "NULLSPACE"
_arguments = ["SEQUENCE", "EXPERIMENTS"]
class = "plot"
type = "nullspace"
name = "vio"
#legend_loc = "lower right"
sequence = "<SEQUENCE>"
experiments = "<EXPERIMENTS>"

###################
## instantiate nullspace plot template for EuRoC
[[results]]
  [results._template]
  _name = "NULLSPACE"
  SEQUENCE = {_argument = "product", _value = "<SEQUENCES_EUROC>"}
  EXPERIMENTS = [
    "vio_sqrt-64",
    "vio_sqrt-32",
    "vio_sq-64",
    "vio_sq-32",
  ]


################################################################################
################################################################################
[[results]]
class = "section"
name = "Trajectories TUMVI"
show = "<SHOW_TRAJECTORY_PLOTS>"

###################
## instantiate trajectory plot template for TUMVI
[[results]]
  [results._template]
  _name = "TRAJECTORY"
  AXES = "xy"
  SEQUENCE = {_argument = "product", _value = "<SEQUENCES_TUMVI>"}
  EXPERIMENTS = [
      "vio_sqrt-32",
      "vio_sq-32",
  ]


################################################################################
################################################################################
[[results]]
class = "section"
name = "Eigenvalues TUMVI"
show = "<SHOW_EIGENVALUE_PLOTS>"

###################
## instantiate eigenvalue plot template for TUMVI
[[results]]
  [results._template]
  _name = "EIGENVALUES"
  SEQUENCE = {_argument = "product", _value = "<SEQUENCES_TUMVI>"}
  EXPERIMENTS = [
    "vio_sqrt-64",
    "vio_sqrt-32",
    "vio_sq-64",
    "vio_sq-32",
  ]


################################################################################
################################################################################
[[results]]
class = "section"
name = "Nullspace TUMVI"
show = "<SHOW_NULLSPACE_PLOTS>"

###################
## instantiate nullspace plot template for TUMVI
[[results]]
  [results._template]
  _name = "NULLSPACE"
  SEQUENCE = {_argument = "product", _value = "<SEQUENCES_TUMVI>"}
  EXPERIMENTS = [
    "vio_sqrt-64",
    "vio_sqrt-32",
    "vio_sq-64",
    "vio_sq-32",
  ]


################################################################################
################################################################################
[[results]]
class = "section"
name = "Trajectories Kitti"
show = "<SHOW_TRAJECTORY_PLOTS>"

###################
## instantiate trajectory plot template for Kitti
[[results]]
  [results._template]
  _name = "TRAJECTORY"
  AXES = "xz"
  SEQUENCE = {_argument = "product", _value = "<SEQUENCES_KITTI>"}
  EXPERIMENTS = [
      "vo_sqrt-32",
      "vo_sq-32",
  ]


################################################################################
################################################################################
[[results]]
class = "section"
name = "Eigenvalues Kitti"
show = "<SHOW_EIGENVALUE_PLOTS>"

###################
## instantiate eigenvalue plot template for Kitti
[[results]]
  [results._template]
  _name = "EIGENVALUES"
  SEQUENCE = {_argument = "product", _value = "<SEQUENCES_KITTI>"}
  EXPERIMENTS = [
    "vo_sqrt-64",
    "vo_sqrt-32",
    "vo_sq-64",
    "vo_sq-32",
  ]


################################################################################
################################################################################
[[results]]
class = "section"
name = "Nullspace Kitti"
show = "<SHOW_NULLSPACE_PLOTS>"

###################
## instantiate nullspace plot template for Kitti
[[results]]
  [results._template]
  _name = "NULLSPACE"
  SEQUENCE = {_argument = "product", _value = "<SEQUENCES_KITTI>"}
  EXPERIMENTS = [
    "vo_sqrt-64",
    "vo_sqrt-32",
    "vo_sq-64",
    "vo_sq-32",
  ]