/** BSD 3-Clause License This file is part of the Basalt project. https://gitlab.com/VladyslavUsenko/basalt.git Copyright (c) 2019, Vladyslav Usenko and Nikolaus Demmel. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // GUI functions void draw_image_overlay(pangolin::View& v, size_t cam_id); void draw_scene(); void load_data(const std::string& calib_path); bool next_step(); bool prev_step(); void draw_plots(); void alignButton(); void alignDeviceButton(); // Pangolin variables constexpr int UI_WIDTH = 200; using Button = pangolin::Var>; pangolin::DataLog imu_data_log, vio_data_log, error_data_log; pangolin::Plotter* plotter; pangolin::Var show_frame("ui.show_frame", 0, 0, 1500); pangolin::Var show_obs("ui.show_obs", true, false, true); pangolin::Var show_ids("ui.show_ids", false, false, true); pangolin::Var show_est_pos("ui.show_est_pos", true, false, true); pangolin::Var show_est_vel("ui.show_est_vel", false, false, true); pangolin::Var show_est_bg("ui.show_est_bg", false, false, true); pangolin::Var show_est_ba("ui.show_est_ba", false, false, true); pangolin::Var show_gt("ui.show_gt", true, false, true); pangolin::Var show_device_gt("ui.show_device_gt", true, false, true); Button next_step_btn("ui.next_step", &next_step); Button prev_step_btn("ui.prev_step", &prev_step); pangolin::Var continue_btn("ui.continue", false, false, true); pangolin::Var continue_fast("ui.continue_fast", true, false, true); Button align_svd_btn("ui.align_svd", &alignButton); pangolin::Var follow("ui.follow", true, false, true); // Visualization variables std::unordered_map vis_map; tbb::concurrent_bounded_queue out_vis_queue; tbb::concurrent_bounded_queue out_state_queue; std::vector vio_t_ns; Eigen::vector vio_t_w_i; std::vector gt_t_ns; Eigen::vector gt_t_w_i; std::string marg_data_path; size_t last_frame_processed = 0; tbb::concurrent_unordered_map timestamp_to_id; std::mutex m; std::condition_variable cv; bool step_by_step = false; // VIO variables basalt::Calibration calib; basalt::VioDatasetPtr vio_dataset; basalt::VioConfig vio_config; basalt::OpticalFlowBase::Ptr opt_flow_ptr; basalt::VioEstimatorBase::Ptr vio; // Feed functions void feed_images() { std::cout << "Started input_data thread " << std::endl; for (size_t i = 0; i < vio_dataset->get_image_timestamps().size(); i++) { if (step_by_step) { std::unique_lock lk(m); cv.wait(lk); } basalt::OpticalFlowInput::Ptr data(new basalt::OpticalFlowInput); data->t_ns = vio_dataset->get_image_timestamps()[i]; data->img_data = vio_dataset->get_image_data(data->t_ns); timestamp_to_id[data->t_ns] = i; opt_flow_ptr->input_queue.push(data); } // Indicate the end of the sequence opt_flow_ptr->input_queue.push(nullptr); std::cout << "Finished input_data thread " << std::endl; } void feed_imu() { for (size_t i = 0; i < vio_dataset->get_gyro_data().size(); i++) { basalt::ImuData::Ptr data(new basalt::ImuData); data->t_ns = vio_dataset->get_gyro_data()[i].timestamp_ns; data->accel = vio_dataset->get_accel_data()[i].data; data->gyro = vio_dataset->get_gyro_data()[i].data; vio->imu_data_queue.push(data); } vio->imu_data_queue.push(nullptr); } int main(int argc, char** argv) { bool show_gui = true; bool print_queue = false; bool terminate = false; std::string cam_calib_path; std::string dataset_path; std::string dataset_type; std::string config_path; std::string result_path; int num_threads = 0; CLI::App app{"App description"}; app.add_option("--show-gui", show_gui, "Show GUI"); app.add_option("--cam-calib", cam_calib_path, "Ground-truth camera calibration used for simulation.") ->required(); app.add_option("--dataset-path", dataset_path, "Path to dataset.") ->required(); app.add_option("--dataset-type", dataset_type, "Dataset type .") ->required(); app.add_option("--marg-data", marg_data_path, "Path to folder where marginalization data will be stored."); app.add_option("--print-queue", print_queue, "Print queue."); app.add_option("--config-path", config_path, "Path to config file."); app.add_option("--result-path", result_path, "Path to result file where the system will write RMSE ATE."); app.add_option("--num-threads", num_threads, "Number of threads."); app.add_option("--step-by-step", step_by_step, "Path to config file."); if (num_threads > 0) { tbb::task_scheduler_init init(num_threads); } try { app.parse(argc, argv); } catch (const CLI::ParseError& e) { return app.exit(e); } if (!config_path.empty()) { vio_config.load(config_path); } load_data(cam_calib_path); { basalt::DatasetIoInterfacePtr dataset_io = basalt::DatasetIoFactory::getDatasetIo(dataset_type); dataset_io->read(dataset_path); vio_dataset = dataset_io->get_data(); vio_dataset->get_image_timestamps().erase( vio_dataset->get_image_timestamps().begin()); show_frame.Meta().range[1] = vio_dataset->get_image_timestamps().size() - 1; show_frame.Meta().gui_changed = true; opt_flow_ptr = basalt::OpticalFlowFactory::getOpticalFlow(vio_config, calib); for (size_t i = 0; i < vio_dataset->get_gt_pose_data().size(); i++) { gt_t_ns.push_back(vio_dataset->get_gt_timestamps()[i]); gt_t_w_i.push_back(vio_dataset->get_gt_pose_data()[i].translation()); } } const int64_t start_t_ns = vio_dataset->get_image_timestamps().front(); { vio = basalt::VioEstimatorFactory::getVioEstimator( vio_config, calib, 0.0001, basalt::constants::g); vio->initialize(Eigen::Vector3d::Zero(), Eigen::Vector3d::Zero()); opt_flow_ptr->output_queue = &vio->vision_data_queue; if (show_gui) vio->out_vis_queue = &out_vis_queue; vio->out_state_queue = &out_state_queue; } basalt::MargDataSaver::Ptr marg_data_saver; if (!marg_data_path.empty()) { marg_data_saver.reset(new basalt::MargDataSaver(marg_data_path)); vio->out_marg_queue = &marg_data_saver->in_marg_queue; // Save gt. { std::string p = marg_data_path + "/gt.cereal"; std::ofstream os(p, std::ios::binary); { cereal::BinaryOutputArchive archive(os); archive(gt_t_ns); archive(gt_t_w_i); } os.close(); } } vio_data_log.Clear(); std::thread t1(&feed_images); std::thread t2(&feed_imu); std::shared_ptr t3; if (show_gui) t3.reset(new std::thread([&]() { basalt::VioVisualizationData::Ptr data; while (true) { out_vis_queue.pop(data); if (data.get()) { vis_map[data->t_ns] = data; } else { break; } } std::cout << "Finished t3" << std::endl; })); std::thread t4([&]() { basalt::PoseVelBiasState::Ptr data; while (true) { out_state_queue.pop(data); if (!data.get()) break; int64_t t_ns = data->t_ns; // std::cerr << "t_ns " << t_ns << std::endl; Sophus::SE3d T_w_i = data->T_w_i; Eigen::Vector3d vel_w_i = data->vel_w_i; Eigen::Vector3d bg = data->bias_gyro; Eigen::Vector3d ba = data->bias_accel; vio_t_ns.emplace_back(data->t_ns); vio_t_w_i.emplace_back(T_w_i.translation()); if (show_gui) { std::vector vals; vals.push_back((t_ns - start_t_ns) * 1e-9); for (int i = 0; i < 3; i++) vals.push_back(vel_w_i[i]); for (int i = 0; i < 3; i++) vals.push_back(T_w_i.translation()[i]); for (int i = 0; i < 3; i++) vals.push_back(bg[i]); for (int i = 0; i < 3; i++) vals.push_back(ba[i]); vio_data_log.Log(vals); } } std::cout << "Finished t4" << std::endl; }); std::shared_ptr t5; if (print_queue) { t5.reset(new std::thread([&]() { while (!terminate) { std::cout << "opt_flow_ptr->input_queue " << opt_flow_ptr->input_queue.size() << " opt_flow_ptr->output_queue " << opt_flow_ptr->output_queue->size() << " out_state_queue " << out_state_queue.size() << std::endl; std::this_thread::sleep_for(std::chrono::seconds(1)); } })); } auto time_start = std::chrono::high_resolution_clock::now(); if (show_gui) { pangolin::CreateWindowAndBind("Main", 1800, 1000); glEnable(GL_DEPTH_TEST); pangolin::View& img_view_display = pangolin::CreateDisplay() .SetBounds(0.4, 1.0, pangolin::Attach::Pix(UI_WIDTH), 0.4) .SetLayout(pangolin::LayoutEqual); pangolin::View& plot_display = pangolin::CreateDisplay().SetBounds( 0.0, 0.4, pangolin::Attach::Pix(UI_WIDTH), 1.0); plotter = new pangolin::Plotter(&imu_data_log, 0.0, 100, -10.0, 10.0, 0.01f, 0.01f); plot_display.AddDisplay(*plotter); pangolin::CreatePanel("ui").SetBounds(0.0, 1.0, 0.0, pangolin::Attach::Pix(UI_WIDTH)); std::vector> img_view; while (img_view.size() < calib.intrinsics.size()) { std::shared_ptr iv(new pangolin::ImageView); size_t idx = img_view.size(); img_view.push_back(iv); img_view_display.AddDisplay(*iv); iv->extern_draw_function = std::bind(&draw_image_overlay, std::placeholders::_1, idx); } Eigen::Vector3d cam_p(-0.5, -3, -5); cam_p = vio->getT_w_i_init().so3() * calib.T_i_c[0].so3() * cam_p; pangolin::OpenGlRenderState camera( pangolin::ProjectionMatrix(640, 480, 400, 400, 320, 240, 0.001, 10000), pangolin::ModelViewLookAt(cam_p[0], cam_p[1], cam_p[2], 0, 0, 0, pangolin::AxisZ)); pangolin::View& display3D = pangolin::CreateDisplay() .SetAspect(-640 / 480.0) .SetBounds(0.4, 1.0, 0.4, 1.0) .SetHandler(new pangolin::Handler3D(camera)); while (!pangolin::ShouldQuit()) { glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); if (follow) { size_t frame_id = show_frame; int64_t t_ns = vio_dataset->get_image_timestamps()[frame_id]; auto it = vis_map.find(t_ns); if (it != vis_map.end()) { auto T_w_i = it->second->states.back(); T_w_i.so3() = Sophus::SO3d(); camera.Follow(T_w_i.matrix()); } } display3D.Activate(camera); glClearColor(1.0f, 1.0f, 1.0f, 1.0f); draw_scene(); img_view_display.Activate(); if (show_frame.GuiChanged()) { for (size_t cam_id = 0; cam_id < calib.intrinsics.size(); cam_id++) { size_t frame_id = static_cast(show_frame); int64_t timestamp = vio_dataset->get_image_timestamps()[frame_id]; std::vector img_vec = vio_dataset->get_image_data(timestamp); pangolin::GlPixFormat fmt; fmt.glformat = GL_LUMINANCE; fmt.gltype = GL_UNSIGNED_SHORT; fmt.scalable_internal_format = GL_LUMINANCE16; img_view[cam_id]->SetImage( img_vec[cam_id].img->ptr, img_vec[cam_id].img->w, img_vec[cam_id].img->h, img_vec[cam_id].img->pitch, fmt); } draw_plots(); } if (show_est_vel.GuiChanged() || show_est_pos.GuiChanged() || show_est_ba.GuiChanged() || show_est_bg.GuiChanged()) { draw_plots(); } pangolin::FinishFrame(); if (continue_btn) { if (!next_step()) std::this_thread::sleep_for(std::chrono::milliseconds(50)); } else { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } if (continue_fast) { int64_t t_ns = vio->last_processed_t_ns; if (timestamp_to_id.count(t_ns)) { show_frame = timestamp_to_id[t_ns]; show_frame.Meta().gui_changed = true; } if (vio->finished) { continue_fast = false; } } } } terminate = true; t1.join(); t2.join(); if (t3.get()) t3->join(); t4.join(); if (t5.get()) t5->join(); auto time_end = std::chrono::high_resolution_clock::now(); if (!result_path.empty()) { double error = basalt::alignSVD(vio_t_ns, vio_t_w_i, gt_t_ns, gt_t_w_i); auto exec_time_ns = std::chrono::duration_cast( time_end - time_start); std::ofstream os(result_path); { cereal::JSONOutputArchive ar(os); ar(cereal::make_nvp("rms_ate", error)); ar(cereal::make_nvp("num_frames", vio_dataset->get_image_timestamps().size())); ar(cereal::make_nvp("exec_time_ns", exec_time_ns.count())); } os.close(); } return 0; } void draw_image_overlay(pangolin::View& v, size_t cam_id) { UNUSED(v); // size_t frame_id = show_frame; // basalt::TimeCamId tcid = // std::make_pair(vio_dataset->get_image_timestamps()[frame_id], // cam_id); size_t frame_id = show_frame; auto it = vis_map.find(vio_dataset->get_image_timestamps()[frame_id]); if (show_obs) { glLineWidth(1.0); glColor3f(1.0, 0.0, 0.0); glEnable(GL_BLEND); glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); if (it != vis_map.end() && cam_id < it->second->projections.size()) { const auto& points = it->second->projections[cam_id]; if (points.size() > 0) { double min_id = points[0][2], max_id = points[0][2]; for (const auto& points2 : it->second->projections) for (const auto& p : points2) { min_id = std::min(min_id, p[2]); max_id = std::max(max_id, p[2]); } for (const auto& c : points) { const float radius = 6.5; float r, g, b; getcolor(c[2] - min_id, max_id - min_id, b, g, r); glColor3f(r, g, b); pangolin::glDrawCirclePerimeter(c[0], c[1], radius); if (show_ids) pangolin::GlFont::I().Text("%d", int(c[3])).Draw(c[0], c[1]); } } glColor3f(1.0, 0.0, 0.0); pangolin::GlFont::I() .Text("Tracked %d points", points.size()) .Draw(5, 20); } } } void draw_scene() { glPointSize(3); glColor3f(1.0, 0.0, 0.0); glEnable(GL_BLEND); glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); glColor3ubv(cam_color); if (!vio_t_w_i.empty()) { Eigen::vector sub_gt(vio_t_w_i.begin(), vio_t_w_i.begin() + show_frame); pangolin::glDrawLineStrip(sub_gt); } glColor3ubv(gt_color); if (show_gt) pangolin::glDrawLineStrip(gt_t_w_i); size_t frame_id = show_frame; int64_t t_ns = vio_dataset->get_image_timestamps()[frame_id]; auto it = vis_map.find(t_ns); if (it != vis_map.end()) { for (const auto& p : it->second->states) for (size_t i = 0; i < calib.T_i_c.size(); i++) render_camera((p * calib.T_i_c[i]).matrix(), 2.0f, state_color, 0.1f); for (const auto& p : it->second->frames) for (size_t i = 0; i < calib.T_i_c.size(); i++) render_camera((p * calib.T_i_c[i]).matrix(), 2.0f, pose_color, 0.1f); for (size_t i = 0; i < calib.T_i_c.size(); i++) render_camera((it->second->states.back() * calib.T_i_c[i]).matrix(), 2.0f, cam_color, 0.1f); glColor3ubv(pose_color); pangolin::glDrawPoints(it->second->points); } pangolin::glDrawAxis(Sophus::SE3d().matrix(), 1.0); } void load_data(const std::string& calib_path) { std::ifstream os(calib_path, std::ios::binary); if (os.is_open()) { cereal::JSONInputArchive archive(os); archive(calib); std::cout << "Loaded camera with " << calib.intrinsics.size() << " cameras" << std::endl; } else { std::cerr << "could not load camera calibration " << calib_path << std::endl; std::abort(); } } bool next_step() { if (show_frame < int(vio_dataset->get_image_timestamps().size()) - 1) { show_frame = show_frame + 1; show_frame.Meta().gui_changed = true; cv.notify_one(); return true; } else { return false; } } bool prev_step() { if (show_frame > 1) { show_frame = show_frame - 1; show_frame.Meta().gui_changed = true; return true; } else { return false; } } void draw_plots() { plotter->ClearSeries(); plotter->ClearMarkers(); if (show_est_pos) { plotter->AddSeries("$0", "$4", pangolin::DrawingModeLine, pangolin::Colour::Red(), "position x", &vio_data_log); plotter->AddSeries("$0", "$5", pangolin::DrawingModeLine, pangolin::Colour::Green(), "position y", &vio_data_log); plotter->AddSeries("$0", "$6", pangolin::DrawingModeLine, pangolin::Colour::Blue(), "position z", &vio_data_log); } if (show_est_vel) { plotter->AddSeries("$0", "$1", pangolin::DrawingModeLine, pangolin::Colour::Red(), "velocity x", &vio_data_log); plotter->AddSeries("$0", "$2", pangolin::DrawingModeLine, pangolin::Colour::Green(), "velocity y", &vio_data_log); plotter->AddSeries("$0", "$3", pangolin::DrawingModeLine, pangolin::Colour::Blue(), "velocity z", &vio_data_log); } if (show_est_bg) { plotter->AddSeries("$0", "$7", pangolin::DrawingModeLine, pangolin::Colour::Red(), "gyro bias x", &vio_data_log); plotter->AddSeries("$0", "$8", pangolin::DrawingModeLine, pangolin::Colour::Green(), "gyro bias y", &vio_data_log); plotter->AddSeries("$0", "$9", pangolin::DrawingModeLine, pangolin::Colour::Blue(), "gyro bias z", &vio_data_log); } if (show_est_ba) { plotter->AddSeries("$0", "$10", pangolin::DrawingModeLine, pangolin::Colour::Red(), "accel bias x", &vio_data_log); plotter->AddSeries("$0", "$11", pangolin::DrawingModeLine, pangolin::Colour::Green(), "accel bias y", &vio_data_log); plotter->AddSeries("$0", "$12", pangolin::DrawingModeLine, pangolin::Colour::Blue(), "accel bias z", &vio_data_log); } double t = vio_dataset->get_image_timestamps()[show_frame] * 1e-9; plotter->AddMarker(pangolin::Marker::Vertical, t, pangolin::Marker::Equal, pangolin::Colour::White()); } void alignButton() { basalt::alignSVD(vio_t_ns, vio_t_w_i, gt_t_ns, gt_t_w_i); }