#include "depth_guide_v2.h" #include "core/imgui_utility.hpp" #include "image_process/process_funcs.h" #include app_depth_guide_v2::app_depth_guide_v2(const create_config &_conf) { main_conf = _conf; auto conf = main_conf.ext_conf; // initialize object manager OBJ_SAVE(img_color, image_u8c3()); OBJ_SAVE(img_depth, image_f32c1()); OBJ_SAVE(img_remap, image_ptr()); OBJ_SAVE(pc_raw, pc_ptr()); OBJ_SAVE(scene_out, scene_ptr()); OBJ_SAVE(img_out_left, image_ptr()); OBJ_SAVE(img_out_right, image_ptr()); OBJ_SAVE(img_out, image_ptr()); // initialize meta OBJ_PIN_META(pc_raw, META_SERIES_NAME, meta_hash("pc_raw")); OBJ_PIN_META(img_remap, META_SERIES_NAME, meta_hash("img_remap")); OBJ_PIN_META(pc_raw, META_DEPTH_MIN, (float) 0.0f); // TODO: make adjustable OBJ_PIN_META(pc_raw, META_DEPTH_MAX, (float) 3000.0f); OBJ_SIG(img_color)->connect([](auto _) { gen_pc_rgbd::call( {img_color, img_depth, img_remap, pc_raw, default_cuda_stream}); }); OBJ_SIG(pc_raw)->connect([this](auto _) { aug_manager->update(aug_helper->get_camera()); auto out_conf = camera_augment_helper_v2::output_config{ .size = aug_stereo->get_suggest_size(), .stream = default_cuda_stream, }; out_conf.img_name = img_out_left; aug_left->render_image(out_conf); out_conf.img_name = img_out_right; aug_right->render_image(out_conf); }); // initialize sophiar sophiar_thread = std::make_unique([conf_path = LOAD_STR("sophiar_config")] { sophiar::run_sophiar(conf_path); }); sophiar_conn = std::make_unique(); sophiar_start_var = LOAD_STR("sophiar_start_var"); // initialize modules auto orb_cam_conf = orb_camera_ui::create_config{ .cf_name = img_color, .df_name = img_depth, .remap_name = img_remap, .ctx = main_conf.asio_ctx, .stream = default_cuda_stream, }; orb_cam = std::make_unique(orb_cam_conf); auto aug_list_v1 = augment_manager::item_list_from_yaml(LOAD_LIST("augment_list")); auto aug_list = augment_manager_v2::item_list_from_v1(aug_list_v1); auto aug_conf = augment_manager_v2::create_config{ .scene_name = scene_out, .sophiar_conn = sophiar_conn.get(), .stream = default_cuda_stream, }; aug_conf.item_list.push_back( {.type = augment_manager_v2::ITEM_PC, .disp_name = "PC Raw", .name = pc_raw, .transform_var = LOAD_STR("camera_transform_var")}); std::copy(aug_list.begin(), aug_list.end(), std::back_inserter(aug_conf.item_list)); aug_manager = std::make_unique(aug_conf); auto guide_conf = cursor_guide::create_config{ .item_list = cursor_guide::item_list_from_aug(aug_list), .sophiar_conn = sophiar_conn.get(), }; aug_guide = std::make_unique(guide_conf); auto view_conf = camera_augment_helper_v2::create_config{ .camera = camera_augment_helper_v2::create_config::freedom_camera_config{ .transform_var = LOAD_STR("camera_transform_var")}, .sophiar_conn = sophiar_conn.get(), .manager = aug_manager.get(), .act_guide = aug_guide.get(), .ctx = main_conf.asio_ctx, }; aug_helper = std::make_unique(view_conf); aug_helper->set_active_guidance(true); view_conf.camera = camera_augment_helper_v2::create_config::relative_camera_config{ .parent = aug_helper.get(), .transform = glm::translate(glm::vec3(-baseline_dis / 2, 0, 0))}; aug_left = std::make_unique(view_conf); aug_left->sync_with(aug_helper.get()); view_conf.camera = camera_augment_helper_v2::create_config::relative_camera_config{ .parent = aug_helper.get(), .transform = glm::translate(glm::vec3(baseline_dis / 2, 0, 0))}; aug_right = std::make_unique(view_conf); aug_right->sync_with(aug_helper.get()); auto output_size = cv::Size( LOAD_NUMBER(int, "output_width"), LOAD_NUMBER(int, "output_height")); auto stereo_conf = stereo_augment_helper::create_config{ .left_name = img_out_left, .right_name = img_out_right, .out_name = img_out, .fbo_size = output_size, .flip_image = true, .stream = default_cuda_stream, }; aug_stereo = std::make_unique(stereo_conf); auto out_conf = image_streamer::create_config{ .img_name = img_out, .encode_scene = false, .asio_ctx = main_conf.asio_ctx, .cuda_ctx = main_conf.cuda_ctx, .stream = default_cuda_stream }; out_streamer = std::make_unique(out_conf); auto reg_conf = registration_config{ .conn = sophiar_conn.get(), .probe_model_path = LOAD_STR("probe_model"), }; reg.reset(registration::create(reg_conf)); for (auto reg_item: LOAD_LIST("registration_list")) { auto item_conf = registration_target::from_yaml(reg_item); reg->add_target(item_conf); } } void app_depth_guide_v2::show_ui() { if (ImGui::Begin("Depth Guide Control")) { ImGui::PushItemWidth(200); if (ImGui::CollapsingHeader("Camera")) { auto id_guard = imgui_id_guard("camera"); orb_cam->show(); } if (ImGui::CollapsingHeader("Augment")) { ImGui::SeparatorText("Tracker"); if (ImGui::Button("Start")) { start_tracking(); } ImGui::SameLine(); ImGui::Checkbox("Registration", &enable_reg); { ImGui::SeparatorText("Scene"); auto id_guard = imgui_id_guard("augment_scene"); aug_manager->show(); } { ImGui::SeparatorText("Camera"); auto id_guard = imgui_id_guard("augment_camera"); aug_helper->show(); } { ImGui::SeparatorText("Stereo"); auto id_guard = imgui_id_guard("stereo"); aug_stereo->show(); } } if (ImGui::CollapsingHeader("Streamer")) { auto id_guard = imgui_id_guard("streamer"); out_streamer->show(); } ImGui::PopItemWidth(); } ImGui::End(); if (enable_reg) { reg->process(); reg->show(); } } void app_depth_guide_v2::render_background() { aug_helper->render(); } void app_depth_guide_v2::start_tracking() { // sophiar_conn->start_object(sophiar_start_var); // work in another thread to prevent blocking auto t = std::thread([this] { auto conn = sophiar::local_connection(); conn.start_object(sophiar_start_var); }); t.detach(); } app_depth_guide_v2::~app_depth_guide_v2() { // sophiar sophiar::stop_sophiar(); sophiar_thread->join(); }