Render line as mesh.

src/impl/apps/endo_guide/endo_guide.cpp

@@ -48,12 +48,13 @@ app_endo_guide::app_endo_guide(const create_config &_conf) {
 //    });
 
     auto bg_viewer_conf = image_viewer::create_config{
-            .mode = VIEW_COLOR_ONLY, .flip_y = true,
+            .mode = VIEW_STEREO, .flip_y = true,
             .stream = default_cuda_stream,
     };
-    auto &bg_extra_conf = bg_viewer_conf.extra.color;
-    bg_extra_conf.fmt = COLOR_RGB;
-    bg_extra_conf.name = left_out;
+    auto &bg_extra_conf = bg_viewer_conf.extra.stereo;
+    bg_extra_conf.c_fmt = COLOR_RGB;
+    bg_extra_conf.left_name = img_out;
+    bg_extra_conf.right_name = left_out;
     bg_viewer = std::make_unique<image_viewer>(bg_viewer_conf);
 
     auto saver_conf = image_saver::create_config{.ctx = main_conf.asio_ctx};
@@ -134,11 +135,11 @@ app_endo_guide::app_endo_guide(const create_config &_conf) {
     };
     monitor = std::make_unique<sophiar_monitor>(monitor_conf);
 
-//    auto sim_info = camera_calibrator::simulate_info_type{
-//            .data_path = "/home/tpx/project/DepthGuide/cmake-build-debug/exp-20240605/calib/calib_data_c5.txt",
-//            .img_size = cv::Size(1920, 1080),
-//    };
-//    cam_calib->simulate_process(sim_info);
+    auto sim_info = camera_calibrator::simulate_info_type{
+            .data_path = "/home/tpx/project/DepthGuide/cmake-build-debug/exp-20240605/calib/calib_data_c5.txt",
+            .img_size = cv::Size(1920, 1080),
+    };
+    cam_calib->simulate_process(sim_info);
 
 //    auto reg_conf = registration_config{
 //            .conn = sophiar_conn.get(),
@@ -183,6 +184,14 @@ void app_endo_guide::show_ui() {
                 ImGui::SeparatorText("Scene");
                 auto id_guard = imgui_id_guard("augment_scene");
                 aug_manager->show();
+                if (ImGui::TreeNode("Endo Camera")) {
+                    aug_helper->get_camera_helper()->show();
+                    ImGui::TreePop();
+                }
+                if (ImGui::TreeNode("MVS Camera")) {
+                    left_cam.aug_helper->get_camera_helper()->show();
+                    ImGui::TreePop();
+                }
             }
         }
 
@@ -197,14 +206,18 @@ void app_endo_guide::show_ui() {
         }
 
         if (ImGui::CollapsingHeader("Debug")) {
-            if (ImGui::TreeNode("Image Saver")) {
-                out_saver->show();
+            if (ImGui::TreeNode("Background")) {
+                bg_viewer->show();
                 ImGui::TreePop();
             }
-            if (ImGui::TreeNode("Evaluation")) {
-                eval_calib->show();
+            if (ImGui::TreeNode("Image Saver")) {
+                out_saver->show();
                 ImGui::TreePop();
             }
+//            if (ImGui::TreeNode("Evaluation")) {
+//                eval_calib->show();
+//                ImGui::TreePop();
+//            }
         }
     }
     ImGui::End();
@@ -216,7 +229,14 @@ void app_endo_guide::show_ui() {
 }
 
 void app_endo_guide::render_background() {
+    auto vp = query_viewport_size();
+    glViewport(0.25f * vp.width, 0, 0.75f * vp.width, vp.height);
     bg_viewer->render();
+    for (auto k = 0; k < 3; ++k) {
+        auto delta = vp.height / 3.f;
+        glViewport(0, k * delta, 0.25f * vp.width, delta);
+        acl->render_aux(aug_manager.get(), 2 - k);
+    }
 }
 
 void app_endo_guide::start_tracking() {

src/module/guidance/acl_guide.h

@@ -3,6 +3,7 @@
 
 #include "core/local_connection.h" // sophiar
 #include "core/yaml_utility.hpp"
+#include "module/augment_manager_v2.h"
 #include "render/render_scene.h"
 
 #include <memory>
@@ -34,6 +35,8 @@ public:
 
     void pre_render_slot(const scene_ptr &info);
 
+    void render_aux(augment_manager_v2 *m, int index);
+
 private:
     struct impl;
     std::unique_ptr<impl> pimpl;

src/module/guidance/guide_helper.h

@@ -5,4 +5,10 @@
 
 mesh_ptr get_sphere_symbol(float radius);
 
+mesh_ptr get_line_symbol();
+
+glm::mat4 get_line_transform(float radius,
+                             const glm::vec3 &pa,
+                             const glm::vec3 &pb);
+
 #endif //DEPTHGUIDE_GUIDE_HELPER_H

src/module/guidance/impl/acl_guide.cpp

@@ -30,9 +30,15 @@ struct acl_guide::impl {
             if (!keypoint.has_value()) return {};
             auto trans = i->conf.sophiar_conn
                     ->query_transform_variable(transform_var_name);
-            if (!trans.has_value()) return {};
-            return transform_p(to_mat4(*trans), *keypoint);
+            if (trans.has_value()) {
+                last_trans = to_mat4(*trans);
+            }
+            if (!last_trans.has_value()) return {};
+            return transform_p(*last_trans, *keypoint);
         }
+
+    private:
+        std::optional<glm::mat4> last_trans;
     };
 
     item_store_type femur, tibia;
@@ -44,17 +50,26 @@ struct acl_guide::impl {
 
     glm::vec3 line_color = glm::vec3(1, 0, 0);
     glm::vec3 drill_color = glm::vec3(0, 1, 0);
-    GLfloat line_width = 5.f; // pix
-    GLfloat point_size = 2.f; // mm
-    ogl_range_info lw_info;
+    GLfloat line_radius = 1.f; // mm
+    GLfloat point_radius = 2.5f; // mm
     GLfloat drill_pos = 100.f, drill_neg = -10.f;
 
+    static constexpr float pi = std::numbers::pi;
+    glm::mat4 aux_view_mat[3];
+
+    static constexpr auto tmp_file_name = "acl_plan.txt";
+
     impl(const create_config &_conf) {
         conf = _conf;
         *(item_base_type *) &femur = conf.femur; // copy base type part
         *(item_base_type *) &tibia = conf.tibia;
 
-        lw_info = query_line_width_info();
+        aux_view_mat[0] = glm::rotate(pi, glm::vec3(1, 0, 0))
+                          * glm::rotate(-pi / 2, glm::vec3(0, 0, 1)); // front view
+        aux_view_mat[1] = glm::rotate(aux_view_mat[0],
+                                      -pi / 2.f, glm::vec3(0, 1, 0)); // side view
+        aux_view_mat[2] = glm::rotate(aux_view_mat[0],
+                                      -pi / 2.f, glm::vec3(1, 0, 0)); // top view
     }
 
     void collect_point() {
@@ -104,6 +119,24 @@ struct acl_guide::impl {
         }
     }
 
+    void save_plan() {
+        auto v1 = femur.keypoint, v2 = tibia.keypoint;
+        if (!v1.has_value() || !v2.has_value()) return;
+        auto fout = std::fstream(tmp_file_name, std::fstream::out);
+        fout << fmt::format("{} {} {} ", v1->x, v1->y, v1->z);
+        fout << fmt::format("{} {} {}\n", v2->x, v2->y, v2->z);
+    }
+
+    void load_plan() {
+        if (!std::filesystem::exists(tmp_file_name)) return;
+        auto fin = std::fstream(tmp_file_name, std::fstream::in);
+        glm::vec3 kp;
+        fin >> kp.x >> kp.y >> kp.z;
+        femur.keypoint = kp;
+        fin >> kp.x >> kp.y >> kp.z;
+        tibia.keypoint = kp;
+    }
+
     void show() {
         if (!is_running) {
             if (ImGui::Button("Start")) {
@@ -113,6 +146,14 @@ struct acl_guide::impl {
             if (ImGui::Button("Stop")) {
                 post(*conf.ctx, [this] { stop(); });
             }
+            ImGui::SameLine();
+            if (ImGui::Button("Save")) {
+                save_plan();
+            }
+            ImGui::SameLine();
+            if (ImGui::Button("Load")) {
+                load_plan();
+            }
         }
         if (is_running) {
             ImGui::Checkbox("Enable", &enable_collect);
@@ -126,9 +167,8 @@ struct acl_guide::impl {
                 auto guard = imgui_valid_style_guard(tibia.keypoint.has_value());
                 ImGui::RadioButton("Tibia", &next_index, ACL_TIBIA);
             }
-            ImGui::DragFloat("Line Width", &line_width, lw_info.delta,
-                             lw_info.min, lw_info.max, "%.01f");
-            ImGui::DragFloat("Point Radius", &point_size, 0.1, 0.1, 10, "%.1f");
+            ImGui::DragFloat("Line Radius", &line_radius, 0.1, 0.1, 5., "%.1f");
+            ImGui::DragFloat("Point Radius", &point_radius, 0.1, 0.1, 5., "%.1f");
             ImGui::ColorEdit3("Line Color", glm::value_ptr(line_color));
 
             ImGui::PushItemWidth(75);
@@ -147,13 +187,25 @@ struct acl_guide::impl {
 
         auto add_point = [&](glm::vec3 p) {
             auto item_info = scene_render_info::mesh_info{
-                    .mesh = get_sphere_symbol(point_size),
+                    .mesh = get_sphere_symbol(point_radius),
                     .material = {.ambient = glm::vec3(), .diffuse = line_color,},
             };
             info->items.push_back(
                     {.info = item_info, .transform = glm::translate(p),});
         };
 
+        auto add_line = [&](glm::vec3 pa, glm::vec3 pb,
+                            glm::vec3 color,
+                            glm::mat4 transform = glm::mat4(1.)) {
+            auto item_info = scene_render_info::mesh_info{
+                    .mesh = get_line_symbol(),
+                    .material = {.ambient = glm::vec3(), .diffuse = color,},
+            };
+            info->items.push_back(
+                    {.info = item_info, .transform = transform
+                                                     * get_line_transform(line_radius, pa, pb),});
+        };
+
         auto v1 = femur.world_point(this);
         if (v1.has_value()) {
             add_point(*v1);
@@ -166,32 +218,41 @@ struct acl_guide::impl {
 
         if (v1.has_value() && v2.has_value()) {
             auto p1 = *v1, p2 = *v2;
-
             static constexpr auto extra_length = 100.f; // mm
             auto dir = glm::normalize(p1 - p2);
             auto r1 = p1 + extra_length * dir;
             auto r2 = p2 - extra_length * dir;
-            auto item_info = scene_render_info::line_info{
-                    .p1 = r1, .p2 = r2,
-                    .color = line_color, .line_width = line_width,
-            };
-            info->items.push_back(
-                    {.info = item_info, .transform = glm::mat4(1.f),});
+            add_line(r1, r2, line_color);
         }
 
         // show drill
         if (auto trans = conf.sophiar_conn
                     ->query_transform_variable(conf.drill_transform_var_name);
                 trans.has_value()) {
-            auto item_info = scene_render_info::line_info{
-                    .p1 = glm::vec3(0, 0, drill_neg), .p2 = glm::vec3(0, 0, drill_pos),
-                    .color = drill_color, .line_width = line_width,
-            };
-            info->items.push_back(
-                    {.info = item_info, .transform = to_mat4(*trans),});
+            add_line(glm::vec3(0, 0, drill_neg),
+                     glm::vec3(0, 0, drill_pos),
+                     drill_color, to_mat4(*trans));
         }
     }
 
+    void render_aux(augment_manager_v2 *m, int index) {
+        static constexpr auto aux_dis = 200.f; // mm
+        static constexpr auto aux_fov = 60.f; // in degree
+
+        auto v1 = femur.world_point(this);
+        auto v2 = tibia.world_point(this);
+        if (!v1.has_value() || !v2.has_value()) return;
+        auto p1 = *v1, p2 = *v2, pc = (p1 + p2) / 2.f;
+        assert(index >= 0 && index < 3);
+        auto cam = camera_info{
+                .transform = glm::translate(pc)
+                             * aux_view_mat[index]
+                             * glm::translate(glm::vec3(0, 0, -aux_dis)),
+                .fov = aux_fov,
+        };
+        m->render(cam);
+    }
+
 };
 
 void acl_guide::create_config::fill_from_yaml(const YAML::Node &conf) {
@@ -220,4 +281,8 @@ void acl_guide::show() {
 
 void acl_guide::pre_render_slot(const scene_ptr &info) {
     pimpl->pre_render_slot(info);
+}
+
+void acl_guide::render_aux(augment_manager_v2 *m, int index) {
+    pimpl->render_aux(m, index);
 }

src/module/guidance/impl/guide_helper.cpp

@@ -1,6 +1,9 @@
 #include "module/guidance/guide_helper.h"
+#include "core/math_helper.hpp"
 
+#include <vtkCylinderSource.h>
 #include <vtkSphereSource.h>
+#include <vtkTriangleFilter.h>
 
 namespace guide_helper_impl {
 
@@ -8,6 +11,8 @@ namespace guide_helper_impl {
             std::unordered_map<float, mesh_ptr>;
     mesh_cache_type sphere_cache;
 
+    mesh_ptr line_symbol;
+
 }
 
 using namespace guide_helper_impl;
@@ -27,3 +32,35 @@ mesh_ptr get_sphere_symbol(float radius) {
     sphere_cache.emplace(radius, mesh);
     return mesh;
 }
+
+mesh_ptr get_line_symbol() {
+    if (line_symbol != nullptr) [[likely]] {
+        return line_symbol;
+    }
+    vtkNew<vtkCylinderSource> source;
+    source->SetRadius(1.);
+    source->SetHeight(1.);
+    source->SetCenter(0., .5, 0);
+    source->SetResolution(16);
+    source->CappingOn();
+    source->Update();
+    vtkNew<vtkTriangleFilter> filter;
+    filter->SetInputData(source->GetOutput());
+    filter->Update();
+    line_symbol = mesh_type::from_vtk(filter->GetOutput());
+    return line_symbol;
+}
+
+glm::mat4 get_line_transform(float r,
+                             const glm::vec3 &pa,
+                             const glm::vec3 &pb) {
+    auto y_dir = glm::vec3(0, 1, 0);
+    auto line_dir = glm::normalize(pb - pa);
+    auto rot_dir = glm::cross(y_dir, line_dir);
+    auto rot_ang = std::acos(glm::dot(y_dir, line_dir));
+    auto ret = glm::rotate(rot_ang, rot_dir);
+    ret = glm::translate(pa) * ret;
+    auto scala = glm::vec3(r, glm::distance(pa, pb), r);
+    ret = ret * glm::scale(scala);
+    return ret;
+}

src/module/impl/augment_manager_v2_impl.h

@@ -20,7 +20,7 @@ struct augment_manager_v2::impl {
         bool flip_y = false;
 
         // for mesh
-        float ambient_factor = 0.5f;
+        float ambient_factor = 0.f;
 
         bool enable_depth_alpha = false;
         float alpha_factor = 0.1;
@@ -59,7 +59,7 @@ struct augment_manager_v2::impl {
     smart_cuda_stream *stream = nullptr;
 
     bool enable = true; // enable this module
-    bool enable_light_follow_camera = false;
+    bool enable_light_follow_camera = true;
     bool ignore_missing = false; // show item even if missing from the tracker
 
     vgm::Vec3 light_direction = {0.0f, 0.0f, -1.0f};