Switched to old render pipeline.

data/config_remote_ar_v2.yaml

@@ -5,10 +5,20 @@ right_camera_name: "RightEye"
 
 ndi_ip: 192.168.1.202
 ndi_port: 8765
-sophiar_config: ./sophiar_config_endo_guide.json
+sophiar_config: ./sophiar_config_endo_guide_oldhead.json
 sophiar_start_name: tracker_all
 camera_ref_transform_var: camera_ref_in_tracker
 
+monitor:
+  - name: Camera
+    var: camera_ref_in_tracker
+  - name: MVS Camera
+    var: mvs_ref_in_tracker
+  - name: Head
+    var: head_ref_in_tracker
+  - name: Probe
+    var: probe_ref_in_tracker
+
 left_camera:
   fx: 3569.14196423103
   fy: 3568.02459571424
@@ -31,14 +41,13 @@ right_camera:
 
 left_camera_transform: left_camera_in_tracker_denoised
 right_camera_transform: right_camera_in_tracker_denoised
-scene_info:
-  - name: BoneTransform
-    transform: femur_in_tracker_denoised
-    children:
-      - name: Bone
-        model: /home/tpx/ext/data/models/old_head/bone.stl
-      - name: Nerve
-        model: /home/tpx/ext/data/models/old_head/nerve.stl
+augment_list:
+  - name: Head
+    transform_var: head_in_tracker_denoised
+    model: /home/tpx/ext/data/models/old_head/bone.stl
+  - name: ExtraA
+    transform_var: head_in_tracker_denoised
+    model: /home/tpx/ext/data/models/old_head/nerve.stl
 
 probe_model: /home/tpx/data/models/Probe.stl
 registration_list:

data/sophiar_config_endo_guide_oldhead.json

@@ -16,13 +16,13 @@
       "name": "head_in_head_ref",
       "type": "transform_obj",
       "value": [
-        -158.57010140440562,
-        109.08408048026303,
-        311.7159241940251,
-        0.6426166606045021,
-        -0.3019245048766848,
-        -0.18681286301124206,
-        0.6789597742738753
+        233.96470027218044,
+        42.187788316685904,
+        306.58865752584546,
+        0.8190555328954703,
+        -0.26342165437712295,
+        0.24510986086006864,
+        -0.4468536921048027
       ]
     },
     {
@@ -116,9 +116,9 @@
             "name": "probe",
             "parent": "probe_ref",
             "transform": [
-              -0.69,
-              -13.95,
-              -186.84,
+              0,
+              0,
+              0,
               1,
               0,
               0,
@@ -154,26 +154,26 @@
             "name": "left_camera",
             "parent": "mvs_ref",
             "transform": [
-              33.2854,
-              -37.7917,
-              -24.4682,
-              0.6318,
-              0.3547,
-              0.4719,
-              -0.5023
+              1.69529294997093,
+              -40.8504461910221,
+              -38.4356249156025,
+              0.328217190586743,
+              0.623382258094322,
+              0.318037852337519,
+              -0.634444607965983
             ]
           },
           {
             "name": "right_camera",
             "parent": "mvs_ref",
             "transform": [
-              37.3992,
-              -56.7108,
-              -85.8847,
-              0.6304,
-              0.3605,
-              0.4738,
-              -0.4982
+              0.0941597771490770,
+              -41.0431819166041,
+              -105.682270133671,
+              0.329203554783163,
+              0.626106195513828,
+              0.316605284270741,
+              -0.631962930423275
             ]
           }
         ]
@@ -222,13 +222,13 @@
       "type": "ndi_interface",
       "name": "ndi",
       "init_config": {
-        "address_type": "serial",
-        "ip": "10.0.0.5",
+        "address_type": "ethernet",
+        "ip": "192.168.1.202",
         "tcp_port": 8765,
         "com_port": "/dev/ttyUSB0",
         "tool_list": [
           {
-            "rom_path": "/home/tpx/data/roms/GlassProbe_4Ball_4_SHC.rom",
+            "rom_path": "/home/tpx/data/roms/Probe_Small_4Ball.rom",
             "serial_number": "3DD50000",
             "outputs": {
               "transform": "probe_ref_in_tracker"
@@ -242,7 +242,7 @@
             }
           },
           {
-            "rom_path": "/home/tpx/data/roms/Glass_4Ball_1_Camera_20240312.rom",
+            "rom_path": "/home/tpx/data/roms/NR_1_STCAM_20241115.rom",
             "outputs": {
               "transform": "mvs_ref_in_tracker"
             }

src/core/impl/image_utility_v2.cpp

@@ -65,6 +65,8 @@ void generic_image::impl::storage_info::reset() {
 }
 
 generic_image::impl::impl(generic_image::create_config conf) {
+    raise(SIGTRAP); // TODO: for debug
+
     type = conf.type;
     pix_fmt = conf.pixel;
     size = conf.size;

src/core_v2/cuda_helper.h

@@ -87,4 +87,6 @@ struct cuda_ogl_buffer_proxy : private boost::noncopyable {
     //@formatter:on
 };
 
+#define GET_CUDA_STREAM(s) (((s) == nullptr) ? current_cuda_stream() : (s)->cuda)
+
 #endif //CUDA_HELPER_H

src/image_process_v5/image_augment.cpp

@@ -12,44 +12,44 @@
 
 using namespace osg;
 
-namespace {
-    sp_image create_remap_image(const camera_intrinsic_v0 &info) {
-        const auto img_size = cv::Size(info.width, info.height);
-        const auto pix_num = img_size.area();
-        auto pix_mat = cv::Mat(1, pix_num, CV_32FC2);
-        for (auto iy = 0; iy < info.height; ++iy)
-            for (auto ix = 0; ix < info.width; ++ix) {
-                const auto id = iy * info.width + ix;
-                pix_mat.at<cv::Vec2f>(id) = cv::Vec2f(ix, iy);
-            }
-        auto ret = sp_image::create(CV_32FC2, img_size);
-        const auto write_helper = write_access_helper(ret.host());
-        auto ret_mat = cv::Mat(1, pix_num, CV_32FC2,
-                               ret.start_ptr(write_helper.ptr()));
-        cv::undistortPoints(pix_mat, ret_mat,
-                            info.intrinsic_cv_mat(), info.dist_cv_mat());
-        return ret;
-    }
-
-    cv::Rect2d calc_remap_range(const sp_image &remap) {
-        assert(CV_MAT_CN(remap.cv_type()) == 2);
-        const auto read_helper = read_access_helper(remap.host());
-        const auto cv_mat = remap.cv_mat(read_helper.ptr());
-        auto cv_xy = std::vector<cv::Mat>();
-        cv::split(cv_mat, cv_xy);
-        assert(cv_xy.size() == 2);
-        double x_min, x_max, y_min, y_max;
-        cv::minMaxLoc(cv_xy[0], &x_min, &x_max);
-        cv::minMaxLoc(cv_xy[1], &y_min, &y_max);
-        return {
-            x_min, y_min,
-            x_max - x_min, y_max - y_min
-        };
-    }
-
-    constexpr auto max_camera_y_axis_len = 3.f;
-    constexpr auto max_camera_x_axis_len = 6.f;
-}
+// namespace {
+//     sp_image create_remap_image(const camera_intrinsic_v0 &info) {
+//         const auto img_size = cv::Size(info.width, info.height);
+//         const auto pix_num = img_size.area();
+//         auto pix_mat = cv::Mat(1, pix_num, CV_32FC2);
+//         for (auto iy = 0; iy < info.height; ++iy)
+//             for (auto ix = 0; ix < info.width; ++ix) {
+//                 const auto id = iy * info.width + ix;
+//                 pix_mat.at<cv::Vec2f>(id) = cv::Vec2f(ix, iy);
+//             }
+//         auto ret = sp_image::create(CV_32FC2, img_size);
+//         const auto write_helper = write_access_helper(ret.host());
+//         auto ret_mat = cv::Mat(1, pix_num, CV_32FC2,
+//                                ret.start_ptr(write_helper.ptr()));
+//         cv::undistortPoints(pix_mat, ret_mat,
+//                             info.intrinsic_cv_mat(), info.dist_cv_mat());
+//         return ret;
+//     }
+//
+//     cv::Rect2d calc_remap_range(const sp_image &remap) {
+//         assert(CV_MAT_CN(remap.cv_type()) == 2);
+//         const auto read_helper = read_access_helper(remap.host());
+//         const auto cv_mat = remap.cv_mat(read_helper.ptr());
+//         auto cv_xy = std::vector<cv::Mat>();
+//         cv::split(cv_mat, cv_xy);
+//         assert(cv_xy.size() == 2);
+//         double x_min, x_max, y_min, y_max;
+//         cv::minMaxLoc(cv_xy[0], &x_min, &x_max);
+//         cv::minMaxLoc(cv_xy[1], &y_min, &y_max);
+//         return {
+//             x_min, y_min,
+//             x_max - x_min, y_max - y_min
+//         };
+//     }
+//
+//     constexpr auto max_camera_y_axis_len = 3.f;
+//     constexpr auto max_camera_x_axis_len = 6.f;
+// }
 
 struct image_augment::impl {
     create_config conf;
@@ -75,42 +75,43 @@ struct image_augment::impl {
     ref_ptr<Texture2DSP> out_tex;
 
     void update_remap() {
-        const auto info = OBJ_QUERY(camera_intrinsic_v0, conf.cam_info_name);
-        const auto info_ts = OBJ_TS(conf.cam_info_name);
-        const auto out_size = cv::Size(info.width, info.height);
-        const auto remap_np = create_remap_image(info);
-        const cv::Rect2f remap_range = calc_remap_range(remap_np);
-
-        const auto x_min = remap_range.x, x_max = x_min + remap_range.width;
-        const auto y_min = remap_range.y, y_max = y_min + remap_range.height;
-        auto x_len = std::max(-x_min, x_max);
-        auto y_len = std::max(-y_min, y_max);
-        x_len = std::min(x_len, max_camera_x_axis_len);
-        y_len = std::min(y_len, max_camera_y_axis_len);
-        const auto cam_fov = 2.f * std::atan(y_len);
-
-        const auto aspect = x_len / y_len;
-        auto aug_size = cv::Size(info.height * aspect, // width
-                                 info.height);
-        aug_size *= 2; // double render size
-        const auto remap_tex = image_remap_np_to_tex(remap_np, cam_fov, aspect);
-
-        MAIN_DETACH([=, this] {
-            setupTextureHelper(aug_tex, aug_size);
-            aug_cam->setViewport(0, 0, aug_size.width, aug_size.height);
-            aug_cam->attach(Camera::COLOR_BUFFER0, aug_tex);
-            aug_img_osg->setImageTex(aug_tex);
-            aug_img_osg->setRemapImage(remap_tex);
-            cam_conf.fov = glm::degrees(cam_fov);
-            cam_conf.aspect = aspect;
-
-            setupTextureHelper(out_tex, out_size);
-            const auto out_cam = viewer->getCamera();
-            out_cam->setViewport(0, 0, out_size.width, out_size.height);
-            out_cam->attach(Camera::COLOR_BUFFER0, out_tex);
-
-            last_update_ts = info_ts;
-        });
+        assert(false); // TODO: broken
+        // const auto info = OBJ_QUERY(camera_intrinsic_v0, conf.cam_info_name);
+        // const auto info_ts = OBJ_TS(conf.cam_info_name);
+        // const auto out_size = cv::Size(info.width, info.height);
+        // const auto remap_np = create_remap_image(info);
+        // const cv::Rect2f remap_range = calc_remap_range(remap_np);
+        //
+        // const auto x_min = remap_range.x, x_max = x_min + remap_range.width;
+        // const auto y_min = remap_range.y, y_max = y_min + remap_range.height;
+        // auto x_len = std::max(-x_min, x_max);
+        // auto y_len = std::max(-y_min, y_max);
+        // x_len = std::min(x_len, max_camera_x_axis_len);
+        // y_len = std::min(y_len, max_camera_y_axis_len);
+        // const auto cam_fov = 2.f * std::atan(y_len);
+        //
+        // const auto aspect = x_len / y_len;
+        // auto aug_size = cv::Size(info.height * aspect, // width
+        //                          info.height);
+        // aug_size *= 2; // double render size
+        // const auto remap_tex = image_remap_np_to_tex(remap_np, cam_fov, aspect);
+        //
+        // MAIN_DETACH([=, this] {
+        //     setupTextureHelper(aug_tex, aug_size);
+        //     aug_cam->setViewport(0, 0, aug_size.width, aug_size.height);
+        //     aug_cam->attach(Camera::COLOR_BUFFER0, aug_tex);
+        //     aug_img_osg->setImageTex(aug_tex);
+        //     aug_img_osg->setRemapImage(remap_tex);
+        //     cam_conf.fov = glm::degrees(cam_fov);
+        //     cam_conf.aspect = aspect;
+        //
+        //     setupTextureHelper(out_tex, out_size);
+        //     const auto out_cam = viewer->getCamera();
+        //     out_cam->setViewport(0, 0, out_size.width, out_size.height);
+        //     out_cam->attach(Camera::COLOR_BUFFER0, out_tex);
+        //
+        //     last_update_ts = info_ts;
+        // });
     }
 
     void image_callback_impl() {

src/image_process_v5/image_viewer.cpp

@@ -1,79 +1,45 @@
 #include "image_viewer.h"
 #include "image_process_v5/osg_helper.h"
 #include "core/imgui_utility.hpp"
+#include "render/render_tools.h"
 
-#include <osgViewer/Viewer>
 #include <utility>
 
 #define image_viewer image_viewer_v2
 
 struct image_viewer::impl {
     create_config conf;
+    color_image_render renderer;
 
-    osg::ref_ptr<osg::Geode> geode;
-    osg::ref_ptr<osgViewer::Viewer> viewer;
-
-    struct item_info_type : create_config::item_info {
-        osg::ref_ptr<ImageGeomSP> img_osg;
-        bool visible = true;
-    };
-
-    using item_list_type = std::vector<item_info_type>;
-    item_list_type items;
-
+    using item_info_type = create_config::item_info;
     item_info_type *current_item = nullptr;
 
     explicit impl(create_config _conf)
         : conf(std::move(_conf)) {
-        geode = new osg::Geode;
-        viewer = new osgViewer::Viewer();
-        viewer->setSceneData(geode);
-        viewer->setUpViewerAsEmbeddedInWindow(0, 0, 800, 600);
-        const auto camera = viewer->getCamera();
-        camera->setClearColor({0, 0, 0, 0});
-
-        std::ranges::transform(
-            conf.items, std::back_inserter(items),
-            [this](const create_config::item_info &item) {
-                auto ret = item_info_type();
-                *(create_config::item_info *) &ret = item;
-                ret.img_osg = new ImageGeomSP();
-                return ret;
-            });
-
         // display first item by default
-        assert(!items.empty());
-        geode->addDrawable(items[0].img_osg);
-        current_item = &items[0];
+        assert(!conf.items.empty());
+        current_item = &conf.items[0];
     }
 
-    void render() const {
-        const auto vp = query_viewport_size();
-        for (auto &item: items) {
-            if (!item.visible) continue;
-            try {
-                auto img = OBJ_QUERY(sp_image, item.name);
-                img.ensure_not_empty();
-                item.img_osg->setImageSP(img);
-                item.img_osg->setViewportRange(vp.aspectRatio(), item.flip);
-                item.img_osg->setNodeMask(-1); // enable
-            } catch (...) {
-                item.img_osg->setNodeMask(0); // disable
-            }
-        }
-        viewer->getCamera()->setViewport(0, 0, vp.width, vp.height);
-        viewer->frame();
+    void render() {
+        try {
+            const auto img = OBJ_QUERY(sp_image, current_item->name);
+            img.ensure_not_empty();
+            auto conf = color_image_render::config_type();
+            conf.flip_y = current_item->flip;
+            conf.fmt = COLOR_RGB;
+            conf.alpha = 1.f;
+            renderer.render(img, conf);
+        } catch (...) { (void) 0; }
     }
 
     void show_ui() {
-        for (auto &item: items) {
+        for (auto &item: conf.items) {
             if (ImGui::RadioButton(item.display_name.c_str(),
                                    &item == current_item)) {
-                geode->removeDrawable(current_item->img_osg);
-                geode->addDrawable(item.img_osg);
                 current_item = &item;
             }
-            if (&item != &*items.rbegin()) {
+            if (&item != &*conf.items.rbegin()) {
                 ImGui::SameLine();
             }
         }

src/image_process_v5/sp_image.cpp

@@ -74,6 +74,10 @@ int sp_image::cv_type() const {
     return type_map.query(type).cv_type;
 }
 
+int sp_image::channels() const {
+    return CV_MAT_CN(cv_type());
+}
+
 cv::Mat sp_image::cv_mat(void *ptr) const {
     return cv::Mat(cv_size(), cv_type(), start_ptr(ptr), pitch());
 }

src/image_process_v5/sp_image.h

@@ -18,6 +18,7 @@ struct sp_image : ndarray_proxy<image_rank>,
     const base_type *array_base() const { return this; }
     [[nodiscard]] cv::Size cv_size() const;
     [[nodiscard]] int cv_type() const;
+    [[nodiscard]] int channels() const;
     [[nodiscard]] cv::Mat cv_mat(void *ptr) const;
     [[nodiscard]] cv::cuda::GpuMat cv_gpu_mat(void *ptr) const;
     [[nodiscard]] sp_image sub_view(cv::Size size, cv::Size start = {}) const;

src/impl/apps/remote_ar/remote_ar_v2.cpp

@@ -48,15 +48,15 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf)
 
     if (true) {
         auto sub_conf = video_stabilization_ui::create_config{
-            .in_name = left_img_fix_id, .out_name = left_img_stab_id
+            .in_name = left_aug_id, .out_name = left_img_stab_id
         };
         video_stab.emplace(sub_conf);
     }
 
     if (true) {
         auto sub_conf = stereo_output_helper::create_config();
-        sub_conf.left_name = left_img_id;
-        sub_conf.right_name = right_img_id;
+        sub_conf.left_name = left_aug_id;
+        sub_conf.right_name = right_aug_id;
         sub_conf.out_name = output_img_id;
         // sub_conf.size = cv::Size(1920, 804);
         sub_conf.halve_width = false;
@@ -66,7 +66,7 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf)
     if (true) {
         auto sub_conf = image_viewer_v2::create_config();
         sub_conf.items.emplace_back(uvc_img_id, "Endoscope", true);
-        sub_conf.items.emplace_back(left_aug_id, "Left", true);
+        sub_conf.items.emplace_back(left_img_stab_id, "Left", true);
         sub_conf.items.emplace_back(right_aug_id, "Right", true);
         bg_viewer.emplace(sub_conf);
     }
@@ -80,8 +80,8 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf)
 
     if (true) {
         auto sub_conf = versatile_saver::create_config();
-        sub_conf.items.emplace_back(left_img_id, "Left", true);
-        sub_conf.items.emplace_back(right_img_id, "Right", true);
+        sub_conf.items.emplace_back(left_aug_id, "Left", true);
+        sub_conf.items.emplace_back(right_aug_id, "Right", true);
         sub_conf.items.emplace_back(uvc_img_id, "Endoscope", true);
         saver.emplace(sub_conf);
     }
@@ -89,6 +89,13 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf)
     start_sophiar(LOAD_STR("sophiar_config"),
                   LOAD_STR("sophiar_start_name"));
 
+    if (true) {
+        auto sub_conf = sophiar_monitor::create_config();
+        sub_conf.item_list = sophiar_monitor::item_list_from_yaml(LOAD_LIST("monitor"));
+        sub_conf.sophiar_conn = g_sophiar_conn;
+        monitor.emplace(sub_conf);
+    }
+
     auto reg_conf = registration_config{
         .conn = g_sophiar_conn,
         .probe_model_path = LOAD_STR("probe_model"),
@@ -99,45 +106,37 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf)
         reg->add_target(item_conf);
     }
 
-    aug_scene = SceneSP::createFromYAML(LOAD_LIST("scene_info"));
-
     if (true) {
-        auto sub_conf = image_augment::create_config();
-        sub_conf.in_name = left_img_id;
-        sub_conf.out_name = left_aug_id;
-        sub_conf.scene = aug_scene;
-        sub_conf.cam_info_name = left_cam_info_id;
-        sub_conf.cam_name = "Stereo Camera";
-        sub_conf.cam_transform_name = LOAD_STR("left_camera_transform");
-        left_aug.emplace(sub_conf);
+        auto aug_list_v1 = augment_manager::item_list_from_yaml(LOAD_LIST("augment_list"));
+        auto aug_list = augment_manager_v2::item_list_from_v1(aug_list_v1, extra_name);
+        auto aug_conf = augment_manager_v2::create_config();
+        aug_conf.sophiar_conn = g_sophiar_conn;
+        std::copy(aug_list.begin(), aug_list.end(), std::back_inserter(aug_conf.item_list));
+        aug_manager.emplace(aug_conf);
+        // aug_manager->pre_render_sig.connect(
+        //     [this](const auto &info) { acl->pre_render_slot(info); });
     }
 
     if (true) {
-        auto sub_conf = image_augment::create_config();
-        sub_conf.in_name = right_img_id;
+        auto sub_conf = image_augment_helper_v2::create_config();
+        sub_conf.img_flip_y = false;
+        sub_conf.img_name = left_img_fix_id;
+        sub_conf.out_name = left_aug_id;
+        sub_conf.manager = &aug_manager.value();
+        sub_conf.ctx = main_conf.asio_ctx;
+        sub_conf.sophiar_conn = g_sophiar_conn;
+        sub_conf.transform_var = LOAD_STR("left_camera_transform");
+        left_cam.aug_helper.emplace(sub_conf);
+        left_cam.aug_helper->set_camera_info(
+            camera_intrinsic::from_yaml(LOAD_SUB("left_camera")).to_v0());
+
+        sub_conf.img_name = right_img_id;
         sub_conf.out_name = right_aug_id;
-        sub_conf.scene = aug_scene;
-        sub_conf.cam_info_name = right_cam_info_id;
-        sub_conf.cam_transform_name = LOAD_STR("right_camera_transform");
-        right_aug.emplace(sub_conf);
-        right_aug->sync_with(&left_aug.value());
+        sub_conf.transform_var = LOAD_STR("right_camera_transform");
+        right_cam.aug_helper.emplace(sub_conf);
+        right_cam.aug_helper->set_camera_info(
+            camera_intrinsic::from_yaml(LOAD_SUB("right_camera")).to_v0());
     }
-
-    OBJ_SAVE(left_cam_info_id, camera_intrinsic::from_yaml(LOAD_SUB("left_camera")).to_v0());
-    OBJ_SAVE(right_cam_info_id, camera_intrinsic::from_yaml(LOAD_SUB("right_camera")).to_v0());
-
-    // if (true) {
-    //     auto sub_conf = ndi_stray_point_tracker::create_config();
-    //     sub_conf.ip_addr = LOAD_STR("ndi_ip");
-    //     sub_conf.port = LOAD_NUMBER(uint16_t, "ndi_port");
-    //     create_ndi_stray_points_tracker(sub_conf);
-    // }
-
-    // if (true) {
-    //     auto sub_conf = oblique_calibrator::create_config();
-    //     sub_conf.camera_ref_name = LOAD_STR("camera_ref_transform_var");
-    //     endo_calib.emplace(sub_conf);
-    // }
 }
 
 app_remote_ar_v2::~app_remote_ar_v2() = default;
@@ -177,18 +176,49 @@ void app_remote_ar_v2::show_ui() {
         //     endo_calib->show_ui();
         // }
 
-        if (ImGui::CollapsingHeader("AR Navigation")) {
+        if (ImGui::CollapsingHeader("Augmentation")) {
+            auto guard = imgui_id_guard("augmentation");
+
+            ImGui::SeparatorText("Actions");
+            // if (ImGui::Button("Start")) {
+            //     start_tracking();
+            // }
+            // ImGui::SameLine();
             ImGui::Checkbox("Registration", &enable_reg);
-            if (ImGui::TreeNode("Camera Control")) {
-                left_aug->show_ui();
-                ImGui::TreePop();
+
+            ImGui::SeparatorText("Status");
+            monitor->show();
+
+            ImGui::SeparatorText("Scene"); {
+                auto id_guard = imgui_id_guard("augment_scene");
+                aug_manager->show();
             }
-            if (ImGui::TreeNode("Scene Control")) {
-                aug_scene->showUI();
+
+            ImGui::SeparatorText("Camera");
+            // auto helper = aug_helper->get_camera_helper();
+            // if (helper != nullptr && ImGui::TreeNode("Endo Camera")) {
+            //     helper->show();
+            //     ImGui::TreePop();
+            // }
+            auto helper = left_cam.aug_helper->get_camera_helper();
+            if (helper != nullptr && ImGui::TreeNode("MVS Camera")) {
+                helper->show();
                 ImGui::TreePop();
             }
         }
 
+        if (ImGui::CollapsingHeader("AR Navigation")) {
+            ImGui::Checkbox("Registration", &enable_reg);
+            // if (ImGui::TreeNode("Camera Control")) {
+            //     left_aug->show_ui();
+            //     ImGui::TreePop();
+            // }
+            // if (ImGui::TreeNode("Scene Control")) {
+            //     aug_scene->showUI();
+            //     ImGui::TreePop();
+            // }
+        }
+
         if (ImGui::CollapsingHeader("Streamer")) {
             auto id_guard = imgui_id_guard("streamer");
             streamer->show();

src/impl/apps/remote_ar/remote_ar_v2.h

@@ -13,6 +13,10 @@
 #include "module_v3/registration.h"
 #include "module_v5/versatile_saver.h"
 #include "module_v5/oblique_calibrator.h"
+#include "module/augment_manager_v2.h"
+#include "module/image_augment_helper_v2.h"
+#include "module/camera_augment_helper_v2.h"
+#include "module/sophiar_monitor.h"
 
 class app_remote_ar_v2 final : public app_base {
 public:
@@ -42,6 +46,8 @@ private:
             left_cam_info_id = 9,
             right_cam_info_id = 10;
 
+    obj_name_type extra_name = 100;
+
     std::optional<mvs_camera_ui> mvs_cam;
     std::optional<uvc_camera_ui> uvc_cam;
     std::optional<image_enhance_ui> img_isp;
@@ -50,14 +56,20 @@ private:
     std::optional<image_viewer_v2> bg_viewer;
     std::optional<image_streamer> streamer;
     std::optional<versatile_saver> saver;
-    std::optional<oblique_calibrator> endo_calib;
+    // std::optional<oblique_calibrator> endo_calib;
+    std::optional<sophiar_monitor> monitor;
 
     bool enable_reg = false;
     std::unique_ptr<registration> reg;
 
-    osg::ref_ptr<SceneSP> aug_scene;
-    std::optional<image_augment> left_aug;
-    std::optional<image_augment> right_aug;
+    std::optional<augment_manager_v2> aug_manager;
+    struct {
+        std::optional<image_augment_helper_v2> aug_helper;
+    } left_cam, right_cam;
+
+    // osg::ref_ptr<SceneSP> aug_scene;
+    // std::optional<image_augment> left_aug;
+    // std::optional<image_augment> right_aug;
 };
 
 

src/impl/main_impl.cpp

@@ -84,7 +84,7 @@ void init_window() {
     assert(ret == GLFW_TRUE);
     glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
     glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 6);
-    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_COMPAT_PROFILE);
+    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
     // TODO: select width and height
     window = glfwCreateWindow(800, 600, "An not simple platform for visual navigation", nullptr, nullptr);
     assert(window != nullptr);
@@ -97,8 +97,8 @@ void init_window() {
     SPDLOG_INFO("Loaded OpenGL {}.{}", GLAD_VERSION_MAJOR(version), GLAD_VERSION_MINOR(version));
 
     // // enable color blending
-    // glEnable(GL_BLEND);
-    // glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+    glEnable(GL_BLEND);
+    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
 
 #ifndef NDEBUG
     // log opengl error

src/module/image_augment_helper_v2.h

@@ -12,11 +12,11 @@ public:
 
     struct create_config {
         obj_name_type img_name;
-        obj_name_type mask_name; // image_ptr, CV_8UC1, mask of augment area
+        obj_name_type mask_name = invalid_obj_name; // image_ptr, CV_8UC1, mask of augment area
         obj_name_type out_name; // image output, image_ptr, CV_8UC3
         bool img_flip_y = true;
         augment_manager_v2 *manager;
-        smart_cuda_stream *stream;
+        smart_cuda_stream *stream = nullptr;
 
         // for camera augment helper
         using io_context = boost::asio::io_context;

src/module/impl/augment_manager_v2.cpp

@@ -115,6 +115,7 @@ void augment_manager_v2::impl::update(const camera_info &info, bool no_commit) {
 }
 
 void augment_manager_v2::impl::render() {
+    if (!enable) return;
     render_scene(last_scene_info);
     q_this->post_render_sig(last_scene_info);
 }

src/module/impl/image_augment_helper_v2.cpp

@@ -1,8 +1,9 @@
 #include "image_augment_helper_v2_impl.h"
-#include "image_process/cuda_impl/pixel_convert.cuh"
-#include "image_process/process_funcs.h"
+#include "image_process_v5/image_process.h"
 #include "third_party/scope_guard.hpp"
 
+#include <opencv2/calib3d.hpp>
+
 image_augment_helper_v2::impl::impl(create_config _conf) {
     conf = _conf;
     fbo_conf = fbo_conf_type{
@@ -38,33 +39,71 @@ void image_augment_helper_v2::impl::render_augment() {
 void image_augment_helper_v2::impl::img_callback(obj_name_type name) {
     assert(name == conf.img_name);
 
-    auto img = to_image(name);
-    if (img == nullptr) return;
-    fbo_conf.size = img->size();
+    auto img = sp_image();
+    try {
+        img = OBJ_QUERY(sp_image, name);
+    } catch (...) { return; }
+
+    if (img.empty()) return;
+    fbo_conf.size = img.cv_size();
     fbo.create(fbo_conf);
     fbo.bind();
     auto closer = sg::make_scope_guard([this] { fbo.unbind(); });
 
-    auto img_conf = color_image_render::config_type{
-            .flip_y = conf.img_flip_y, .stream = conf.stream,
-    };
-    img_ren.render_v2(img, img_conf);
+    auto img_conf = color_image_render::config_type();
+    img_conf.fmt = COLOR_RGB;
+    img_conf.flip_y = conf.img_flip_y;
+    img_conf.stream = conf.stream;
+    img_ren.render(img, img_conf);
     render_augment();
 
     q_this->post_render_sig();
-    output_img = img_down->download_v2(COLOR_RGB);
+    output_img = img_down->download_sp(COLOR_RGB);
     OBJ_SAVE(conf.out_name, output_img);
 }
 
+namespace {
+    sp_image create_remap_image(const camera_intrinsic_v0 &info) {
+        const auto img_size = cv::Size(info.width, info.height);
+        const auto pix_num = img_size.area();
+        auto pix_mat = cv::Mat(1, pix_num, CV_32FC2);
+        for (auto iy = 0; iy < info.height; ++iy)
+            for (auto ix = 0; ix < info.width; ++ix) {
+                const auto id = iy * info.width + ix;
+                pix_mat.at<cv::Vec2f>(id) = cv::Vec2f(ix, iy);
+            }
+        auto ret = sp_image::create(CV_32FC2, img_size);
+        const auto write_helper = write_access_helper(ret.host());
+        auto ret_mat = cv::Mat(1, pix_num, CV_32FC2,
+                               ret.start_ptr(write_helper.ptr()));
+        cv::undistortPoints(pix_mat, ret_mat,
+                            info.intrinsic_cv_mat(), info.dist_cv_mat());
+        return ret;
+    }
+
+    cv::Rect2f calc_remap_range(const sp_image &remap) {
+        assert(CV_MAT_CN(remap.cv_type()) == 2);
+        const auto read_helper = read_access_helper(remap.host());
+        const auto cv_mat = remap.cv_mat(read_helper.ptr());
+        auto cv_xy = std::vector<cv::Mat>();
+        cv::split(cv_mat, cv_xy);
+        assert(cv_xy.size() == 2);
+        double x_min, x_max, y_min, y_max;
+        cv::minMaxLoc(cv_xy[0], &x_min, &x_max);
+        cv::minMaxLoc(cv_xy[1], &y_min, &y_max);
+        return cv::Rect2f(
+            x_min, y_min,
+            x_max - x_min, y_max - y_min
+        );
+    }
+
+    constexpr auto max_camera_y_axis_len = 3.f;
+    constexpr auto max_camera_x_axis_len = 6.f;
+}
+
 void image_augment_helper_v2::impl::set_camera_info(const camera_intrinsic_v0 &info) {
-    cv::Rect2f remap_bound;
-    image_ptr remap_img;
-    auto remap_conf = gen_undistort_remap_image::config_direct{
-            .cam_info = info, .mask = to_image(conf.mask_name),
-            .img = &remap_img, .bound = &remap_bound,
-            .stream = conf.stream,
-    };
-    gen_undistort_remap_image::call_direct(remap_conf);
+    auto remap_img = create_remap_image(info);
+    auto remap_bound = calc_remap_range(remap_img);
 
     auto x_min = remap_bound.x, x_max = x_min + remap_bound.width;
     auto y_min = remap_bound.y, y_max = y_min + remap_bound.height;
@@ -81,11 +120,8 @@ void image_augment_helper_v2::impl::set_camera_info(const camera_intrinsic_v0 &i
     fbo_aug_conf.size = aug_size;
 
     // convert to texture coordinate
-    auto remap_cuda = remap_img->cuda<float2>(conf.stream);
-    call_np_to_tex(remap_cuda, remap_cuda,
-                   cam_fov, aspect, conf.stream->cuda);
-    remap_img->cuda_modified(conf.stream);
-    remap_tex.upload(remap_img, conf.stream);
+    remap_img = image_remap_np_to_tex(remap_img, cam_fov, aspect);
+    remap_tex.upload(remap_img);
 
     auto cam_conf = camera_augment_helper_v2::create_config{
             .camera = camera_augment_helper_v2::create_config::fixed_camera_config{
@@ -98,11 +134,11 @@ void image_augment_helper_v2::impl::set_camera_info(const camera_intrinsic_v0 &i
 }
 
 void image_augment_helper_v2::impl::render() {
-    if (output_img == nullptr) return;
+    if (output_img.empty()) return;
     auto img_conf = color_image_render::config_type{
             .flip_y = false, .stream = conf.stream,
     };
-    out_ren.render_v2(output_img, img_conf);
+    out_ren.render(output_img, img_conf);
 }
 
 camera_augment_helper_v2 *image_augment_helper_v2::get_camera_helper() {

src/module/impl/image_augment_helper_v2_impl.h

@@ -30,7 +30,7 @@ struct image_augment_helper_v2::impl {
     color_image_render img_ren;
     std::unique_ptr<viewport_downloader> img_down;
 
-    image_ptr output_img;
+    sp_image output_img;
     color_image_render out_ren;
 
     explicit impl(create_config conf);

src/module/viewport_downloader.hpp

@@ -55,6 +55,26 @@ public:
         }
     }
 
+    sp_image download_sp(color_format fmt = COLOR_RGB) {
+        switch (fmt) {
+            case COLOR_RGB: {
+                return pbo.download_viewport_sp(GL_RGB, GL_UNSIGNED_BYTE);
+            }
+            case COLOR_RGBA: {
+                return pbo.download_viewport_sp(GL_RGBA, GL_UNSIGNED_BYTE);
+            }
+            case COLOR_ARGB: {
+                return pbo.download_viewport_sp(GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV);
+            }
+            case COLOR_DEPTH_32F: {
+                return pbo.download_viewport_sp(GL_DEPTH_COMPONENT, GL_FLOAT);
+            }
+            default: {
+                RET_ERROR_E;
+            }
+        }
+    }
+
 private:
     smart_cuda_stream *stream;
     smart_pixel_buffer pbo;

src/render/impl/render_tools.cpp

@@ -153,6 +153,11 @@ void color_image_render::render_v2(const image_ptr &img, config_type conf) {
     }
 }
 
+void color_image_render::render(const sp_image& img, const config_type &conf) {
+    img_tex.upload(img);
+    render_tex(img.cv_size(), conf);
+}
+
 void depth_image_render::render(obj_name_type name, config_type conf) {
     auto img_type = OBJ_TYPE(name);
     assert(img_type == typeid(image_f32c1));

src/render/impl/render_utility.cpp

@@ -9,7 +9,7 @@
 using boost::iostreams::mapped_file;
 
 std::filesystem::path shader_folder
-        = "/home/tpx/project/DepthGuide/src/render/impl/shader"; // TODO: config shader path
+        = "/home/tpx/ext/project/DepthGuide/src/render/impl/shader"; // TODO: config shader path
 
 cv::Size query_viewport_size() {
     struct {
@@ -164,6 +164,26 @@ void smart_texture::create(GLenum _format, cv::Size _size) {
     set_filter(GL_LINEAR, GL_LINEAR);
 }
 
+void smart_texture::upload(const sp_image &img) {
+    create(get_tex_internal_format(img.cv_type()), img.cv_size());
+    if (img.channels() == 3) {
+        img_pbo.upload(img);
+        upload(img_pbo.id, size, img.cv_type());
+    } else {
+        const auto status = img.mem->status();
+        if (status.cuda_available) {
+            const auto read_helper = read_access_helper(img.cuda());
+            const auto mem_v1 = to_mem_v1(img, read_helper.ptr(), MEM_CUDA);
+            upload_impl(mem_v1, nullptr);
+        } else {
+            assert(status.host_available);
+            const auto read_helper = read_access_helper(img.host());
+            const auto mem_v1 = to_mem_v1(img, read_helper.ptr(), MEM_HOST);
+            upload_impl(mem_v1, nullptr);
+        }
+    }
+}
+
 void smart_texture::upload_impl(const image_mem_info &img, smart_cuda_stream *stream) {
     if (cuda_res == nullptr) {
         CUDA_API_CHECK(cudaGraphicsGLRegisterImage(
@@ -172,14 +192,16 @@ void smart_texture::upload_impl(const image_mem_info &img, smart_cuda_stream *st
     assert(cuda_res != nullptr);
 
     cudaArray_t tex_arr;
-    CUDA_API_CHECK(cudaGraphicsMapResources(1, &cuda_res, stream->cuda));
+    CUDA_API_CHECK(cudaGraphicsMapResources(1, &cuda_res, GET_CUDA_STREAM(stream)));
     CUDA_API_CHECK(cudaGraphicsSubResourceGetMappedArray(&tex_arr, cuda_res, 0, 0));
     CUDA_API_CHECK(cudaMemcpy2DToArrayAsync(
             tex_arr, 0, 0, img.ptr.get(), img.pitch, img.width, img.height,
             img.loc == MEM_CUDA ? cudaMemcpyDeviceToDevice : cudaMemcpyHostToDevice,
-            stream->cuda));
-    CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &cuda_res, stream->cuda));
-    extend_pointer_life(img.ptr, stream);
+            GET_CUDA_STREAM(stream)));
+    CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &cuda_res, GET_CUDA_STREAM(stream)));
+    if (stream != nullptr) {
+        extend_pointer_life(img.ptr, stream);
+    }
 }
 
 void smart_texture::upload_impl(GLuint pbo_id, GLenum _format, GLenum type) {
@@ -281,7 +303,7 @@ void *smart_pixel_buffer::up_mapped_ptr(smart_cuda_stream *stream, size_t *_ptr_
     assert(cuda_res_up != nullptr);
     void *ptr = nullptr;
     size_t ptr_size = 0;
-    CUDA_API_CHECK(cudaGraphicsMapResources(1, &cuda_res_up, stream->cuda));
+    CUDA_API_CHECK(cudaGraphicsMapResources(1, &cuda_res_up, GET_CUDA_STREAM(stream)));
     CUDA_API_CHECK(cudaGraphicsResourceGetMappedPointer(&ptr, &ptr_size, cuda_res_up));
     if (_ptr_size != nullptr) {
         *_ptr_size = ptr_size;
@@ -290,7 +312,28 @@ void *smart_pixel_buffer::up_mapped_ptr(smart_cuda_stream *stream, size_t *_ptr_
 }
 
 void smart_pixel_buffer::up_unmap(smart_cuda_stream *stream) {
-    CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &cuda_res_up, stream->cuda));
+    CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &cuda_res_up, GET_CUDA_STREAM(stream)));
+}
+
+void smart_pixel_buffer::upload(const sp_image &img) {
+    create(img.byte_size());
+    size_t ptr_size = 0;
+    const auto ptr = up_mapped_ptr(nullptr, &ptr_size);
+    assert(ptr_size >= img.byte_size());
+    const auto status = img.mem->status();
+    if (status.cuda_available) {
+        const auto read_helper = read_access_helper(img.cuda());
+        CUDA_API_CHECK(cudaMemcpy2DAsync(
+            ptr, img.byte_width(), img.start_ptr(read_helper.ptr()), img.pitch(),
+            img.byte_width(), img.height(), cudaMemcpyDeviceToDevice, current_cuda_stream()));
+    } else {
+        assert(status.host_available);
+        const auto read_helper = read_access_helper(img.host());
+        CUDA_API_CHECK(cudaMemcpy2DAsync(
+            ptr, img.byte_width(), img.start_ptr(read_helper.ptr()), img.pitch(),
+            img.byte_width(), img.height(), cudaMemcpyHostToDevice, current_cuda_stream()));
+    }
+    up_unmap(nullptr);
 }
 
 void smart_pixel_buffer::upload_impl(const image_mem_info &img, smart_cuda_stream *stream) {
@@ -341,6 +384,15 @@ image_ptr smart_pixel_buffer::download_viewport(GLenum format, GLenum type,
     return img;
 }
 
+sp_image smart_pixel_buffer::download_viewport_sp(GLenum format, GLenum type) {
+    const auto img_type = CV_MAKETYPE(get_type_depth(type), get_format_channels(format));
+    auto img = sp_image::create(img_type, query_viewport_size());
+    download_viewport(format, type);
+    const auto write_helper = write_access_helper(img.cuda());
+    download_impl(to_mem_v1(img, write_helper.ptr(), MEM_CUDA), nullptr);
+    return img;
+}
+
 void smart_pixel_buffer::download_impl(const image_mem_info &img, smart_cuda_stream *stream) {
     if (cuda_res_down == nullptr) {
         CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(
@@ -350,14 +402,14 @@ void smart_pixel_buffer::download_impl(const image_mem_info &img, smart_cuda_str
 
     void *ptr = nullptr;
     size_t ptr_size = 0;
-    CUDA_API_CHECK(cudaGraphicsMapResources(1, &cuda_res_down, stream->cuda));
+    CUDA_API_CHECK(cudaGraphicsMapResources(1, &cuda_res_down, GET_CUDA_STREAM(stream)));
     CUDA_API_CHECK(cudaGraphicsResourceGetMappedPointer(&ptr, &ptr_size, cuda_res_down));
     assert(ptr_size >= img.width * img.height);
     CUDA_API_CHECK(cudaMemcpy2DAsync(
             img.ptr.get(), img.pitch, ptr, img.width, img.width, img.height,
             img.loc == MEM_CUDA ? cudaMemcpyDeviceToDevice : cudaMemcpyDeviceToHost,
-            stream->cuda));
-    CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &cuda_res_down, stream->cuda));
+            GET_CUDA_STREAM(stream)));
+    CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &cuda_res_down, GET_CUDA_STREAM(stream)));
 }
 
 smart_frame_buffer::~smart_frame_buffer() {

src/render/render_tools.h

@@ -4,6 +4,7 @@
 #include "core/object_manager.h"
 #include "render_utility.h"
 #include "render_texture.h"
+#include "image_process_v5/sp_image.h"
 
 #include <boost/container/small_vector.hpp>
 
@@ -46,6 +47,8 @@ public:
 
     void render_v2(const image_ptr &img, config_type conf);
 
+    void render(const sp_image& img, const config_type &conf);
+
 private:
     smart_texture img_tex;
     smart_texture ext_tex[2];

src/render/render_utility.h

@@ -5,6 +5,7 @@
 #include "core/image_utility.hpp"
 #include "core/image_utility_v2.h"
 #include "core/pc_utility.h"
+#include "image_process_v5/sp_image.h"
 
 #include <glad/gl.h>
 #include <glm/glm.hpp>
@@ -170,6 +171,8 @@ public:
         upload_impl(img->memory_v1(stream), stream);
     }
 
+    void upload(const sp_image &img);
+
     template<typename T>
     void upload(const image_info_type<T> &img, smart_cuda_stream *stream) {
         create(sizeof(T) * img.size.area()); // TODO: create inside upload_impl
@@ -196,6 +199,8 @@ public:
     image_ptr download_viewport(GLenum format, GLenum type,
                                 smart_cuda_stream *stream);
 
+    sp_image download_viewport_sp(GLenum format, GLenum type);
+
 private:
 
     size_t capacity = 0;
@@ -271,6 +276,8 @@ public:
         }
     }
 
+    void upload(const sp_image &img);
+
 private:
 
     void deallocate();