Removed OSG.

CMakeLists.txt

@@ -76,7 +76,7 @@ add_executable(${PROJECT_NAME} src/main.cpp
 add_subdirectory(src/core_v2)
 add_subdirectory(src/device_v5)
 add_subdirectory(src/image_process_v5)
-add_subdirectory(src/render_osg)
+#add_subdirectory(src/render_osg)
 add_subdirectory(src/module_v5)
 
 target_include_directories(${PROJECT_NAME} PRIVATE src)
@@ -150,7 +150,7 @@ target_compile_definitions(${PROJECT_NAME} PRIVATE GLM_ENABLE_EXPERIMENTAL)
 if (WIN32)
     set(IMGUI_DIR C:/BuildEssentials/Library/imgui-1.89.5)
 else ()
-    set(IMGUI_DIR /home/tpx/ext/src/imgui-1.91.0)
+    set(IMGUI_DIR /home/tpx/ext/src/imgui-1.91.8)
 endif ()
 set(IMGUI_BACKENDS_DIR ${IMGUI_DIR}/backends)
 target_include_directories(${PROJECT_NAME} PRIVATE ${IMGUI_DIR} ${IMGUI_BACKENDS_DIR})
@@ -275,7 +275,7 @@ if (WIN32)
     set(NVENC_LIB_DIR ${NVCODEC_DIR}/Lib/x64)
     find_library(NVENC_LIB nvencodeapi HINTS ${NVENC_LIB_DIR})
 else ()
-    set(NVCODEC_DIR /home/tpx/ext/src/Video_Codec_SDK_12.2.72)
+    set(NVCODEC_DIR /home/tpx/ext/src/Video_Codec_SDK_13.0.19)
     find_library(NVENC_LIB nvidia-encode)
 endif ()
 set(NVCODEC_INCLUDE_DIR ${NVCODEC_DIR}/Interface)
@@ -288,7 +288,7 @@ if (WIN32)
     set(NVCODEC_DIR C:/BuildEssentials/CUDA/Video_Codec_SDK_12.0.16)
     find_library(NVDEC_LIB nvcuvid HINTS ${NVCODEC_DIR}/Lib/x64)
 else ()
-    set(NVCODEC_DIR /home/tpx/src/Video_Codec_SDK_12.0.16)
+    set(NVCODEC_DIR /home/tpx/ext/src/Video_Codec_SDK_13.0.19)
     find_library(NVDEC_LIB nvcuvid)
 endif ()
 set(NVCODEC_INCLUDE_DIR ${NVCODEC_DIR}/Interface)

data/config_remote_ar_v2.yaml

@@ -20,24 +20,26 @@ monitor:
     var: tibia_ref_in_tracker
   - name: Probe
     var: probe_ref_in_tracker
+  - name: Drill
+    var: drill_ref_in_tracker
 
 left_camera:
-  fx: 3569.14196423103
-  fy: 3568.02459571424
-  cx: 1229.37096752839
-  cy: 1026.81619824569
-  k0: -0.0681166229505913
-  k1: 0.0751335815980414
+  fx: 3575.08023770980
+  fy: 3574.29097339419
+  cx: 1229.19314281847
+  cy: 1026.36690138943
+  k0: -0.0650342272207188
+  k1: 0.0627253420197537
   width: 2448
   height: 2048
 
 right_camera:
-  fx: 3581.91081422450
-  fy: 3580.97918934816
-  cx: 1218.86459178897
-  cy: 1034.61633754506
-  k0: -0.0753067946100483
-  k1: 0.106748846894580
+  fx: 3586.22005491040
+  fy: 3585.28366099848
+  cx: 1219.17483543191
+  cy: 1033.15166609626
+  k0: -0.0796614145323204
+  k1: 0.140429147463810
   width: 2448
   height: 2048
 
@@ -60,8 +62,8 @@ registration_list:
     target_var: femur_in_femur_ref
     probe_var: probe_in_femur
 
-oblique_rotation_center: 3W1sRPPKA0Q=
-oblique_rotation_ball: FOR0QeXogcMZz//B/nbywZ+afj/zk9W9+IGCOowc0j34RHw/Mu4KPg==
+oblique_rotation_center: CAZsRG6RA0Q=
+oblique_rotation_ball: 3uB4QUxQgcPIHf3BFhz4wVwpfz/IG6O9C6xlPKOwoj1WD38/adgCPQ==
 #oblique_rotation_ball: hmhzQTb9gcPFqADCRCLoweuvfj8v9M693iKOu791zT0Nj34/srELvQ==
 
 acl_guide:

data/sophiar_config_endo_guide.json

@@ -242,26 +242,26 @@
             "name": "left_camera",
             "parent": "mvs_ref",
             "transform": [
-              1.69529294997093,
-              -40.8504461910221,
-              -38.4356249156025,
-              0.328217190586743,
-              0.623382258094322,
-              0.318037852337519,
-              -0.634444607965983
+              43.0024873290043,
+              -36.6555005431208,
+              -39.1675882200710,
+              0.331666968316702,
+              0.624070730348051,
+              0.322514541601977,
+              -0.629696050571802
             ]
           },
           {
             "name": "right_camera",
             "parent": "mvs_ref",
             "transform": [
-              0.0941597771490770,
-              -41.0431819166041,
-              -105.682270133671,
-              0.329203554783163,
-              0.626106195513828,
-              0.316605284270741,
-              -0.631962930423275
+              43.4854235778036,
+              -37.6467021757064,
+              -104.326293757405,
+              0.330462419537311,
+              0.628820745114726,
+              0.321594183888926,
+              -0.626064086716146
             ]
           },
           {
@@ -273,13 +273,13 @@
             "name": "drill",
             "parent": "drill_ref",
             "transform": [
-              -134.93834363508242,
-              -38.64937251450866,
-              76.05756748057985,
-              0.5651659820674402,
-              0.4249557773624757,
-              -0.4256067427602504,
-              -0.5646759252146404
+              -88.71110327911822,
+              -44.459110684986626,
+              -51.32877691381532,
+              -0.606536084660151,
+              0.3634748657130698,
+              -0.3346859426031122,
+              0.6228846761832131
             ]
           }
         ]
@@ -369,40 +369,40 @@
         "com_port": "/dev/ttyUSB0",
         "tool_list": [
           {
-            "rom_path": "/home/tpx/data/roms/Probe_Small_4Ball.rom",
+            "rom_path": "/home/tpx/data/roms/20241207Exp/Probe_Small_4BallFix.rom",
             "serial_number": "3DD50000",
             "outputs": {
               "transform": "probe_ref_in_tracker"
             }
           },
           {
-            "rom_path": "/home/tpx/data/roms/Glass_4Ball_2.rom",
+            "rom_path": "/home/tpx/data/roms/20241207Exp/Glass_4Ball_2Fix.rom",
             "outputs": {
               "transform": "camera_ref_in_tracker",
               "marker_uncertainty": "camera_ref_in_tracker_error"
             }
           },
           {
-            "rom_path": "/home/tpx/data/roms/NR_1_STCAM_20241115.rom",
+            "rom_path": "/home/tpx/data/roms/20241207Exp/Glass_4Ball_1Fix.rom",
             "outputs": {
               "transform": "mvs_ref_in_tracker"
             }
           },
           {
-            "rom_path": "/home/tpx/data/roms/Glass_3Ball_7_Drill.rom",
+            "rom_path": "/home/tpx/data/roms/20241207Exp/Glass_3Ball_7Fix.rom",
             "outputs": {
               "transform": "drill_ref_in_tracker"
             }
           },
           {
-            "rom_path": "/home/tpx/data/roms/Glass_3Ball_5.rom",
+            "rom_path": "/home/tpx/data/roms/20241207Exp/Glass_3Ball_6Fix.rom",
             "serial_number": "39B33001",
             "outputs": {
               "transform": "femur_ref_in_tracker"
             }
           },
           {
-            "rom_path": "/home/tpx/data/roms/Glass_3Ball_6.rom",
+            "rom_path": "/home/tpx/data/roms/20241207Exp/Glass_3Ball_5Fix.rom",
             "outputs": {
               "transform": "tibia_ref_in_tracker"
             }
@@ -485,7 +485,7 @@
         "stable_type": "point",
         "input_var_name": "probe_tip_in_femur_ref",
         "output_var_name": "picked_point_in_femur_ref",
-        "linear_tolerance_mm": 0.05,
+        "linear_tolerance_mm": 0.15,
         "temporal_interval_s": 3
       },
       "dependencies": [
@@ -499,7 +499,7 @@
         "stable_type": "point",
         "input_var_name": "probe_tip_in_tibia_ref",
         "output_var_name": "picked_point_in_tibia_ref",
-        "linear_tolerance_mm": 0.05,
+        "linear_tolerance_mm": 0.15,
         "temporal_interval_s": 3
       },
       "dependencies": [
@@ -574,6 +574,14 @@
         "variable_out": "tibia_in_tracker_denoised"
       }
     },
+    {
+      "type": "transform_obj_recorder",
+      "name": "camera_ref_recorder",
+      "start_config": {
+        "variable_name": "camera_ref_in_tracker",
+        "save_file": "camera_ref.txt"
+      }
+    },
     {
       "type": "empty_object",
       "name": "tracker_all",
@@ -588,7 +596,8 @@
         "tibia_denoiser",
         "camera_denoiser",
         "left_camera_denoiser",
-        "right_camera_denoiser"
+        "right_camera_denoiser",
+        "camera_ref_recorder"
       ]
     }
   ]

src/device/impl/uvc_camera.cpp

@@ -76,10 +76,11 @@ namespace uvc_camera_impl {
             assert(dev_desc != nullptr);
             std::string dev_name;
             if (dev_desc->serialNumber != nullptr) {
-                dev_name = fmt::format("{}, {} ({})",
-                                       safe_str(dev_desc->manufacturer),
-                                       safe_str(dev_desc->product),
-                                       safe_str(dev_desc->serialNumber));
+                dev_name = fmt::format("{}: {}, {} ({})",
+                    p_dev - dev_list,
+                    safe_str(dev_desc->manufacturer),
+                    safe_str(dev_desc->product),
+                    safe_str(dev_desc->serialNumber));
             } else {
                 dev_name = fmt::format("{:x}:{:x}",
                                        dev_desc->idVendor, dev_desc->idProduct);

src/image_process/impl/camera_calibrator.cpp

@@ -429,6 +429,7 @@ namespace camera_calibrator_impl {
                                        cam_cv.intrinsic, cam_cv.dist_coeffs,
                                        chess_r, chess_t);
                 assert(ok);
+                break;
 
                 // rotate corners if needed
                 auto rot_mat = rodrigues_to_mat(chess_r);

src/image_process_v5/CMakeLists.txt

@@ -1,9 +1,9 @@
 target_sources(${PROJECT_NAME} PRIVATE
-        image_augment.cpp
+#        image_augment.cpp
         image_enhance.cpp
         image_viewer.cpp
         image_process.cpp
-        osg_helper.cpp
+#        osg_helper.cpp
         sp_image.cpp
         video_stabilization.cpp
         process_python/fast_sam.cpp

src/image_process_v5/image_viewer.cpp

@@ -1,5 +1,4 @@
 #include "image_viewer.h"
-#include "image_process_v5/osg_helper.h"
 #include "core/imgui_utility.hpp"
 #include "render/render_tools.h"
 

src/impl/apps/debug/app_debug.cpp

@@ -1,49 +1,5 @@
 #include "app_debug.h"
-#include "render/render_utility.h"
-#include "render_osg/osg_scene.h"
 
-#include <osgViewer/Viewer>
-#include <osgUtil/UpdateVisitor>
+app_debug::app_debug(const create_config &conf) { }
 
-osg::ref_ptr<SceneSP> root;
-osg::ref_ptr<osgViewer::Viewer> viewer;
-
-app_debug::app_debug(const create_config &conf) {
-    auto model_conf = MeshSP::create_config();
-    model_conf.name = "Probe";
-    model_conf.model_path =
-            "/media/tpx/590d7229-f129-4612-b7ff-6d2cb3fe49b2/home/tpx/project/RemoteAR3/data/models/femur.stl";
-    osg::ref_ptr model = new MeshSP(model_conf);
-
-    root = new SceneSP();
-    root->addChild(model);
-
-    viewer = new osgViewer::Viewer();
-    viewer->setSceneData(root);
-    viewer->setUpViewerAsEmbeddedInWindow(0, 0, 800, 600);
-
-    auto bs = root->getBound();
-    auto center = bs.center();
-    auto radius = bs.radius();
-
-    auto eye = center + osg::Vec3(0.0f, 0.0f, radius * 2.0f); // position camera along z-axis
-    auto up = osg::Vec3(0.0f, 1.0f, 0.0f); // up vector
-    auto lookAt = center;
-
-    osg::Matrixd viewMatrix;
-    viewMatrix.makeLookAt(eye, lookAt, up);
-    auto camera = viewer->getCamera();
-    camera->setViewMatrix(viewMatrix);
-    camera->setClearColor({1, 1, 1, 1});
-    root->setCamera(camera);
-}
-
-void app_debug::render_background() {
-    const auto vp = query_viewport_size();
-    auto camera = viewer->getCamera();
-    camera->setViewport(0, 0, vp.width, vp.height);
-    camera->setProjectionMatrixAsPerspective(60.0f, 1. * vp.width / vp.height, 0.1f, 10000.0f);
-    osg::ref_ptr visitor = new osgUtil::UpdateVisitor();
-    visitor->apply(*root);
-    viewer->frame();
-}
+void app_debug::render_background() { }

src/impl/apps/remote_ar/remote_ar_v2.cpp

@@ -5,7 +5,6 @@
 #include "device_v5/ndi_stray_point_tracker.h"
 #include "image_process/camera_utility.hpp"
 #include "image_process_v5/image_process.h"
-#include "image_process_v5/osg_helper.h"
 #include "image_process_v5/sp_image.h"
 
 // from sophiar
@@ -127,7 +126,7 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf) : main_conf(std::move(_c
         sub_conf.len_ref_name               = "len_ref_in_tracker";
         sub_conf.board_type                 = camera_calibrator::CALIB_CHESS;
         sub_conf.pattern_size               = cv::Size(11, 8);
-        sub_conf.corner_distance            = 1.5; // mm
+        sub_conf.corner_distance            = 5; // mm
         sub_conf.camera_ref_error_name      = "camera_ref_in_tracker_error";
         sub_conf.camera_to_len_ref_name     = "camera_in_len_ref";
         sub_conf.cb_func                    = [this](const auto &ret) {
@@ -142,7 +141,7 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf) : main_conf(std::move(_c
         endo_calib->set_rotation_center(LOAD_STR("oblique_rotation_center"));
         endo_calib->set_rotation_ball(LOAD_STR("oblique_rotation_ball"));
         auto cam_sim      = camera_calibrator::simulate_info_type();
-        cam_sim.data_path = "calib_data_v2.txt";
+        cam_sim.data_path = "calib_data_20250412.txt";
         cam_sim.img_size  = cv::Size(1920, 1080);
         endo_calib->set_camera_calib(cam_sim);
     }
@@ -152,7 +151,9 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf) : main_conf(std::move(_c
         sub_conf.items.emplace_back(uvc_img_id, "Endo", true);
         sub_conf.items.emplace_back(uvc_inv_rot_id, "EndoInvRot", true);
         sub_conf.items.emplace_back(uvc_out_id, "EndoAug", true);
-        sub_conf.items.emplace_back(left_aug_id, "Left", true);
+        sub_conf.items.emplace_back(left_img_id, "LeftImg", true);
+        sub_conf.items.emplace_back(left_fix_id, "LeftFix", true);
+        sub_conf.items.emplace_back(left_aug_id, "LeftAug", true);
         sub_conf.items.emplace_back(right_aug_id, "Right", true);
         bg_viewer.emplace(sub_conf);
     }

src/impl/apps/remote_ar/remote_ar_v2.h

@@ -3,11 +3,9 @@
 
 #include "device/mvs_camera_ui.h"
 #include "device/uvc_camera_ui.h"
-#include "image_process_v5/image_augment.h"
 #include "image_process_v5/image_enhance.h"
 #include "image_process_v5/image_process.h"
 #include "image_process_v5/image_viewer.h"
-#include "image_process_v5/video_stabilization.h"
 #include "impl/app_base.h"
 #include "module/augment_manager_v2.h"
 #include "module/camera_augment_helper_v2.h"

src/module/impl/image_streamer_impl.h

@@ -46,7 +46,7 @@ struct image_streamer::impl {
     std::unique_ptr<io_context> aux_ctx;
 
     // for UDP and UDP (FEC)
-    uint16_t sender_mtu = 1450;
+    uint16_t sender_mtu = 1250;
 
     // for UDP (FEC)
     float sender_parity_rate = 0.2f;

src/module_v5/CMakeLists.txt

@@ -2,6 +2,7 @@ target_sources(${PROJECT_NAME} PRIVATE
         versatile_saver.cpp
         transform_provider.cpp
         oblique_calibrator.cpp
+        oblique_eval.cpp
         zmq_client.cpp)
 
 add_subdirectory(algorithms)

src/module_v5/algorithms/fitting_circle_3d.cpp

@@ -72,6 +72,7 @@ fitting_circle_3d::operator()(const input_type &input) {
 
     MatrixX3f P = input.points.transpose();
     const Vector3f mean = P.colwise().mean();
+
     P.array().rowwise() -= mean.array().transpose();
 
     // Calculate the SVD of the centered points

src/module_v5/oblique_calibrator.cpp

@@ -343,6 +343,7 @@ namespace {
             assert(name == conf.image_name);
             try {
                 const auto img = OBJ_QUERY(sp_image, conf.image_name);
+                if ( img.empty() ) return;
                 if ( const auto angle = calc_rotation_angle(img); angle ) {
                     img_angle = *angle;
                 } else {

src/module_v5/oblique_calibrator.h

@@ -39,7 +39,7 @@ public:
     void render() const; // forward to camera calib
     void set_rotation_center(const std::string &str) const;
     void set_rotation_ball(const std::string &str) const;
-    void set_camera_calib(const camera_calibrator::simulate_info_type& sim_info) const;
+    void set_camera_calib(const camera_calibrator::simulate_info_type &sim_info) const;
 
 private:
     struct impl;

src/module_v5/versatile_saver.cpp

@@ -1,11 +1,12 @@
 #include "versatile_saver.h"
+
 #include "core/imgui_utility.hpp"
-#include "image_process_v5/sp_image.h"
 #include "image_process_v5/image_process.h"
-
-#include <ImGuiFileDialog.h>
+#include "image_process_v5/sp_image.h"
 
 #include <boost/asio/post.hpp>
+#include <core/timestamp_helper.hpp>
+#include <ImGuiFileDialog.h>
 
 using obj_type = object_manager_v2::obj_query_config;
 using boost::asio::post;
@@ -103,7 +104,7 @@ struct versatile_saver::impl {
     }
 
     std::string get_save_filename(item_info_type &item) {
-        const auto filename = fmt::format("{}_{}", item.save_name, save_cnt);
+        const auto filename = fmt::format("{}_{:.1f}", item.save_name, sophiar::current_timestamp()/1000.f);
         const auto save_folder = std::filesystem::path(filepath_buf);
         if (!exists(save_folder)) {
             create_directories(save_folder);

src/network_v3/receiver_udp_fec.cpp

@@ -74,7 +74,7 @@ struct receiver_udp_fec::impl {
     static impl *create(const create_config &conf) {
         const auto ret = new impl();
         ret->par_conf = {.cb_func = conf.cb_func, .enable_log = conf.enable_log};
-        ret->server_ep = udp::endpoint{address::from_string(conf.server_addr), conf.server_port};
+        ret->server_ep = udp::endpoint{make_address(conf.server_addr), conf.server_port};
         ret->socket = std::make_unique<udp::socket>(*conf.ctx);
         ret->socket->connect(ret->server_ep);
         ret->socket->set_option(udp::socket::receive_buffer_size{udp_buffer_size});

src/network_v3/sender_udp_fec.cpp

@@ -165,7 +165,7 @@ struct sender_udp_fec::impl {
         auto crc = boost::crc_32_type{};
         crc.process_bytes(out_buf.ptr + sizeof(uint32_t),
                           frag_header_size + block_size - sizeof(uint32_t));
-        write_binary_number(out_buf.ptr, crc.checksum());
+        write_binary_number(out_buf.ptr, (uint32_t) crc.checksum());
 
         // send packet
         assert(socket != nullptr);
@@ -216,7 +216,7 @@ struct sender_udp_fec::impl {
         static constexpr auto fake_stun_server = "38.59.254.192"; // TODO: setting in config file
         static constexpr auto fake_stun_port = 5281;
         auto fake_stun_ep = udp::endpoint{
-                address::from_string(fake_stun_server), fake_stun_port};
+                make_address(fake_stun_server), fake_stun_port};
 
         out_buf.reserve(2 * sizeof(uint8_t));
         auto ptr = write_binary_number(out_buf.ptr, 'R');

src/render_v3/vtk_viewer.cpp

@@ -209,7 +209,7 @@ struct vtk_viewer::impl {
         auto render_size_cv = to_cv_size(render_size);
         if (render_size_cv.area() <= 0)return;
         render(render_size_cv, true);
-        ImGui::Image(reinterpret_cast<void *>(get_tex()), render_size, {1, 0}, {0, 1});
+        ImGui::Image((ImTextureID) get_tex(), render_size, {1, 0}, {0, 1});
         ImGui::EndChild();
     }