Output image size can be dynamically changed.

data/config.yaml

@@ -4,21 +4,21 @@ camera:
     right: RightEye
   intrinsic:
     left:
-      fx: 3571.52848771264
-      fy: 3570.57318451437
-      cx: 1233.52703624453
-      cy: 1025.15117581285
-      k0: -0.0579298417960705
-      k1: 0.0537095025162189
+      fx: 3591.43951726492
+      fy: 3590.12059005116
+      cx: 1228.96467407067
+      cy: 1026.27011971545
+      k0: -0.0568617986513312
+      k1: 0.0588730412975145
       width: 2448
       height: 2048
     right:
-      fx: 3573.03298854121
-      fy: 3572.30496881714
-      cx: 1223.80603997798
-      cy: 1035.75801617604
-      k0: -0.0736243464203426
-      k1: 0.126764214115614
+      fx: 3600.88065108231
+      fy: 3599.67273902144
+      cx: 1229.67535197217
+      cy: 1038.94918883869
+      k0: -0.0763206665373899
+      k1: 0.169733866632155
       width: 2448
       height: 2048
   capture:
@@ -31,7 +31,7 @@ main_window:
   height: 600
 
 output:
-  width: 1920
+  width: 5000
   height: 1080
   hevc_bitrate: 5 # Mbps
   jpeg_quality: 15
@@ -49,7 +49,7 @@ sophiar:
   augment:
     - name: Femur
       trans_var: femur_in_tracker_denoised
-      stl_file: ./models/femur.stl
+      stl_file: ./models/Femur_2.stl
       registration:
         collect_obj: point_picker_in_femur_ref
         collect_var: picked_point_in_femur_ref
@@ -57,9 +57,12 @@ sophiar:
         probe_var: probe_in_femur
     - name: Tibia
       trans_var: tibia_in_tracker_denoised
-      stl_file: ./models/tibia.stl
+      stl_file: ./models/Tibia_1.stl
       registration:
         collect_obj: point_picker_in_tibia_ref
         collect_var: picked_point_in_tibia_ref
         target_var: tibia_in_tibia_ref
-        probe_var: probe_in_tibia
+        probe_var: probe_in_tibia
+    - name: TibiaCut
+      trans_var: tibia_in_tracker_denoised
+      stl_file: ./models/Tibia_1_AfterClip.STL

data/sophiar_config.json

@@ -12,13 +12,13 @@
       "name": "femur_in_femur_ref",
       "type": "transform_obj",
       "value": [
-        49.7073,
-        -72.6173,
-        -9.09289,
-        0.440928,
-        0.542853,
-        0.490098,
-        0.520286
+        324.96737353558717,
+        192.97678258884025,
+        -67.4123353235597,
+        -0.3374618708678985,
+        0.7283860379847338,
+        -0.05998432687891823,
+        0.5932749328162472
       ]
     },
     {
@@ -29,13 +29,13 @@
       "name": "tibia_in_tibia_ref",
       "type": "transform_obj",
       "value": [
-        -62.500806571785446,
-        413.39747324976304,
-        -167.02739487371574,
-        -0.28854626194860417,
-        0.4854199096619968,
-        -0.2804163071762634,
-        0.7761927986582033
+        111.38885209055383,
+        -111.38354372578567,
+        55.48848071109328,
+        0.044347938544689505,
+        0.5450279992560348,
+        -0.4325668477263556,
+        0.7168428437404429
       ]
     },
     {
@@ -187,26 +187,26 @@
             "name": "left_camera",
             "parent": "camera_ref",
             "transform": [
-              1.2530,
-              36.5557,
-              -89.9613,
-              0.6107,
-              -0.3547,
-              -0.6114,
-              -0.3569
+              8.2979,
+              -2.9563,
+              -32.4969,
+              0.1872,
+              -0.6788,
+              0.1839,
+              0.6859
             ]
           },
           {
             "name": "right_camera",
             "parent": "camera_ref",
             "transform": [
-              1.3691,
-              35.9684,
-              -24.6520,
-              0.6149,
-              -0.3536,
-              -0.6076,
-              -0.3574
+              7.9742,
+              -2.8386,
+              -98.9961,
+              0.1839,
+              -0.6813,
+              0.1875,
+              0.6832
             ]
           }
         ]
@@ -260,7 +260,7 @@
       "type": "ndi_interface",
       "name": "ndi",
       "init_config": {
-        "address_type": "serial",
+        "address_type": "ethernet",
         "ip": "10.0.0.5",
         "tcp_port": 8765,
         "com_port": "/dev/ttyUSB0",

src/frame_sender/sender_udp_fec.cpp

@@ -248,12 +248,12 @@ struct sender_udp_fec::impl {
         }
 
         // parse request
-        if (length != request_size) return;
+//        if (length != request_size) return; // there may be extra data with I request
         request_type req;
-        read_request(in_buf.ptr, &req);
+        auto ptr = read_request(in_buf.ptr, &req);
         auto crc = boost::crc_32_type{};
         crc.process_bytes(in_buf.ptr + sizeof(uint32_t),
-                          request_size - sizeof(uint32_t));
+                          length - sizeof(uint32_t));
         if (crc.checksum() != req.request_checksum) { // checksum failed
             // TODO show log
             return;
@@ -273,6 +273,16 @@ struct sender_udp_fec::impl {
                 return;
             }
             case 'I': {
+                if (length == request_size + 2 * sizeof(uint16_t)) { // requested width and height
+                    uint16_t width, height;
+                    ptr = read_binary_number(ptr, &width);
+                    ptr = read_binary_number(ptr, &height);
+                    mq().update_variable<int>(OUTPUT_WIDTH, width);
+                    mq().update_variable<int>(OUTPUT_HEIGHT, height);
+                    mq().update_variable(ENCODER_SHOULD_RESET, true);
+                    SPDLOG_INFO("Output size changed to {}x{}", width, height);
+                }
+
                 remote_ep = std::make_unique<udp::endpoint>(request_ep);
                 last_confirm_ts = current_timestamp();
                 q_this->request_idr_frame();

src/image_process.cpp

@@ -232,7 +232,6 @@ struct monocular_processor::impl {
     smart_gpu_buffer<float> hsv_v_max, hsv_v_sum_log;
     smart_gpu_buffer<enhance_coeff> enhance_ext;
 
-
     void enhance_image(const cv::cuda::GpuMat &in, cv::cuda::GpuMat *out, cudaStream_t stream) {
         assert(in.type() == CV_8UC3);
 

src/main_ext.cpp

@@ -95,7 +95,6 @@ std::unique_ptr<vtk_viewer> augment_viewer;
 std::unique_ptr<std::thread> encoder_thread;
 bool output_full_frame = false;
 bool output_halve_width = false;
-int output_width = -1, output_height = -1;
 encoder_type chosen_encoder = ENCODER_JPEG;
 nvenc_config main_nvenc_conf;
 nvjpeg_config main_nvjpeg_conf;
@@ -195,6 +194,7 @@ struct camera_related {
         process_conf.crude_debayer = use_crude_debayer;
         process_conf.undistort = undistort_image;
         process_conf.enhance = enhance_image;
+        process_conf.resample_height = mq().query_variable<int>(OUTPUT_HEIGHT);
 
         // process image
         processor->process(*raw_ptr, img_dev.get(), process_conf, *stream);
@@ -293,12 +293,10 @@ void load_config() {
 
     // load output config
     auto output_conf = conf["output"];
-    output_width = output_conf["width"].as<int>();
-    output_height = output_conf["height"].as<int>();
+    mq().update_variable(OUTPUT_WIDTH, output_conf["width"].as<int>());
+    mq().update_variable(OUTPUT_HEIGHT, output_conf["height"].as<int>());
     main_nvenc_conf.bitrate_mbps = output_conf["hevc_bitrate"].as<float>();
     main_nvjpeg_conf.quality = output_conf["jpeg_quality"].as<int>();
-    left->process_conf.resample_height = output_height; // use output height as resample height
-    right->process_conf.resample_height = output_height;
 
     // load sender config
     auto sender_conf = conf["sender"];
@@ -334,6 +332,7 @@ void load_config() {
     // make variables exist
     mq().update_variable(SENDER_CONNECTED, false);
     mq().update_variable(REQUEST_IDR, false);
+    mq().update_variable(ENCODER_SHOULD_RESET, false);
 }
 
 void initialize_main_window() {
@@ -410,6 +409,7 @@ void initialize_main_window() {
     augment_viewer = std::make_unique<vtk_viewer>();
     augment_viewer->set_camera_view_angle(augment_render_angle);
     for (auto &item: augment_items) {
+        item.actor->GetProperty()->SetColor(1, 0, 0); // TODO: make color configurable
         augment_viewer->add_actor(item.actor);
     }
     reg.reset(registration::create({&sophiar_conn,
@@ -649,7 +649,8 @@ void start_encoder() {
         frame_size.width = left->img_dev->size().width * 2;
         frame_size.height = left->img_dev->size().height;
     } else {
-        frame_size = {output_width, output_height};
+        frame_size = {mq().query_variable<int>(OUTPUT_WIDTH),
+                      mq().query_variable<int>(OUTPUT_HEIGHT)};
     }
     output_fbo->create(frame_size);
 
@@ -1091,6 +1092,13 @@ void render_main_window() {
 }
 
 void generate_output_frame() {
+    // re-create encoder if needed
+    if (mq().query_variable<bool>(ENCODER_SHOULD_RESET)) {
+        stop_encoder();
+        start_encoder();
+        mq().update_variable(ENCODER_SHOULD_RESET, false);
+    }
+
     // offline drawing
     output_fbo->bind();
     auto closer = sg::make_scope_guard([&] { output_fbo->unbind(); });

src/variable_defs.h

@@ -8,6 +8,7 @@ constexpr auto ENCODER_BUSY = 2;
 constexpr auto ENCODER_CONFIG = 3;
 constexpr auto OUTPUT_FRAME = 4;
 constexpr auto ENCODER_SHOULD_STOP = 5;
+constexpr auto ENCODER_SHOULD_RESET = 15;
 
 constexpr auto CUDA_CONTEXT = 6;
 constexpr auto CUDA_STREAM_OUTPUT = 16;
@@ -17,6 +18,9 @@ constexpr auto SENDER_CONNECTED = 8;
 constexpr auto SENDER_SHOULD_STOP = 9;
 constexpr auto SENDER_ENABLE_LOG = 10;
 
+constexpr auto OUTPUT_WIDTH = 11;
+constexpr auto OUTPUT_HEIGHT = 12;
+
 // global variable declaration
 
 #endif //REMOTEAR3_VARIABLE_DEFS_H