Pārlūkot izejas kodu

Implemented variable io with Sophiar.

jcsyshc 2 gadi atpakaļ
vecāks
revīzija
34eaef5d69

+ 2 - 0
CMakeLists.txt

@@ -6,6 +6,7 @@ set(CMAKE_CXX_STANDARD 20)
 add_executable(RemoteAR2 src/main.cpp
         src/augment_renderer.cpp
         src/frame_sender.cpp
+        src/sophiar_connect.cpp
         src/third_party/rs.c)
 
 # OpenGL config
@@ -51,6 +52,7 @@ target_sources(${PROJECT_NAME} PRIVATE
         ${IMGUI_DIR}/imgui_demo.cpp
         ${IMGUI_BACKENDS_DIR}/imgui_impl_glfw.cpp
         ${IMGUI_BACKENDS_DIR}/imgui_impl_opengl3.cpp)
+target_compile_definitions(${PROJECT_NAME} PRIVATE HAVE_IMGUI)
 
 # spdlog config
 find_package(spdlog REQUIRED)

+ 4 - 2
src/augment_renderer.cpp

@@ -168,9 +168,11 @@ bool augment_renderer::render(const render_config &config) {
     return pimpl->render();
 }
 
-void augment_renderer::add_model(vtkActor *actor) {
+void augment_renderer::add_scene(scene_manager *scene) {
     assert(pimpl->vtk_render != nullptr);
-    pimpl->vtk_render->AddActor(actor);
+    for (const auto &actor: scene->actor_pool) {
+        pimpl->vtk_render->AddActor(actor.actor);
+    }
 }
 
 void augment_renderer::set_camera_pose(const Eigen::Isometry3d &pose) {

+ 2 - 1
src/augment_renderer.h

@@ -1,6 +1,7 @@
 #ifndef REMOTEAR2_AUGMENT_RENDERER_H
 #define REMOTEAR2_AUGMENT_RENDERER_H
 
+#include "scene_manager.hpp"
 #include "texture_renderer.h"
 
 #include <opencv2/core/cuda.hpp>
@@ -29,7 +30,7 @@ public:
 
     void set_background(const cv::cuda::GpuMat *background);
 
-    void add_model(vtkActor *actor);
+    void add_scene(scene_manager *scene);
 
     void set_camera_pose(const Eigen::Isometry3d &pose);
 

+ 76 - 91
src/main.cpp

@@ -2,6 +2,8 @@
 #include "config.h"
 #include "frame_buffer_helper.hpp"
 #include "frame_sender.h"
+#include "scene_manager.hpp"
+#include "sophiar_connect.h"
 #include "stereo_camera.hpp"
 #include "texture_renderer.h"
 #include "video_encoder.h"
@@ -15,10 +17,6 @@
 #include <imgui_impl_opengl3.h>
 
 #include <ExternalVTKWidget.h>
-#include <vtkActor.h>
-#include <vtkMatrix4x4.h>
-#include <vtkPolyDataMapper.h>
-#include <vtkSTLReader.h>
 
 #include <glad/gl.h>
 #include <GLFW/glfw3.h>
@@ -41,17 +39,6 @@ Eigen::Isometry3d port_info_to_transform(const Transform &info) {
     return translate * rotation;
 }
 
-// for debug;
-void transform_to_vtk_matrix(const Eigen::Isometry3d &trans, vtkMatrix4x4 *matrix) {
-    if (matrix == nullptr) return;
-    for (int i = 0; i < 4; ++i) {
-        for (int j = 0; j < 4; ++j) {
-            matrix->SetElement(i, j, trans(i, j));
-        }
-    }
-    matrix->Modified();
-}
-
 using boost::iostreams::mapped_file;
 
 CUcontext cuda_ctx;
@@ -111,31 +98,6 @@ int main() {
     cuInit(0);
     create_cuda_context(&cuda_ctx);
 
-    // for debug; setup NDI
-    auto ref_cam_trans_left = Eigen::Translation3d(-11.3375, 21.2730, -37.5694) *
-                              Eigen::Quaterniond(0.0009, -0.7088, -0.0091, 0.7053);
-    auto ref_cam_trans_right = Eigen::Translation3d(-11.2100, 22.1807, -97.3032) *
-                               Eigen::Quaterniond(0.0011, 0.7066, 0.0120, -0.7075);
-
-    CombinedApi capi;
-    ret = capi.connect("10.0.0.5");
-    assert(ret == 0);
-    ret = capi.initialize();
-    assert(ret == 0);
-    int cam_port, tool_port;
-    tool_port = capi.portHandleRequest();
-    capi.loadSromToPort("/home/tpx/data/roms/GlassProbe_4Ball_3ST.rom", tool_port);
-    cam_port = capi.portHandleRequest();
-    capi.loadSromToPort("/home/tpx/data/roms/Glass_4Ball_1.rom", cam_port);
-    auto portHandles = capi.portHandleSearchRequest(PortHandleSearchRequestOption::NotInit);
-    for (auto &handler: portHandles) {
-        ret = capi.portHandleInitialize(handler.getPortHandle());
-        assert(ret == 0);
-        ret = capi.portHandleEnable(handler.getPortHandle());
-        assert(ret == 0);
-    }
-    capi.startTracking();
-
     // working staffs
     stereo_camera camera;
     texture_renderer tex_renderer;
@@ -148,8 +110,28 @@ int main() {
     video_encoder encoder;
     encoder.initialize();
 
+    // config sophiar
+    using namespace sophiar;
+    variable_io var_io;
+//    auto femur_pose_id = var_io.add_variable("femur_in_tracker", VT_transform, VD_Input);
+//    auto tibia_pose_id = var_io.add_variable("tibia_in_tracker", VT_transform, VD_Input);
+    auto left_cam_pose_id = var_io.add_variable("left_camera_in_tracker", VT_transform, VD_Input);
+    auto right_cam_pose_id = var_io.add_variable("right_camera_in_tracker", VT_transform, VD_Input);
+
+    // for debug; config sophiar
+    auto probe_pose_id = var_io.add_variable("probe_in_tracker", VT_transform, VD_Input);
+    var_io.connect("127.0.0.1", 5278);
+
     auto vtk_external = vtkSmartPointer<ExternalVTKWidget>::New();
 
+    scene_manager scene;
+    scene.var_io = &var_io;
+//    scene.add_actor("femur.stl", femur_pose_id);
+//    scene.add_actor("tibia.stl", tibia_pose_id);
+
+    // for debug;
+    scene.add_actor("/home/tpx/data/stls/GlassProbe_4Ball_3.STL", probe_pose_id);
+
     augment_renderer left_ar, right_ar;
     auto left_remap_file = mapped_file{"left_proj.dat", boost::iostreams::mapped_file_base::readonly};
     auto right_remap_file = mapped_file{"right_proj.dat", boost::iostreams::mapped_file_base::readonly};
@@ -165,37 +147,26 @@ int main() {
     right_ar.initialize(&tex_renderer, &right_cam_info);
     left_ar.set_background(&camera.left_rgb_image);
     right_ar.set_background(&camera.right_rgb_image);
+    left_ar.add_scene(&scene);
+    right_ar.add_scene(&scene);
     left_remap_file.close();
     right_remap_file.close();
 
-    // for debug; setup VTK scene
-    vtkNew <vtkSTLReader> reader;
-    reader->SetFileName("/home/tpx/data/stls/GlassProbe_4Ball_3.STL");
-    vtkNew <vtkPolyDataMapper> mapper;
-    mapper->SetInputConnection(reader->GetOutputPort());
-    vtkNew <vtkActor> actor;
-    vtkNew <vtkMatrix4x4> tool_matrix;
-    actor->SetUserMatrix(tool_matrix);
-    actor->SetMapper(mapper);
-    left_ar.add_model(actor);
-    right_ar.add_model(actor);
-
     int camera_fps = default_camera_fps;
     float exposure_time_ms = default_camera_exposure_time_ms;
     float analog_gain = default_camera_analog_gain;
     float output_bitrate_mbps = default_video_stream_bitrate / 1e6f;
-    int sender_port = 5277;
+    int sender_port = 5279;
 
-    FILE *video_save_file = nullptr;
-    auto video_save_file_closer = sg::make_scope_guard([&]() {
-        if (video_save_file == nullptr) return;
-        fclose(video_save_file);
-    });
+    int preview_cam = 0; // left camera
+    bool save_video = true; // whether encoded bitstream should be saved
+    bool save_frame_length = true; // save frame length for tiny player to replay
 
     // main loop
     while (!glfwWindowShouldClose(main_window)) {
 
         glfwPollEvents();
+        var_io.retrieve_update();
 
         ImGui_ImplOpenGL3_NewFrame();
         ImGui_ImplGlfw_NewFrame();
@@ -206,9 +177,10 @@ int main() {
             encoder.stop_encode();
         }
 
-        // ImGui::ShowDemoWindow();
+        ImGui::ShowDemoWindow();
 
         if (ImGui::Begin("Remote AR Control")) {
+            ImGui::PushItemWidth(200);
 
             // camera control
             if (ImGui::CollapsingHeader("Camera")) {
@@ -245,16 +217,21 @@ int main() {
                         ImGui::BeginDisabled();
                     }
 
-                    ImGui::PushItemWidth(200);
                     ImGui::SliderInt("Frame Rate (fps)", &camera_fps, 1, 60);
                     ImGui::DragFloat("Exposure Time (ms)", &exposure_time_ms,
                                      0.1, 1, 1e3f / (float) camera_fps, "%.01f");
                     ImGui::DragFloat("Analog Gain (dB)", &analog_gain, 0.1, 0, 24, "%.01f");
-                    ImGui::PopItemWidth();
 
                     if (camera.is_capturing()) {
                         ImGui::EndDisabled();
                     }
+
+                    if (camera.is_capturing()) {
+                        ImGui::SeparatorText("Preview Camera");
+                        ImGui::RadioButton("Left", &preview_cam, 0);
+                        ImGui::SameLine();
+                        ImGui::RadioButton("Right", &preview_cam, 1);
+                    }
                 }
 
                 ImGui::PopID();
@@ -267,19 +244,13 @@ int main() {
                 ImGui::SeparatorText("Actions");
                 if (!encoder.is_encoding()) {
                     if (ImGui::Button("Start")) {
-                        // create save file
-//                        assert(video_save_file == nullptr); // TODO: move into encoder
-//                        auto file_name = fmt::format("record_{:%Y_%m_%d_%H_%M_%S}.hevc",
-//                                                     std::chrono::system_clock::now());
-//                        video_save_file = fopen(file_name.c_str(), "wb");
                         encoder.start_encode(output_fbo.tex_width, output_fbo.tex_height,
-                                             camera_fps, (int) (output_bitrate_mbps * 1e6));
+                                             camera_fps, (int) (output_bitrate_mbps * 1e6),
+                                             save_video, save_frame_length);
                     }
                 } else {
                     if (ImGui::Button("Close")) {
                         encoder.stop_encode();
-//                        fclose(video_save_file);
-//                        video_save_file = nullptr;
                     }
                 }
 
@@ -288,9 +259,12 @@ int main() {
                     ImGui::BeginDisabled();
                 }
 
-                ImGui::PushItemWidth(200);
                 ImGui::DragFloat("Bitrate (Mbps)", &output_bitrate_mbps, 0.1, 1, 20, "%.01f");
-                ImGui::PopItemWidth();
+                ImGui::Checkbox("Save Video", &save_video);
+                if (save_video) {
+                    ImGui::SameLine();
+                    ImGui::Checkbox("Save Frame Length", &save_frame_length);
+                }
 
                 if (encoder.is_encoding()) {
                     ImGui::EndDisabled();
@@ -327,25 +301,21 @@ int main() {
                 ImGui::PopID();
             }
 
-        }
-        ImGui::End();
-        ImGui::Render();
+            if (ImGui::CollapsingHeader("Sophiar Navigation")) {
+                ImGui::PushID("Sophiar");
 
-        // for debug; NDI work
-        auto tool_data = capi.getTrackingDataBX();
-        for (auto &tool: tool_data) {
-            auto port = tool.transform.toolHandle;
-            if (port == 65535) continue;
-            if (tool.transform.isMissing()) continue;
-            auto trans = port_info_to_transform(tool.transform);
-            if (port == cam_port) {
-                left_ar.set_camera_pose(trans * ref_cam_trans_left);
-                right_ar.set_camera_pose(trans * ref_cam_trans_right);
-            } else if (port == tool_port) {
-                transform_to_vtk_matrix(trans, tool_matrix);
-                actor->Modified();
+                ImGui::BeginDisabled();
+                var_io.show_input_variable(left_cam_pose_id);
+                var_io.show_input_variable(probe_pose_id);
+                ImGui::EndDisabled();
+
+                ImGui::PopID();
             }
+
+            ImGui::PopItemWidth();
         }
+        ImGui::End();
+        ImGui::Render();
 
         std::chrono::high_resolution_clock::time_point start_time;
         if (camera.is_capturing()) {
@@ -354,6 +324,20 @@ int main() {
             camera.debayer_images();
         }
 
+        // retrieve sophiar update again
+        var_io.retrieve_update();
+
+        // update scene and camera pose
+        scene.update_pose();
+        auto left_cam_pose = var_io.query_transform_variable(left_cam_pose_id);
+        auto right_cam_pose = var_io.query_transform_variable(right_cam_pose_id);
+        if (left_cam_pose.has_value()) {
+            left_ar.set_camera_pose(*left_cam_pose.value());
+        }
+        if (right_cam_pose.has_value()) {
+            right_ar.set_camera_pose(*right_cam_pose.value());
+        }
+
         if (encoder.is_encoding()) {
             // draw frame for streaming
             glBindFramebuffer(GL_DRAW_FRAMEBUFFER, output_fbo.fbo);
@@ -374,9 +358,6 @@ int main() {
             SPDLOG_TRACE("Time used: {}ms, length = {}", std::chrono::duration_cast<std::chrono::milliseconds>(
                     std::chrono::high_resolution_clock::now() - start_time).count(), info.length);
 
-            // save encoded frame
-//            fwrite(info.data, info.length, 1, video_save_file);
-
             // cleanup
             if (sender.is_running()) {
                 sender.send_frame(info);
@@ -393,7 +374,11 @@ int main() {
 
         if (camera.is_capturing()) {
             // draw frame in the screen
-            right_ar.render({-1, 1, 2, -2});
+            if (preview_cam == 0) { // left cam
+                left_ar.render({-1, 1, 2, -2});
+            } else {
+                right_ar.render({-1, 1, 2, -2});
+            }
         }
 
         ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());

+ 72 - 0
src/scene_manager.hpp

@@ -0,0 +1,72 @@
+#ifndef REMOTEAR2_SCENE_MANAGER_HPP
+#define REMOTEAR2_SCENE_MANAGER_HPP
+
+#include "sophiar_connect.h"
+
+#include <vtkActor.h>
+#include <vtkMatrix4x4.h>
+#include <vtkPolyDataMapper.h>
+#include <vtkProperty.h>
+#include <vtkSmartPointer.h>
+#include <vtkSTLReader.h>
+
+#include <vector>
+
+struct scene_manager {
+
+    struct actor_info {
+        vtkSmartPointer<vtkActor> actor;
+        vtkSmartPointer<vtkMatrix4x4> pose;
+        int pose_variable_id = -1; // -1 will not be updated
+    };
+
+    using actor_pool_type = std::vector<actor_info>;
+    actor_pool_type actor_pool;
+
+    sophiar::variable_io *var_io = nullptr;
+
+    static void transform_to_vtk_matrix(const sophiar::transform_type &trans, vtkMatrix4x4 *matrix) {
+        if (matrix == nullptr) return;
+        for (int i = 0; i < 4; ++i) {
+            for (int j = 0; j < 4; ++j) {
+                matrix->SetElement(i, j, trans(i, j));
+            }
+        }
+        matrix->Modified();
+    }
+
+    void add_actor(std::string_view file_name, int variable_id) {
+        vtkNew<vtkSTLReader> reader;
+        reader->SetFileName(file_name.data());
+        reader->Update();
+        vtkNew<vtkPolyDataMapper> mapper;
+        mapper->SetInputData(reader->GetOutput());
+        vtkNew<vtkActor> actor;
+        actor->SetMapper(mapper);
+        vtkNew<vtkMatrix4x4> pose_matrix;
+        actor->SetUserMatrix(pose_matrix);
+
+        auto &next_actor = actor_pool.emplace_back();
+        next_actor.actor = actor;
+        next_actor.pose = pose_matrix;
+        next_actor.pose_variable_id = variable_id;
+    }
+
+    void update_pose() {
+        assert(var_io != nullptr);
+        for (auto &actor: actor_pool) {
+            assert(actor.pose_variable_id != -1);
+            auto next_pose = var_io->query_transform_variable(actor.pose_variable_id);
+            if (next_pose.has_value()) {
+                transform_to_vtk_matrix(*next_pose.value(), actor.pose);
+                actor.actor->SetVisibility(true);
+            } else {
+                actor.actor->SetVisibility(false);
+            }
+            actor.actor->Modified();
+        }
+    }
+
+};
+
+#endif //REMOTEAR2_SCENE_MANAGER_HPP

+ 544 - 0
src/sophiar_connect.cpp

@@ -0,0 +1,544 @@
+#include "sophiar_connect.h"
+
+#include <boost/asio/buffer.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/read.hpp>
+#include <boost/asio/write.hpp>
+#include <boost/endian.hpp>
+#include <boost/smart_ptr.hpp>
+
+#include <spdlog/spdlog.h>
+
+#include <charconv>
+#include <bit>
+#include <variant>
+#include <vector>
+#include <unordered_map>
+
+#ifdef HAVE_IMGUI
+
+#include <imgui.h>
+
+#endif
+
+using namespace boost::asio::ip;
+using namespace std::string_view_literals;
+using boost::asio::buffer;
+using boost::asio::io_context;
+using boost::asio::read;
+using boost::asio::write;
+
+namespace sophiar {
+
+    struct variable_io::impl {
+
+        using var_store_type = std::variant<std::monostate,
+                bool_type, integer_type, float_type,
+                scalar_xyz_type, transform_type, array6_type>;
+        using var_index_type = uint16_t;
+        using var_len_type = uint8_t;
+
+        static constexpr var_len_type var_bin_len[] = {0, 1, 8, 8, 24, 56, 48};
+
+        struct var_info {
+            variable_type type;
+            var_index_type index;
+            std::string_view name;
+            var_store_type value;
+        };
+
+        using var_pool_type = std::vector<var_info>;
+        var_pool_type var_in_pool, var_out_pool;
+
+        using var_index_id_map_type = std::unordered_map<var_index_type, int>;
+        var_index_id_map_type var_index_id_map; // for VD_Input
+
+        tcp::endpoint server_ep;
+
+        boost::scoped_ptr<io_context> context;
+        boost::scoped_ptr<tcp::socket> socket_in, socket_out;
+
+        char *in_data = nullptr, *out_data = nullptr;
+        size_t in_data_len = 0, out_data_len = 0;
+
+        using msg_len_type = uint16_t;
+        msg_len_type next_msg_len = 0;
+        msg_len_type cur_out_len = sizeof(msg_len_type); // used for variable output
+
+        template<typename T>
+        static char *write_binary_number(char *ptr, T val) {
+            static constexpr auto need_swap =
+                    (boost::endian::order::native != boost::endian::order::big);
+            if constexpr (need_swap && sizeof(T) > 1) {
+                boost::endian::endian_reverse_inplace(val);
+            }
+            auto real_ptr = (T *) ptr;
+            *real_ptr = val;
+            return ptr + sizeof(T);
+        }
+
+        template<typename T>
+        static char *read_binary_number(char *ptr, T *val) {
+            static constexpr auto need_swap =
+                    (boost::endian::order::native != boost::endian::order::big);
+            *val = *(T *) ptr;
+            if constexpr (need_swap && sizeof(T) > 1) {
+                boost::endian::endian_reverse_inplace(*val);
+            }
+            return ptr + sizeof(T);
+        }
+
+        template<typename ...Args>
+        static char *write_binary_numbers(char *ptr, Args... args) {
+            return (..., (ptr = write_binary_number(ptr, args)));
+        }
+
+        template<typename ...Args>
+        static char *read_binary_numbers(char *ptr, Args... args) {
+            static_assert((... && std::is_pointer_v<Args>));
+            return (..., (ptr = read_binary_number(ptr, args)));
+        }
+
+        static char *write_variable(char *ptr, const var_store_type &val) {
+            switch (val.index()) {
+
+#define SIMPLE_CASE(TYPE) \
+                case VT_##TYPE: { \
+                    auto real_val = std::get<TYPE##_type>(val); \
+                    return write_binary_number(ptr, real_val); \
+                }
+
+                SIMPLE_CASE(bool)
+                SIMPLE_CASE(integer)
+                SIMPLE_CASE(float)
+
+#undef SIMPLE_CASE
+
+                case VT_scalar_xyz: {
+                    auto &real_val = std::get<scalar_xyz_type>(val);
+                    return write_binary_numbers(ptr, real_val.x(), real_val.y(), real_val.z());
+                }
+                case VT_transform: {
+                    auto &real_val = std::get<transform_type>(val);
+                    auto trans_part = real_val.translation();
+                    auto quat_part = Eigen::Quaterniond{real_val.rotation()};
+                    ptr = write_binary_numbers(ptr, trans_part.x(), trans_part.y(), trans_part.z());
+                    ptr = write_binary_numbers(ptr, quat_part.w(), quat_part.x(), quat_part.y(), quat_part.z());
+                    return ptr;
+                }
+                case VT_array6: {
+                    auto &real_val = std::get<array6_type>(val);
+                    for (auto &elem: real_val) {
+                        ptr = write_binary_number(ptr, elem);
+                    }
+                    return ptr;
+                }
+                default: {
+                    assert(val.index() == 0); // empty variable
+                    return ptr;
+                }
+            }
+            assert(false);
+            return ptr;
+        }
+
+        static char *read_variable(char *ptr, var_store_type *val, variable_type type) {
+            switch (type) {
+
+#define SIMPLE_CASE(TYPE) \
+                case VT_##TYPE: { \
+                    auto &real_val = std::get<TYPE##_type>(*val); \
+                    return read_binary_number(ptr, &real_val); \
+                }
+
+                SIMPLE_CASE(bool)
+                SIMPLE_CASE(integer)
+                SIMPLE_CASE(float)
+
+#undef SIMPLE_CASE
+
+                case VT_scalar_xyz: {
+                    auto &real_val = std::get<scalar_xyz_type>(*val);
+                    return read_binary_numbers(ptr, &real_val.x(), &real_val.y(), &real_val.z());
+                }
+                case VT_transform: {
+                    double tx, ty, tz, qw, qx, qy, qz;
+                    ptr = read_binary_numbers(ptr, &tx, &ty, &tz, &qw, &qx, &qy, &qz);
+                    std::get<transform_type>(*val) =
+                            Eigen::Translation3d(tx, ty, tz) * Eigen::Quaterniond(qw, qx, qy, qz);
+                    return ptr;
+                }
+                case VT_array6: {
+                    auto &real_val = std::get<array6_type>(*val);
+                    for (auto &elem: real_val) {
+                        ptr = read_binary_number(ptr, &elem);
+                    }
+                    return ptr;
+                }
+            }
+            assert(false);
+            return ptr;
+        }
+
+        static void init_var_store(var_store_type *val, variable_type type) {
+            switch (type) {
+
+#define SIMPLE_CASE(TYPE) \
+                case VT_##TYPE : { \
+                    val->emplace<VT_##TYPE>(); \
+                    return; \
+                }
+
+                SIMPLE_CASE(bool)
+                SIMPLE_CASE(integer)
+                SIMPLE_CASE(float)
+                SIMPLE_CASE(scalar_xyz)
+                SIMPLE_CASE(transform)
+                SIMPLE_CASE(array6)
+
+#undef SIMPLE_CASE
+            }
+        }
+
+#ifdef HAVE_IMGUI
+
+        static void show_variable(const var_store_type *val, variable_type type, std::string_view name) {
+            static auto empty_color = (ImVec4) ImColor::HSV(0, 1, 1);
+
+            switch (type) {
+                case VT_scalar_xyz: {
+                    Eigen::Vector3f elems = Eigen::Vector3f::Zero();
+                    if (val->index() == 0) {
+                        ImGui::PushStyleColor(ImGuiCol_Text, empty_color);
+                    } else {
+                        auto &real_val = std::get<scalar_xyz_type>(*val);
+                        elems = real_val.cast<float>();
+                    }
+                    ImGui::InputFloat3(name.data(), elems.data(), "%.1f");
+                    if (val->index() == 0) {
+                        ImGui::PopStyleColor();
+                    }
+                    return;
+                }
+                case VT_transform: {
+                    Eigen::Vector3f trans_part = Eigen::Vector3f::Zero();
+                    Eigen::Quaternionf quat_part = Eigen::Quaternionf::Identity();
+                    auto trans_name = fmt::format("{} (trans)", name);
+                    auto quat_name = fmt::format("{} (quat)", name);
+                    if (val->index() == 0) {
+                        ImGui::PushStyleColor(ImGuiCol_Text, empty_color);
+                    } else {
+                        auto &real_val = std::get<transform_type>(*val);
+                        trans_part = real_val.translation().cast<float>();
+                        quat_part = real_val.rotation().cast<float>();
+                    }
+                    ImGui::InputFloat3(trans_name.c_str(), trans_part.data(), "%.1f");
+                    ImGui::InputFloat4(quat_name.c_str(), quat_part.coeffs().data(), "%.3f");
+                    if (val->index() == 0) {
+                        ImGui::PopStyleColor();
+                    }
+                    return;
+                }
+            }
+        }
+
+#endif
+
+        static size_t least_power_of_2(size_t x) {
+            auto ret = std::bit_width(x);
+            if (std::has_single_bit(x)) --ret;
+            return 1ull << ret;
+        }
+
+        void ensure_buffer(size_t in_len, size_t out_len, bool keep_out_data = false) {
+            if (in_len > in_data_len) [[unlikely]] {
+                auto next_len = least_power_of_2(in_len);
+                free(in_data);
+                in_data = (char *) malloc(next_len);
+                in_data_len = next_len;
+            }
+            if (out_len > out_data_len) [[unlikely]] {
+                auto next_len = least_power_of_2(out_len);
+                auto next_out_data = (char *) malloc(next_len);
+                if (keep_out_data) [[likely]] {
+                    memcpy(next_out_data, out_data, out_data_len);
+                }
+                free(out_data);
+                out_data = next_out_data;
+                out_data_len = next_len;
+            }
+        }
+
+        size_t write_hello_msg(variable_direction dir) {
+            static constexpr auto var_in_cmd = "VAROUTB"sv;
+            static constexpr auto var_out_cmd = "VARINB"sv;
+
+            // calculate required length
+            const auto &pool = (dir == VD_Input ? var_in_pool : var_out_pool);
+            size_t var_cnt = pool.size();
+            if (var_cnt == 0) return 0;
+            size_t var_len = 0;
+            for (auto &var: pool) {
+                var_len += var.name.length();
+            }
+            auto cmd = (dir == VD_Input ? var_in_cmd : var_out_cmd);
+            msg_len_type msg_len = cmd.length() + 1 + (var_cnt - 1) + var_len;
+            auto out_len = sizeof(msg_len_type) + msg_len;
+            ensure_buffer(0, out_len);
+
+            // fill buffer
+            auto out_ptr = write_binary_number(out_data, msg_len);
+            memcpy(out_ptr, cmd.data(), cmd.length());
+            out_ptr += cmd.length();
+            *(out_ptr++) = ' ';
+            for (auto &var: pool) {
+                memcpy(out_ptr, var.name.data(), var.name.length());
+                out_ptr += var.name.length();
+                if (--var_cnt) {
+                    *(out_ptr++) = ',';
+                }
+            }
+            assert(out_ptr - out_data == out_len);
+            return out_len;
+        }
+
+        bool try_receive_msg(tcp::socket *s, bool nonblock = true) {
+            if (next_msg_len == 0) {
+                if (nonblock) {
+                    if (s->available() < sizeof(msg_len_type)) return false;
+                }
+                ensure_buffer(sizeof(msg_len_type), 0);
+                auto bytes_read = read(*s, buffer(in_data, sizeof(msg_len_type)));
+                assert(bytes_read == sizeof(msg_len_type));
+                read_binary_number(in_data, &next_msg_len);
+            }
+            assert(next_msg_len != 0);
+            if (nonblock) {
+                if (s->available() < next_msg_len) return false;
+            }
+            ensure_buffer(sizeof(msg_len_type) + next_msg_len, 0);
+            write_binary_number(in_data, next_msg_len); // in_data may be modified
+            auto bytes_read = read(*s, buffer(in_data + sizeof(msg_len_type), next_msg_len));
+            assert(bytes_read == next_msg_len);
+            next_msg_len = 0;
+            return true;
+        }
+
+        void save_variable_index(variable_direction dir) {
+            msg_len_type msg_len;
+            auto in_ptr = read_binary_number(in_data, &msg_len);
+            auto end_ptr = in_ptr + msg_len;
+            auto &pool = (dir == VD_Input ? var_in_pool : var_out_pool);
+            for (auto &var: pool) {
+                auto ret = std::from_chars(in_ptr, end_ptr, var.index);
+                assert(ret.ec == std::errc{}); // no error
+                if (ret.ptr != end_ptr) {
+                    assert(*ret.ptr == ',');
+                    in_ptr = (char *) ret.ptr + 1;
+                }
+            }
+
+            // build index -> id map
+            if (dir == VD_Input) {
+                for (auto i = 0; i < var_in_pool.size(); ++i) {
+                    auto var_index = var_in_pool[i].index;
+                    var_index_id_map.insert({var_index, i});
+                }
+            }
+        }
+
+        void setup_variable_io(tcp::socket *s, variable_direction dir) {
+            auto msg_len = write_hello_msg(dir);
+            if (msg_len == 0) return;
+            s->connect(server_ep);
+            assert(s->is_open());
+            write(*s, buffer(out_data, msg_len));
+            try_receive_msg(s, false);
+            save_variable_index(dir);
+        }
+
+        void initialize() {
+            next_msg_len = 0;
+            cur_out_len = sizeof(msg_len_type);
+
+            context.reset(new io_context{});
+            socket_in.reset(new tcp::socket{*context});
+            socket_out.reset(new tcp::socket{*context});
+            setup_variable_io(socket_in.get(), VD_Input);
+            setup_variable_io(socket_out.get(), VD_Output);
+        }
+
+        void handle_var_update_msg() {
+            msg_len_type msg_len;
+            auto in_ptr = read_binary_number(in_data, &msg_len);
+            auto end_ptr = in_ptr + msg_len;
+            while (in_ptr != end_ptr) {
+                var_index_type var_index;
+                var_len_type var_len;
+                in_ptr = read_binary_numbers(in_ptr, &var_index, &var_len);
+                auto var_id = var_index_id_map.at(var_index);
+                auto &var_info = var_in_pool[var_id];
+                auto &var_store = var_info.value;
+                auto var_type = var_info.type;
+                if (var_len == 0) { // empty variable
+                    var_store.emplace<0>();
+                } else {
+                    if (var_store.index() == 0) [[unlikely]] {
+                        init_var_store(&var_store, var_type);
+                    }
+                    auto old_ptr = in_ptr;
+                    in_ptr = read_variable(in_ptr, &var_store, var_type);
+                    assert(in_ptr - old_ptr == var_len);
+                }
+            }
+        }
+
+        void update_in_var() {
+            assert(context != nullptr);
+            if (!socket_in->is_open()) return;
+            while (try_receive_msg(socket_in.get())) {
+                handle_var_update_msg();
+            }
+        }
+
+        void update_out_var(int var_id) {
+            const auto &var_info = var_out_pool.at(var_id);
+            auto &var_store = var_info.value;
+            auto var_len = var_bin_len[var_store.index()];
+            auto var_index = var_info.index;
+
+            auto var_frag_len = sizeof(var_index_type) + sizeof(var_len_type) + var_len;
+            auto next_out_len = cur_out_len + var_frag_len;
+            if (next_out_len > std::numeric_limits<msg_len_type>::max()) {
+                commit_out_var();
+                next_out_len = cur_out_len + var_frag_len;
+            }
+            ensure_buffer(0, next_out_len, true);
+            auto out_ptr = out_data + cur_out_len;
+            out_ptr = write_binary_numbers(out_ptr, var_index, var_len);
+            out_ptr = write_variable(out_ptr, var_store);
+            assert(out_ptr - out_data == next_out_len);
+            cur_out_len = next_out_len;
+        }
+
+        void commit_out_var() {
+            assert(cur_out_len >= sizeof(msg_len_type));
+            if (cur_out_len <= sizeof(msg_len_type)) return;
+            write_binary_number(out_data, cur_out_len);
+            write(*socket_out, buffer(out_data, cur_out_len));
+            cur_out_len = sizeof(msg_len_type);
+        }
+
+        ~impl() {
+            free(in_data);
+            free(out_data);
+        }
+
+    };
+
+    variable_io::variable_io()
+            : pimpl(std::make_unique<impl>()) {}
+
+    variable_io::~variable_io() = default;
+
+    int variable_io::add_variable(std::string_view name, variable_type type, variable_direction dir) {
+        auto &pool = (dir == VD_Input ? pimpl->var_in_pool : pimpl->var_out_pool);
+        auto var_id = (int) pool.size();
+        auto &var_info = pool.emplace_back();
+        var_info.type = type;
+        var_info.name = name;
+        return var_id;
+    }
+
+    bool variable_io::connect(std::string_view server, uint16_t port) {
+        pimpl->server_ep = tcp::endpoint{address::from_string(server.data()), port};
+        try {
+            pimpl->initialize();
+        } catch (std::exception &e) {
+            SPDLOG_ERROR("Failed while establishing variable IO: {}", e.what());
+            return false;
+        }
+        return true;
+    }
+
+    bool variable_io::retrieve_update() {
+        try {
+            pimpl->update_in_var();
+        } catch (std::exception &e) {
+            SPDLOG_ERROR("Failed while retrieving variable update: {}", e.what());
+            return false;
+        }
+        return true;
+    }
+
+#define VAL_RET_CASE_IMPL(TYPE) \
+    std::optional<TYPE##_type> variable_io::query_##TYPE##_variable(int variable_id) { \
+        const auto &var_store = pimpl->var_in_pool.at(variable_id).value; \
+        if (var_store.index() == 0) return {}; \
+        return std::get<TYPE##_type>(var_store); \
+    } \
+    void variable_io::update_##TYPE##_variable(int variable_id, std::optional<TYPE##_type> val) { \
+        assert(pimpl->var_out_pool[variable_id].type == VT_##TYPE); \
+        auto &var_store = pimpl->var_out_pool.at(variable_id).value; \
+        if (val.has_value()) { \
+            var_store = val.value(); \
+        } else { \
+            var_store.emplace<0>(); \
+        } \
+        pimpl->update_out_var(variable_id); \
+    }
+
+    VAL_RET_CASE_IMPL(bool)
+
+    VAL_RET_CASE_IMPL(integer)
+
+    VAL_RET_CASE_IMPL(float)
+
+#undef VAL_RET_CASE_IMPL
+
+#define REF_RET_CASE_IMPL(TYPE) \
+    std::optional<const TYPE##_type *> variable_io::query_##TYPE##_variable(int variable_id) { \
+        const auto &var_store = pimpl->var_in_pool.at(variable_id).value; \
+        if (var_store.index() == 0) return {}; \
+        return &std::get<TYPE##_type>(var_store); \
+    } \
+    void variable_io::update_##TYPE##_variable(int variable_id, std::optional<const TYPE##_type *> val) { \
+        assert(pimpl->var_out_pool[variable_id].type == VT_##TYPE); \
+        auto &var_store = pimpl->var_out_pool.at(variable_id).value; \
+        if (val.has_value()) { \
+            var_store = *val.value(); \
+        } else { \
+            var_store.emplace<0>(); \
+        } \
+        pimpl->update_out_var(variable_id); \
+    }
+
+    REF_RET_CASE_IMPL(scalar_xyz)
+
+    REF_RET_CASE_IMPL(transform)
+
+    REF_RET_CASE_IMPL(array6)
+
+#undef REF_RET_CASE_IMPL
+
+    bool variable_io::commit_update() {
+        try {
+            pimpl->commit_out_var();
+        } catch (std::exception &e) {
+            SPDLOG_ERROR("Failed while committing variable update: {}", e.what());
+            return false;
+        }
+        return true;
+    }
+
+#ifdef HAVE_IMGUI
+
+    void variable_io::show_input_variable(int variable_id) {
+        const auto &var_info = pimpl->var_in_pool.at(variable_id);
+        impl::show_variable(&var_info.value, var_info.type, var_info.name);
+    }
+
+#endif
+
+};

+ 84 - 0
src/sophiar_connect.h

@@ -0,0 +1,84 @@
+#ifndef REMOTEAR2_SOPHIAR_CONNECT_H
+#define REMOTEAR2_SOPHIAR_CONNECT_H
+
+#include <Eigen/Geometry>
+
+#include <memory>
+#include <optional>
+
+namespace sophiar {
+
+    enum variable_type {
+        VT_bool = 1,
+        VT_integer = 2,
+        VT_float = 3,
+        VT_scalar_xyz = 4,
+        VT_transform = 5,
+        VT_array6 = 6
+    };
+
+    enum variable_direction {
+        VD_Input, VD_Output
+    };
+
+    using bool_type = bool;
+    using integer_type = uint64_t;
+    using float_type = double;
+    using scalar_xyz_type = Eigen::Vector3d;
+    using transform_type = Eigen::Isometry3d;
+    using array6_type = std::array<double, 6>;
+
+    class variable_io {
+    public:
+
+        variable_io();
+
+        ~variable_io();
+
+        int add_variable(std::string_view name, variable_type type, variable_direction dir);
+
+        bool connect(std::string_view server, uint16_t port);
+
+        bool retrieve_update();
+
+#define VAL_RET_CASE(TYPE) \
+        std::optional<TYPE##_type> query_##TYPE##_variable(int variable_id); \
+        void update_##TYPE##_variable(int variable_id, std::optional<TYPE##_type> val);
+
+        VAL_RET_CASE(bool)
+
+        VAL_RET_CASE(integer)
+
+        VAL_RET_CASE(float)
+
+#undef VAL_RET_CASE
+
+#define REF_RET_CASE(TYPE) \
+        std::optional<const TYPE##_type *> query_##TYPE##_variable(int variable_id); \
+        void update_##TYPE##_variable(int variable_id, std::optional<const TYPE##_type *> val);
+
+        REF_RET_CASE(scalar_xyz)
+
+        REF_RET_CASE(transform)
+
+        REF_RET_CASE(array6)
+
+#undef REF_RET_CASE
+
+        bool commit_update();
+
+#ifdef HAVE_IMGUI
+
+        void show_input_variable(int variable_id);
+
+#endif
+
+    private:
+        struct impl;
+        std::unique_ptr<impl> pimpl;
+    };
+
+}
+
+
+#endif //REMOTEAR2_SOPHIAR_CONNECT_H

+ 31 - 1
src/video_encoder.cpp

@@ -4,6 +4,11 @@
 
 #include <nvEncodeAPI.h>
 
+#include <fmt/chrono.h>
+#include <fmt/format.h>
+
+#include <cstdio>
+
 bool check_nvenc_api_call(NVENCSTATUS api_ret, unsigned int line_number,
                           const char *file_name, const char *api_call_str) {
     if (api_ret == NV_ENC_SUCCESS) [[likely]] return true;
@@ -38,6 +43,8 @@ struct video_encoder::impl {
     NV_ENC_REGISTERED_PTR frame_reg_ptr = nullptr;
 
     bool idr_flag = false;
+    bool save_frame_length = true;
+    FILE *video_save_file = nullptr;
 
     bool initialize() {
         NVENC_API_CHECK(NvEncodeAPICreateInstance(&api));
@@ -119,6 +126,12 @@ struct video_encoder::impl {
         api.nvEncDestroyEncoder(encoder);
         encoder = nullptr;
 
+        // close save file
+        if (video_save_file != nullptr) {
+            fclose(video_save_file);
+            video_save_file = nullptr;
+        }
+
         SPDLOG_INFO("Video encoder stopped.");
     }
 
@@ -180,6 +193,14 @@ struct video_encoder::impl {
         *output_size = lock_config.bitstreamSizeInBytes;
         memcpy(*output_ptr, lock_config.bitstreamBufferPtr, lock_config.bitstreamSizeInBytes);
 
+        // save bitstream
+        if (video_save_file != nullptr) {
+            if (save_frame_length) {
+                fwrite(output_size, sizeof(*output_size), 1, video_save_file);
+            }
+            fwrite(*output_ptr, *output_size, 1, video_save_file);
+        }
+
         // cleanup
         NVENC_API_CHECK(api.nvEncUnlockBitstream(encoder, output_buf));
         NVENC_API_CHECK(api.nvEncUnmapInputResource(encoder, map_params.mappedResource));
@@ -200,12 +221,21 @@ bool video_encoder::initialize() {
     return pimpl->initialize();
 }
 
-bool video_encoder::start_encode(int width, int height, int fps, int bitrate) {
+bool video_encoder::start_encode(int width, int height, int fps, int bitrate,
+                                 bool save_file, bool save_frame_length) {
     pimpl->frame_width = width;
     pimpl->frame_height = height;
     pimpl->frame_pitch = width * 4; // ARGB image
     pimpl->frame_rate = fps;
     pimpl->frame_bitrate = bitrate;
+
+    // create save file
+    assert(pimpl->video_save_file == nullptr);
+    auto file_name = fmt::format("record_{:%Y_%m_%d_%H_%M_%S}.{}",
+                                 std::chrono::system_clock::now(),
+                                 save_frame_length ? "dat" : "hevc");
+    pimpl->video_save_file = fopen(file_name.c_str(), "wb");
+
     return pimpl->start_encode();
 }
 

+ 2 - 1
src/video_encoder.h

@@ -14,7 +14,8 @@ public:
 
     bool initialize();
 
-    bool start_encode(int width, int height, int fps, int bitrate);
+    bool start_encode(int width, int height, int fps, int bitrate,
+                      bool save_file = false, bool save_frame_length = true);
 
     void stop_encode();