Implemented AR rendering with Sophiar.

src/augment_renderer.cpp

@@ -20,9 +20,11 @@ struct augment_renderer::impl {
     texture_renderer *tex_renderer = nullptr;
     const render_config *config = nullptr;
 
+    // Switching between multiple render windows will case OpenGL to flush and waste much time.
+    static vtkSmartPointer<vtkRenderer> vtk_render;
+    static vtkSmartPointer<vtkGenericOpenGLRenderWindow> vtk_window;
+
     vtkSmartPointer<vtkCamera> vtk_cam;
-    vtkSmartPointer<vtkRenderer> vtk_render;
-    vtkSmartPointer<vtkGenericOpenGLRenderWindow> vtk_window;
 
     const camera_intrinsics *cam_info = nullptr;
     bool is_augment = true;
@@ -57,18 +59,19 @@ struct augment_renderer::impl {
                                                     cudaGraphicsRegisterFlagsWriteDiscard));
 
         // setup vtk
-        vtk_render = vtkSmartPointer<vtkRenderer>::New();
-        vtk_window = vtkSmartPointer<vtkGenericOpenGLRenderWindow>::New();
-        vtk_window->InitializeFromCurrentContext();
-        vtk_window->SetSize(cam_info->width, cam_info->height);
-        vtk_window->SetAlphaBitPlanes(true);
-        vtk_window->SetOffScreenRendering(true);
-        vtk_window->FramebufferFlipYOn();
-        vtk_window->AddRenderer(vtk_render);
         vtk_cam = vtkSmartPointer<vtkCamera>::New();
         vtk_cam->SetViewAngle(cam_info->angle);
         vtk_cam->SetClippingRange(default_len_focus, 2000);
-        vtk_render->SetActiveCamera(vtk_cam);
+        if (vtk_window == nullptr) {
+            vtk_render = vtkSmartPointer<vtkRenderer>::New();
+            vtk_window = vtkSmartPointer<vtkGenericOpenGLRenderWindow>::New();
+            vtk_window->InitializeFromCurrentContext();
+            vtk_window->SetSize(cam_info->width, cam_info->height);
+            vtk_window->SetAlphaBitPlanes(true);
+            vtk_window->SetOffScreenRendering(true);
+            vtk_window->FramebufferFlipYOn();
+            vtk_window->AddRenderer(vtk_render);
+        }
 
         return true;
     }
@@ -116,10 +119,11 @@ struct augment_renderer::impl {
         vtk_cam->SetFocalPoint(focal_point.x(), focal_point.y(), focal_point.z());
         vtk_cam->SetViewUp(view_up.x(), view_up.y(), view_up.z());
         vtk_cam->Modified();
-        vtk_render->Modified();
     }
 
     void render_vtk() {
+        vtk_render->SetActiveCamera(vtk_cam);
+        vtk_render->Modified();
         vtk_window->SetIsCurrent(true);
         vtk_window->Render();
         auto vtk_tex = vtk_window->GetDisplayFramebuffer()
@@ -145,6 +149,9 @@ struct augment_renderer::impl {
 
 };
 
+vtkSmartPointer<vtkRenderer> augment_renderer::impl::vtk_render;
+vtkSmartPointer<vtkGenericOpenGLRenderWindow> augment_renderer::impl::vtk_window;
+
 augment_renderer::augment_renderer()
         : pimpl(std::make_unique<impl>()) {}
 
@@ -171,7 +178,7 @@ bool augment_renderer::render(const render_config &config) {
 void augment_renderer::add_scene(scene_manager *scene) {
     assert(pimpl->vtk_render != nullptr);
     for (const auto &actor: scene->actor_pool) {
-        pimpl->vtk_render->AddActor(actor.actor);
+        impl::vtk_render->AddActor(actor.actor);
     }
 }
 

src/augment_renderer.h

@@ -30,7 +30,7 @@ public:
 
     void set_background(const cv::cuda::GpuMat *background);
 
-    void add_scene(scene_manager *scene);
+    static void add_scene(scene_manager *scene);
 
     void set_camera_pose(const Eigen::Isometry3d &pose);
 

src/config.h

@@ -30,9 +30,11 @@ static constexpr auto default_spin_time = std::chrono::milliseconds(100); // 100
 static constexpr auto default_cuda_device_id = 0;
 static constexpr auto default_video_stream_bitrate = 10 * 1e6; // 10mbps
 
-static constexpr auto output_frame_width = 2582;
+static constexpr auto output_frame_width = 1920;
 static constexpr auto output_frame_height = 1080;
 
+static constexpr auto ar_width_normal = 1.0 * output_frame_height * image_width / output_frame_width / image_height;
+
 #define RET_ERROR \
     assert(false); \
     return false; \

src/main.cpp

@@ -17,6 +17,8 @@
 #include <imgui_impl_opengl3.h>
 
 #include <ExternalVTKWidget.h>
+#include <vtkNamedColors.h>
+#include <vtkProperty.h>
 
 #include <glad/gl.h>
 #include <GLFW/glfw3.h>
@@ -113,24 +115,28 @@ int main() {
     // config sophiar
     using namespace sophiar;
     variable_io var_io;
-//    auto femur_pose_id = var_io.add_variable("femur_in_tracker", VT_transform, VD_Input);
-//    auto tibia_pose_id = var_io.add_variable("tibia_in_tracker", VT_transform, VD_Input);
+    auto femur_pose_id = var_io.add_variable("femur_in_tracker", VT_transform, VD_Input);
+    auto tibia_pose_id = var_io.add_variable("tibia_in_tracker", VT_transform, VD_Input);
     auto left_cam_pose_id = var_io.add_variable("left_camera_in_tracker", VT_transform, VD_Input);
     auto right_cam_pose_id = var_io.add_variable("right_camera_in_tracker", VT_transform, VD_Input);
 
     // for debug; config sophiar
-    auto probe_pose_id = var_io.add_variable("probe_in_tracker", VT_transform, VD_Input);
+//    auto probe_pose_id = var_io.add_variable("probe_in_tracker", VT_transform, VD_Input);
     var_io.connect("127.0.0.1", 5278);
 
     auto vtk_external = vtkSmartPointer<ExternalVTKWidget>::New();
 
     scene_manager scene;
     scene.var_io = &var_io;
-//    scene.add_actor("femur.stl", femur_pose_id);
-//    scene.add_actor("tibia.stl", tibia_pose_id);
+    auto femur_actor = scene.add_actor("femur.stl", femur_pose_id);
+    auto tibia_actor = scene.add_actor("tibia.stl", tibia_pose_id);
+
+    vtkNew<vtkNamedColors> colors;
+    femur_actor->GetProperty()->SetColor(colors->GetColor4d("orange").GetData());
+    tibia_actor->GetProperty()->SetColor(colors->GetColor4d("orangered").GetData());
 
     // for debug;
-    scene.add_actor("/home/tpx/data/stls/GlassProbe_4Ball_3.STL", probe_pose_id);
+//    scene.add_actor("/home/tpx/data/stls/GlassProbe_4Ball_3.STL", probe_pose_id);
 
     augment_renderer left_ar, right_ar;
     auto left_remap_file = mapped_file{"left_proj.dat", boost::iostreams::mapped_file_base::readonly};
@@ -147,8 +153,7 @@ int main() {
     right_ar.initialize(&tex_renderer, &right_cam_info);
     left_ar.set_background(&camera.left_rgb_image);
     right_ar.set_background(&camera.right_rgb_image);
-    left_ar.add_scene(&scene);
-    right_ar.add_scene(&scene);
+    augment_renderer::add_scene(&scene);
     left_remap_file.close();
     right_remap_file.close();
 
@@ -306,7 +311,7 @@ int main() {
 
                 ImGui::BeginDisabled();
                 var_io.show_input_variable(left_cam_pose_id);
-                var_io.show_input_variable(probe_pose_id);
+                var_io.show_input_variable(femur_pose_id);
                 ImGui::EndDisabled();
 
                 ImGui::PopID();
@@ -342,8 +347,14 @@ int main() {
             // draw frame for streaming
             glBindFramebuffer(GL_DRAW_FRAMEBUFFER, output_fbo.fbo);
             glViewport(0, 0, output_fbo.tex_width, output_fbo.tex_height);
-            left_ar.render({-1, -1, 1, 2});
-            right_ar.render({0, -1, 1, 2});
+            left_ar.render({-0.5 - ar_width_normal / 2, -1, ar_width_normal, 2});
+            right_ar.render({0.5 - ar_width_normal / 2, -1, ar_width_normal, 2});
+
+            // send IDR frame occasionally to prevent broken video frame
+            static int last_p_count = 0;
+            if (last_p_count > camera_fps * 5) {
+                idr_flag.test_and_set();
+            }
 
             // encode frame
             output_fbo.download_pixels();
@@ -353,6 +364,7 @@ int main() {
                 SPDLOG_INFO("IDR frame requested.");
                 encoder.refresh();
                 idr_flag.clear();
+                last_p_count = 0;
             }
             encoder.encode_frame(output_fbo.pbo_res, (void **) &info.data, &info.length);
             SPDLOG_TRACE("Time used: {}ms, length = {}", std::chrono::duration_cast<std::chrono::milliseconds>(

src/scene_manager.hpp

@@ -35,7 +35,7 @@ struct scene_manager {
         matrix->Modified();
     }
 
-    void add_actor(std::string_view file_name, int variable_id) {
+    vtkActor *add_actor(std::string_view file_name, int variable_id) {
         vtkNew<vtkSTLReader> reader;
         reader->SetFileName(file_name.data());
         reader->Update();
@@ -50,6 +50,8 @@ struct scene_manager {
         next_actor.actor = actor;
         next_actor.pose = pose_matrix;
         next_actor.pose_variable_id = variable_id;
+
+        return actor;
     }
 
     void update_pose() {