Implemented distort augmenting.

CMakeLists.txt

@@ -78,7 +78,7 @@ target_link_libraries(${PROJECT_NAME} ${MVS_LIB})
 target_sources(${PROJECT_NAME} PRIVATE src/mvs_camera.cpp)
 
 # Boost config
-find_package(Boost REQUIRED)
+find_package(Boost REQUIRED COMPONENTS iostreams)
 target_include_directories(${PROJECT_NAME} PRIVATE ${Boost_INCLUDE_DIRS})
 target_link_libraries(${PROJECT_NAME} ${Boost_LIBRARIES})
 

src/augment_renderer.cpp

@@ -14,7 +14,7 @@
 
 struct augment_renderer::impl {
     const cv::cuda::GpuMat *bg_img = nullptr;
-    GLuint bg_tex = 0, bg_pbo = 0;
+    GLuint bg_tex = 0, remap_tex = 0, bg_pbo = 0;
     cudaGraphicsResource *bg_res = nullptr;
 
     texture_renderer *tex_renderer = nullptr;
@@ -39,27 +39,35 @@ struct augment_renderer::impl {
         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
 
         // generate and allocate texture
-        glGenTextures(1, &bg_tex);
+        static_assert(offsetof(impl, remap_tex) - offsetof(impl, bg_tex) == sizeof(GLuint));
+        glGenTextures(2, &bg_tex);
         glBindTexture(GL_TEXTURE_2D, bg_tex);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
         glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, image_width, image_height);
 
+        // config remap texture
+        glBindTexture(GL_TEXTURE_2D, remap_tex);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+        glTexImage2D(GL_TEXTURE_2D, 0, GL_RG32F, image_width, image_height, 0, GL_RG, GL_FLOAT, cam_info->remap_data);
+
         // register background pbo
         CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(&bg_res, bg_pbo,
                                                     cudaGraphicsRegisterFlagsWriteDiscard));
 
         // setup vtk
         vtk_render = vtkSmartPointer<vtkRenderer>::New();
-//        vtk_render->SetBackground(0.4, 0.4, 0.4);
-//        vtk_render->SetBackgroundAlpha(0.1);
         vtk_window = vtkSmartPointer<vtkGenericOpenGLRenderWindow>::New();
         vtk_window->InitializeFromCurrentContext();
-        vtk_window->SetSize(image_width, image_height);
+        vtk_window->SetSize(cam_info->width, cam_info->height);
         vtk_window->SetAlphaBitPlanes(true);
         vtk_window->SetOffScreenRendering(true);
+        vtk_window->FramebufferFlipYOn();
         vtk_window->AddRenderer(vtk_render);
         vtk_cam = vtkSmartPointer<vtkCamera>::New();
+        vtk_cam->SetViewAngle(cam_info->angle);
+        vtk_cam->SetClippingRange(default_len_focus, 2000);
         vtk_render->SetActiveCamera(vtk_cam);
 
         return true;
@@ -88,7 +96,8 @@ struct augment_renderer::impl {
         assert(tex_renderer != nullptr);
         assert(config != nullptr);
         texture_renderer::render_config tex_config{
-                .tex = bg_tex,
+                .image_tex = bg_tex,
+                .remap_tex = 0,
                 .x = config->x,
                 .y = config->y,
                 .width = config->width,
@@ -101,14 +110,11 @@ struct augment_renderer::impl {
 
     void set_camera_pose(const Eigen::Isometry3d &cam_trans) {
         auto trans_part = cam_trans.translation();
-        auto focal_point = trans_part + cam_trans.rotation().col(2) * cam_info->focal_length;
-        auto view_up = cam_trans.rotation().col(1);
+        auto focal_point = trans_part + cam_trans.rotation().col(2) * default_len_focus;
+        auto view_up = -cam_trans.rotation().col(1);
         vtk_cam->SetPosition(trans_part.x(), trans_part.y(), trans_part.z());
         vtk_cam->SetFocalPoint(focal_point.x(), focal_point.y(), focal_point.z());
         vtk_cam->SetViewUp(view_up.x(), view_up.y(), view_up.z());
-        vtk_cam->SetWindowCenter(cam_info->window_center.x, cam_info->window_center.y);
-        vtk_cam->SetViewAngle(cam_info->view_angle);
-        vtk_cam->SetClippingRange(cam_info->focal_length, 2000);
         vtk_cam->Modified();
         vtk_render->Modified();
     }
@@ -119,10 +125,11 @@ struct augment_renderer::impl {
         auto vtk_tex = vtk_window->GetDisplayFramebuffer()
                 ->GetColorAttachmentAsTextureObject(0)->GetHandle();
         texture_renderer::render_config tex_config{
-                .tex = vtk_tex,
-                .x = config->x + config->width,
+                .image_tex = vtk_tex,
+                .remap_tex = remap_tex,
+                .x = config->x,
                 .y = config->y,
-                .width = -config->width,
+                .width = config->width,
                 .height = config->height
         };
         tex_renderer->render(&tex_config);

src/augment_renderer.h

@@ -20,15 +20,9 @@ public:
     ~augment_renderer();
 
     struct camera_intrinsics {
-        float focal_length;
-        float view_angle;
-        float pixel_size;
-        struct {
-            float x, y;
-        } window_center;
-        struct {
-            float k1, k2;
-        } distort;
+        int width, height;
+        float angle;
+        void *remap_data;
     };
 
     bool initialize(texture_renderer *renderer, const camera_intrinsics *camera);

src/config.h

@@ -18,6 +18,8 @@ static constexpr auto rgb_image_pitch = image_width * 3;
 static constexpr auto left_camera_name = "LeftEye";
 static constexpr auto right_camera_name = "RightEye";
 
+static constexpr auto default_len_focus = 8; // 8mm, maybe not useful
+
 static constexpr auto default_camera_exposure_time_ms = 5; // 5ms
 static constexpr auto default_camera_analog_gain = 20; // 20dB
 static constexpr auto default_camera_fps = 30; // 30 fps

src/main.cpp

@@ -8,6 +8,8 @@
 
 #include "third_party/scope_guard.hpp"
 
+#include <boost/iostreams/device/mapped_file.hpp>
+
 #include <imgui.h>
 #include <imgui_impl_glfw.h>
 #include <imgui_impl_opengl3.h>
@@ -50,11 +52,15 @@ void transform_to_vtk_matrix(const Eigen::Isometry3d &trans, vtkMatrix4x4 *matri
     matrix->Modified();
 }
 
+using boost::iostreams::mapped_file;
+
 CUcontext cuda_ctx;
 
 int main() {
 
+#ifndef NDEBUG
     spdlog::set_level(spdlog::level::trace);
+#endif
 
     // setup glfw and main window
     glfwSetErrorCallback([](int error, const char *desc) {
@@ -106,10 +112,10 @@ int main() {
     create_cuda_context(&cuda_ctx);
 
     // for debug; setup NDI
-    auto ref_cam_trans_left = Eigen::Translation3d(0.1829, -38.3006, -34.2251) *
-                              Eigen::Quaterniond(0.3623, 0.6095, 0.3499, -0.6122);
-    auto ref_cam_trans_right = Eigen::Translation3d(1.0799, -39.7982, -94.8817) *
-                               Eigen::Quaterniond(0.3669, 0.6074, 0.3504, -0.6113);
+    auto ref_cam_trans_left = Eigen::Translation3d(-11.3375, 21.2730, -37.5694) *
+                              Eigen::Quaterniond(0.0009, -0.7088, -0.0091, 0.7053);
+    auto ref_cam_trans_right = Eigen::Translation3d(-11.2100, 22.1807, -97.3032) *
+                               Eigen::Quaterniond(0.0011, 0.7066, 0.0120, -0.7075);
 
     CombinedApi capi;
     ret = capi.connect("10.0.0.5");
@@ -142,48 +148,33 @@ int main() {
     video_encoder encoder;
     encoder.initialize();
 
+    auto vtk_external = vtkSmartPointer<ExternalVTKWidget>::New();
+
+    augment_renderer left_ar, right_ar;
+    auto left_remap_file = mapped_file{"left_proj.dat", boost::iostreams::mapped_file_base::readonly};
+    auto right_remap_file = mapped_file{"right_proj.dat", boost::iostreams::mapped_file_base::readonly};
     auto left_cam_info = augment_renderer::camera_intrinsics{
-            .focal_length= 8,
-            .view_angle = 47.6525,
-            .pixel_size = 0.00345,
-            .window_center = {
-                    .x = 1.0 * (1238.96436323518 - 1224) / 1224,
-                    .y =  -1.0 * (1011.68988239492 - 1024) / 1024
-            },
-            .distort = {
-                    .k1 = -0.0716117335673421,
-                    .k2 = 0.0899896700761875
-            }
+            .width = 2463, .height = 2048, .angle = 47.53009170028617,
+            .remap_data = (void *) left_remap_file.const_data()
     };
     auto right_cam_info = augment_renderer::camera_intrinsics{
-            .focal_length= 8,
-            .view_angle = 47.6525,
-            .pixel_size = 0.00345,
-            .window_center = {
-                    .x = 1.0 * (1238.97798867909 - 1224) / 1224,
-                    .y =  -1.0 * (1000.55305718163 - 1024) / 1024
-            },
-            .distort = {
-                    .k1 = -0.0716625513187931,
-                    .k2 = 0.0979453312950340
-            }
+            .width = 2427, .height = 2048, .angle = 48.0713681473623,
+            .remap_data = (void *) right_remap_file.const_data()
     };
-
-    auto vtk_external = vtkSmartPointer<ExternalVTKWidget>::New();
-
-    augment_renderer left_ar, right_ar;
     left_ar.initialize(&tex_renderer, &left_cam_info);
     right_ar.initialize(&tex_renderer, &right_cam_info);
     left_ar.set_background(&camera.left_rgb_image);
     right_ar.set_background(&camera.right_rgb_image);
+    left_remap_file.close();
+    right_remap_file.close();
 
     // for debug; setup VTK scene
-    vtkNew<vtkSTLReader> reader;
+    vtkNew <vtkSTLReader> reader;
     reader->SetFileName("/home/tpx/data/stls/GlassProbe_4Ball_3.STL");
-    vtkNew<vtkPolyDataMapper> mapper;
+    vtkNew <vtkPolyDataMapper> mapper;
     mapper->SetInputConnection(reader->GetOutputPort());
-    vtkNew<vtkActor> actor;
-    vtkNew<vtkMatrix4x4> tool_matrix;
+    vtkNew <vtkActor> actor;
+    vtkNew <vtkMatrix4x4> tool_matrix;
     actor->SetUserMatrix(tool_matrix);
     actor->SetMapper(mapper);
     left_ar.add_model(actor);
@@ -402,7 +393,7 @@ int main() {
 
         if (camera.is_capturing()) {
             // draw frame in the screen
-            left_ar.render({-1, 1, 2, -2});
+            right_ar.render({-1, 1, 2, -2});
         }
 
         ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());

src/texture_renderer.cpp

@@ -26,6 +26,18 @@ static constexpr auto fragment_shader_source = R"(
         }
     )";
 
+static constexpr auto fragment_shader_ext_source = R"(
+        #version 460
+        layout (location = 0) out vec4 color_out;
+        in vec2 tex_coord;
+        uniform sampler2D image_tex;
+        uniform sampler2D remap_tex;
+        void main() {
+            vec2 tex_coord_real = texture(remap_tex, tex_coord).xy;
+            color_out = texture(image_tex, tex_coord_real);
+        }
+    )";
+
 static constexpr GLuint indices[] = {
         0, 1, 3, // first triangle
         1, 2, 3 // second triangle
@@ -35,21 +47,38 @@ struct texture_renderer::impl {
 
     GLuint vertex_array = 0;
     GLuint vertex_buffer = 0, element_buffer = 0;
-    GLuint program = 0;
+    GLuint program = 0, program_ext = 0;
+
+    GLint image_tex_loc = 0, remap_tex_loc = 0;
 
     impl() {
         // build program
         auto vertex_shader = glCreateShader(GL_VERTEX_SHADER);
         auto fragment_shader = glCreateShader(GL_FRAGMENT_SHADER);
+        auto fragment_shader_ext = glCreateShader(GL_FRAGMENT_SHADER);
         compile_shader(vertex_shader, vertex_shader_source, "vertex");
         compile_shader(fragment_shader, fragment_shader_source, "fragment");
+        compile_shader(fragment_shader_ext, fragment_shader_ext_source, "fragment_ext");
+
         program = glCreateProgram();
         glAttachShader(program, vertex_shader);
         glAttachShader(program, fragment_shader);
         glLinkProgram(program);
         check_program();
+
+        program_ext = glCreateProgram();
+        glAttachShader(program_ext, vertex_shader);
+        glAttachShader(program_ext, fragment_shader_ext);
+        glLinkProgram(program_ext);
+        check_program();
+
         glDeleteShader(vertex_shader);
         glDeleteShader(fragment_shader);
+        glDeleteShader(fragment_shader_ext);
+
+        // uniform locations
+        image_tex_loc = glGetUniformLocation(program_ext, "image_tex");
+        remap_tex_loc = glGetUniformLocation(program_ext, "remap_tex");
 
         // create buffers
         static_assert(offsetof(impl, element_buffer) - offsetof(impl, vertex_buffer) == sizeof(GLuint));
@@ -109,16 +138,28 @@ struct texture_renderer::impl {
         free(info_log);
     }
 
-    void render(const texture_renderer::render_config *config) {
+    void render(const render_config *config) {
         auto x = config->x, y = config->y;
         auto width = config->width, height = config->height;
+        auto is_ext = config->remap_tex != 0;
 
         // bindings
-        glUseProgram(program);
+        glUseProgram(is_ext ? program_ext : program);
         glBindVertexArray(vertex_array);
         glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer);
         glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, element_buffer);
-        glBindTexture(GL_TEXTURE_2D, config->tex);
+
+        if (is_ext) {
+            glUniform1i(image_tex_loc, 0);
+            glUniform1i(remap_tex_loc, 1);
+            glActiveTexture(GL_TEXTURE0 + 0);
+            glBindTexture(GL_TEXTURE_2D, config->image_tex);
+            glActiveTexture(GL_TEXTURE0 + 1);
+            glBindTexture(GL_TEXTURE_2D, config->remap_tex);
+        } else {
+            glActiveTexture(GL_TEXTURE0 + 0);
+            glBindTexture(GL_TEXTURE_2D, config->image_tex);
+        }
 
         // fill vertex buffer
         GLfloat vertices[] = {

src/texture_renderer.h

@@ -12,7 +12,7 @@ public:
     ~texture_renderer();
 
     struct render_config {
-        GLuint tex;
+        GLuint image_tex, remap_tex;
         GLfloat x, y;
         GLfloat width, height;
     };