Integrate TP-LSD (cvg#27)

* Remove official TP-LSD submodule * Added TP-LSD line detector * Minor fix in ReadMe * add M6D and M8D definitions in limap. * include tp_lsd into requirements.txt. * minor update on readme. * Install TP-LSD in dev mode Co-authored-by: B1ueber2y <[email protected]>
zhangshaos · Nov 29, 2022 · 28b87cf · 28b87cf
1 parent a24f808
commit 28b87cf
Show file tree

Hide file tree

Showing 12 changed files with 100 additions and 13 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -26,4 +26,7 @@
 [submodule "third-party/hawp"]
 	path = third-party/hawp
 	url = [email protected]:cherubicXN/hawp.git
-	ignore = dirty
+	ignore = dirty
+[submodule "third-party/TP-LSD"]
+	path = third-party/TP-LSD
+	url = [email protected]:rpautrat/TP-LSD.git
diff --git a/README.md b/README.md
@@ -49,3 +49,23 @@ python runners/hypersim/triangulation.py --sfm.fbase sift --line2d.detector.meth
 ```
 In particular, ``skip_exists`` is a very useful option to avoid running point-based SfM and line detection/description repeatedly in each pass.
 
+## Supported line detectors and descriptors
+
+The following line detectors are currently supported:
+- [LSD](https://github.com/iago-suarez/pytlsd)
+- [SOLD2](https://github.com/cvg/SOLD2)
+- [HAWPv3](https://github.com/cherubicXN/hawp)
+
+The line detector [TP-LSD](https://github.com/Siyuada7/TP-LSD) can be additionally used, but needs a separate installation. You will need to switch to GCC 7 for the following compilation (but can use again GCC 9 at test time):
+```bash
+python -m pip install -e ./third-party/TP-LSD/tp_lsd/modeling/DCNv2
+python -m pip install -e ./third-party/TP-LSD
+```
+
+The following line descriptors/matchers are currently supported:
+- [LBD](https://github.com/iago-suarez/pytlbd)
+- [SOLD2](https://github.com/cvg/SOLD2)
+- [LineTR](https://github.com/yosungho/LineTR)
+- [L2D2](https://github.com/hichem-abdellali/L2D2)
+- Nearest neighbor matching of the endpoints with [SuperPoint](https://github.com/magicleap/SuperPointPretrainedNetwork)
+- Matching of the endpoints with [SuperPoint](https://github.com/magicleap/SuperPointPretrainedNetwork) + [SuperGlue](https://github.com/magicleap/SuperGluePretrainedNetwork)
diff --git a/cfgs/fitnmerge/default.yaml b/cfgs/fitnmerge/default.yaml
@@ -35,7 +35,7 @@ line2d:
     max_num_2d_segs: 3000
     do_merge_lines: False
     detector: 
-        method: "lsd" # ["lsd", "sold2", "hawpv3"]
+        method: "lsd" # ["lsd", "sold2", "hawpv3", "tp_lsd"]
         skip_exists: False
     visualize: False
     save_l3dpp: False
@@ -44,6 +44,7 @@ var2d: # in pixels
     sold2: 5.0
     lsd: 2.0
     hawpv3: 5.0
+    tp_lsd: 5.0
 
 ##############################
 # fitting config

diff --git a/cfgs/triangulation/default.yaml b/cfgs/triangulation/default.yaml
@@ -42,7 +42,7 @@ line2d:
     save_l3dpp: False
     compute_descinfo: False
     detector: 
-        method: "lsd" # ["lsd", "sold2", "hawpv3"]
+        method: "lsd" # ["lsd", "sold2", "hawpv3", "tp_lsd"]
         skip_exists: False
     extractor: 
         method: "sold2" # ["sold2", "lbd", "l2d2", "linetr", "superpoint_endpoints"]
@@ -58,6 +58,7 @@ var2d: # in pixels
     sold2: 5.0
     lsd: 2.0
     hawpv3: 5.0
+    tp_lsd: 5.0
 
 ##############################
 # triangulation config

diff --git a/limap/line2d/TP_LSD/__init__.py b/limap/line2d/TP_LSD/__init__.py
@@ -0,0 +1 @@
+from .tp_lsd import TPLSDDetector
diff --git a/limap/line2d/TP_LSD/tp_lsd.py b/limap/line2d/TP_LSD/tp_lsd.py
@@ -0,0 +1,57 @@
+import os, sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from base_detector import BaseDetector, BaseDetectorOptions
+
+import cv2
+import numpy as np
+import torch
+from tp_lsd.utils.reconstruct import TPS_line
+from tp_lsd.utils.utils import load_model
+from tp_lsd.modeling.TP_Net import Res320
+
+
+class TPLSDDetector(BaseDetector):
+    def __init__(self, options = BaseDetectorOptions()):
+        super(TPLSDDetector, self).__init__(options)
+        # Load the TP-LSD model
+        head = {'center': 1, 'dis': 4, 'line': 1}
+        ckpt = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))),
+                            'third-party/TP-LSD/pretraineds/Res512.pth')
+        self.net = load_model(Res320(head), ckpt)
+        self.net = self.net.cuda().eval()
+
+    def get_module_name(self):
+        return "tp_lsd"
+
+    def detect(self, camview):
+        img = camview.read_image(set_gray=False)
+        segs = self.detect_tplsd(img, self.net)
+        return segs
+
+    def detect_tplsd(self, img, net):
+        H, W = img.shape[:2]
+        hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
+        imgv0 = hsv[..., 2]
+        imgv = cv2.resize(imgv0, (0, 0), fx=1. / 4, fy=1. / 4, interpolation=cv2.INTER_LINEAR)
+        imgv = cv2.GaussianBlur(imgv, (5, 5), 3)
+        imgv = cv2.resize(imgv, (W, H), interpolation=cv2.INTER_LINEAR)
+        imgv = cv2.GaussianBlur(imgv, (5, 5), 3)
+
+        imgv1 = imgv0.astype(np.float32) - imgv + 127.5
+        imgv1 = np.clip(imgv1, 0, 255).astype(np.uint8)
+        hsv[..., 2] = imgv1
+        inp = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
+
+        inp = (inp.astype(np.float32) / 255.)
+        inp = torch.from_numpy(inp.transpose(2, 0, 1)).unsqueeze(0).cuda()
+        with torch.no_grad():
+            outputs = net(inp)
+        lines = TPS_line(outputs[-1], 0.25, 0.5, H, W)[0].reshape(-1, 2, 2)
+
+        # Use the line length as score
+        lines = np.concatenate([
+            lines.reshape(-1, 4),
+            np.linalg.norm(lines[:, 0] - lines[:, 1], axis=1, keepdims=True)],
+                               axis=1)
+
+        return lines
diff --git a/limap/line2d/register_detector.py b/limap/line2d/register_detector.py
@@ -14,6 +14,9 @@ def get_detector(cfg_detector, max_num_2d_segs=3000, do_merge_lines=False):
     elif method == "hawpv3":
         from .HAWPv3 import HAWPv3Detector
         return HAWPv3Detector(options)
+    elif method == "tp_lsd":
+        from .TP_LSD import TPLSDDetector
+        return TPLSDDetector(options)
     else:
         raise NotImplementedError
 

diff --git a/limap/triangulation/functions.cc b/limap/triangulation/functions.cc
@@ -215,14 +215,10 @@ std::pair<Line3d, bool> line_triangulation(const Line2d& l1, const CameraView& v
     return std::make_pair(line, true);
 }
 
-Eigen::Matrix6d line_triangulation_covariance(const Line2d& l1, const CameraView& view1,
-                                              const Line2d& l2, const CameraView& view2,
-                                              const Eigen::MatrixXd& covariance) 
+M6D line_triangulation_covariance(const Line2d& l1, const CameraView& view1,
+                                  const Line2d& l2, const CameraView& view2,
+                                  const M8D& covariance) 
 {
-    // check: input covariance should be 8 x 8
-    THROW_CHECK_EQ(covariance.rows(), 8);
-    THROW_CHECK_EQ(covariance.cols(), 8);
-
     // compute matrix form again
     V3D c1_start = view1.ray_direction(l1.start);
     V3D c1_end = view1.ray_direction(l1.end);

diff --git a/limap/triangulation/functions.h b/limap/triangulation/functions.h
@@ -42,9 +42,9 @@ Line3d triangulate_endpoints(const Line2d& l1, const CameraView& view1,
 std::pair<Line3d, bool> line_triangulation(const Line2d& l1, const CameraView& view1,
                                            const Line2d& l2, const CameraView& view2);
 
-Eigen::Matrix6d line_triangulation_covariance(const Line2d& l1, const CameraView& view1,
-                                              const Line2d& l2, const CameraView& view2,
-                                              const Eigen::MatrixXd& covariance);
+M6D line_triangulation_covariance(const Line2d& l1, const CameraView& view1,
+                                  const Line2d& l2, const CameraView& view2,
+                                  const M8D& covariance);
 
 // Asymmetric perspective to (view1, l1)
 // Algebraic line triangulation

diff --git a/limap/util/types.h b/limap/util/types.h
@@ -27,6 +27,8 @@ using M4F = Eigen::Matrix4f;
 using M2D = Eigen::Matrix2d;
 using M3D = Eigen::Matrix3d;
 using M4D = Eigen::Matrix4d;
+using M6D = Eigen::Matrix<double, 6, 6>;
+using M8D = Eigen::Matrix<double, 8, 8>;
 
 const double EPS = 1e-12;
 

diff --git a/requirements.txt b/requirements.txt
@@ -33,4 +33,6 @@ python-json-logger
 ./third-party/pytlbd
 ./third-party/Hierarchical-Localization
 ./third-party/hawp
+-e ./third-party/TP-LSD/tp_lsd/modeling/DCNv2
+-e ./third-party/TP-LSD
 
diff --git a/third-party/TP-LSD b/third-party/TP-LSD