From 28b87cf555cf490c126c35acc484f69acc6a9821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pautrat?= <32239569+rpautrat@users.noreply.github.com> Date: Tue, 29 Nov 2022 13:14:05 +0100 Subject: [PATCH] Integrate TP-LSD (#27) * Remove official TP-LSD submodule * Added TP-LSD line detector * Minor fix in ReadMe * add M6D and M8D definitions in limap. * include tp_lsd into requirements.txt. * minor update on readme. * Install TP-LSD in dev mode Co-authored-by: B1ueber2y --- .gitmodules | 5 ++- README.md | 20 +++++++++++ cfgs/fitnmerge/default.yaml | 3 +- cfgs/triangulation/default.yaml | 3 +- limap/line2d/TP_LSD/__init__.py | 1 + limap/line2d/TP_LSD/tp_lsd.py | 57 +++++++++++++++++++++++++++++++ limap/line2d/register_detector.py | 3 ++ limap/triangulation/functions.cc | 10 ++---- limap/triangulation/functions.h | 6 ++-- limap/util/types.h | 2 ++ requirements.txt | 2 ++ third-party/TP-LSD | 1 + 12 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 limap/line2d/TP_LSD/__init__.py create mode 100644 limap/line2d/TP_LSD/tp_lsd.py create mode 160000 third-party/TP-LSD diff --git a/.gitmodules b/.gitmodules index 8d78f6cd..f68dd2d1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -26,4 +26,7 @@ [submodule "third-party/hawp"] path = third-party/hawp url = git@github.com:cherubicXN/hawp.git - ignore = dirty \ No newline at end of file + ignore = dirty +[submodule "third-party/TP-LSD"] + path = third-party/TP-LSD + url = git@github.com:rpautrat/TP-LSD.git diff --git a/README.md b/README.md index 442b1e93..8c388dfc 100644 --- a/README.md +++ b/README.md @@ -49,3 +49,23 @@ python runners/hypersim/triangulation.py --sfm.fbase sift --line2d.detector.meth ``` In particular, ``skip_exists`` is a very useful option to avoid running point-based SfM and line detection/description repeatedly in each pass. +## Supported line detectors and descriptors + +The following line detectors are currently supported: +- [LSD](https://github.com/iago-suarez/pytlsd) +- [SOLD2](https://github.com/cvg/SOLD2) +- [HAWPv3](https://github.com/cherubicXN/hawp) + +The line detector [TP-LSD](https://github.com/Siyuada7/TP-LSD) can be additionally used, but needs a separate installation. You will need to switch to GCC 7 for the following compilation (but can use again GCC 9 at test time): +```bash +python -m pip install -e ./third-party/TP-LSD/tp_lsd/modeling/DCNv2 +python -m pip install -e ./third-party/TP-LSD +``` + +The following line descriptors/matchers are currently supported: +- [LBD](https://github.com/iago-suarez/pytlbd) +- [SOLD2](https://github.com/cvg/SOLD2) +- [LineTR](https://github.com/yosungho/LineTR) +- [L2D2](https://github.com/hichem-abdellali/L2D2) +- Nearest neighbor matching of the endpoints with [SuperPoint](https://github.com/magicleap/SuperPointPretrainedNetwork) +- Matching of the endpoints with [SuperPoint](https://github.com/magicleap/SuperPointPretrainedNetwork) + [SuperGlue](https://github.com/magicleap/SuperGluePretrainedNetwork) diff --git a/cfgs/fitnmerge/default.yaml b/cfgs/fitnmerge/default.yaml index 565f93ca..59cee9e1 100644 --- a/cfgs/fitnmerge/default.yaml +++ b/cfgs/fitnmerge/default.yaml @@ -35,7 +35,7 @@ line2d: max_num_2d_segs: 3000 do_merge_lines: False detector: - method: "lsd" # ["lsd", "sold2", "hawpv3"] + method: "lsd" # ["lsd", "sold2", "hawpv3", "tp_lsd"] skip_exists: False visualize: False save_l3dpp: False @@ -44,6 +44,7 @@ var2d: # in pixels sold2: 5.0 lsd: 2.0 hawpv3: 5.0 + tp_lsd: 5.0 ############################## # fitting config diff --git a/cfgs/triangulation/default.yaml b/cfgs/triangulation/default.yaml index baa7e6cf..4a51f184 100644 --- a/cfgs/triangulation/default.yaml +++ b/cfgs/triangulation/default.yaml @@ -42,7 +42,7 @@ line2d: save_l3dpp: False compute_descinfo: False detector: - method: "lsd" # ["lsd", "sold2", "hawpv3"] + method: "lsd" # ["lsd", "sold2", "hawpv3", "tp_lsd"] skip_exists: False extractor: method: "sold2" # ["sold2", "lbd", "l2d2", "linetr", "superpoint_endpoints"] @@ -58,6 +58,7 @@ var2d: # in pixels sold2: 5.0 lsd: 2.0 hawpv3: 5.0 + tp_lsd: 5.0 ############################## # triangulation config diff --git a/limap/line2d/TP_LSD/__init__.py b/limap/line2d/TP_LSD/__init__.py new file mode 100644 index 00000000..1216db7b --- /dev/null +++ b/limap/line2d/TP_LSD/__init__.py @@ -0,0 +1 @@ +from .tp_lsd import TPLSDDetector diff --git a/limap/line2d/TP_LSD/tp_lsd.py b/limap/line2d/TP_LSD/tp_lsd.py new file mode 100644 index 00000000..224c48dc --- /dev/null +++ b/limap/line2d/TP_LSD/tp_lsd.py @@ -0,0 +1,57 @@ +import os, sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from base_detector import BaseDetector, BaseDetectorOptions + +import cv2 +import numpy as np +import torch +from tp_lsd.utils.reconstruct import TPS_line +from tp_lsd.utils.utils import load_model +from tp_lsd.modeling.TP_Net import Res320 + + +class TPLSDDetector(BaseDetector): + def __init__(self, options = BaseDetectorOptions()): + super(TPLSDDetector, self).__init__(options) + # Load the TP-LSD model + head = {'center': 1, 'dis': 4, 'line': 1} + ckpt = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), + 'third-party/TP-LSD/pretraineds/Res512.pth') + self.net = load_model(Res320(head), ckpt) + self.net = self.net.cuda().eval() + + def get_module_name(self): + return "tp_lsd" + + def detect(self, camview): + img = camview.read_image(set_gray=False) + segs = self.detect_tplsd(img, self.net) + return segs + + def detect_tplsd(self, img, net): + H, W = img.shape[:2] + hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV) + imgv0 = hsv[..., 2] + imgv = cv2.resize(imgv0, (0, 0), fx=1. / 4, fy=1. / 4, interpolation=cv2.INTER_LINEAR) + imgv = cv2.GaussianBlur(imgv, (5, 5), 3) + imgv = cv2.resize(imgv, (W, H), interpolation=cv2.INTER_LINEAR) + imgv = cv2.GaussianBlur(imgv, (5, 5), 3) + + imgv1 = imgv0.astype(np.float32) - imgv + 127.5 + imgv1 = np.clip(imgv1, 0, 255).astype(np.uint8) + hsv[..., 2] = imgv1 + inp = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) + + inp = (inp.astype(np.float32) / 255.) + inp = torch.from_numpy(inp.transpose(2, 0, 1)).unsqueeze(0).cuda() + with torch.no_grad(): + outputs = net(inp) + lines = TPS_line(outputs[-1], 0.25, 0.5, H, W)[0].reshape(-1, 2, 2) + + # Use the line length as score + lines = np.concatenate([ + lines.reshape(-1, 4), + np.linalg.norm(lines[:, 0] - lines[:, 1], axis=1, keepdims=True)], + axis=1) + + return lines diff --git a/limap/line2d/register_detector.py b/limap/line2d/register_detector.py index e475b9cf..872ae1d6 100644 --- a/limap/line2d/register_detector.py +++ b/limap/line2d/register_detector.py @@ -14,6 +14,9 @@ def get_detector(cfg_detector, max_num_2d_segs=3000, do_merge_lines=False): elif method == "hawpv3": from .HAWPv3 import HAWPv3Detector return HAWPv3Detector(options) + elif method == "tp_lsd": + from .TP_LSD import TPLSDDetector + return TPLSDDetector(options) else: raise NotImplementedError diff --git a/limap/triangulation/functions.cc b/limap/triangulation/functions.cc index 4c56973f..875da341 100644 --- a/limap/triangulation/functions.cc +++ b/limap/triangulation/functions.cc @@ -215,14 +215,10 @@ std::pair line_triangulation(const Line2d& l1, const CameraView& v return std::make_pair(line, true); } -Eigen::Matrix6d line_triangulation_covariance(const Line2d& l1, const CameraView& view1, - const Line2d& l2, const CameraView& view2, - const Eigen::MatrixXd& covariance) +M6D line_triangulation_covariance(const Line2d& l1, const CameraView& view1, + const Line2d& l2, const CameraView& view2, + const M8D& covariance) { - // check: input covariance should be 8 x 8 - THROW_CHECK_EQ(covariance.rows(), 8); - THROW_CHECK_EQ(covariance.cols(), 8); - // compute matrix form again V3D c1_start = view1.ray_direction(l1.start); V3D c1_end = view1.ray_direction(l1.end); diff --git a/limap/triangulation/functions.h b/limap/triangulation/functions.h index daa11700..bffb366d 100644 --- a/limap/triangulation/functions.h +++ b/limap/triangulation/functions.h @@ -42,9 +42,9 @@ Line3d triangulate_endpoints(const Line2d& l1, const CameraView& view1, std::pair line_triangulation(const Line2d& l1, const CameraView& view1, const Line2d& l2, const CameraView& view2); -Eigen::Matrix6d line_triangulation_covariance(const Line2d& l1, const CameraView& view1, - const Line2d& l2, const CameraView& view2, - const Eigen::MatrixXd& covariance); +M6D line_triangulation_covariance(const Line2d& l1, const CameraView& view1, + const Line2d& l2, const CameraView& view2, + const M8D& covariance); // Asymmetric perspective to (view1, l1) // Algebraic line triangulation diff --git a/limap/util/types.h b/limap/util/types.h index a46d5f5e..c537448b 100644 --- a/limap/util/types.h +++ b/limap/util/types.h @@ -27,6 +27,8 @@ using M4F = Eigen::Matrix4f; using M2D = Eigen::Matrix2d; using M3D = Eigen::Matrix3d; using M4D = Eigen::Matrix4d; +using M6D = Eigen::Matrix; +using M8D = Eigen::Matrix; const double EPS = 1e-12; diff --git a/requirements.txt b/requirements.txt index 95d91e79..11c9aa47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,4 +33,6 @@ python-json-logger ./third-party/pytlbd ./third-party/Hierarchical-Localization ./third-party/hawp +-e ./third-party/TP-LSD/tp_lsd/modeling/DCNv2 +-e ./third-party/TP-LSD diff --git a/third-party/TP-LSD b/third-party/TP-LSD new file mode 160000 index 00000000..55580503 --- /dev/null +++ b/third-party/TP-LSD @@ -0,0 +1 @@ +Subproject commit 555805033e705ed6bfbf4d691528681bf4ae6706