From 7490bec8a129469eadd0b6096402aa4ac37c325f Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Sat, 19 Oct 2024 14:35:29 +0200 Subject: [PATCH 01/20] dense matcher based line matching --- limap/line2d/dense/__init__.py | 2 + limap/line2d/dense/dense_matcher/__init__.py | 1 + limap/line2d/dense/dense_matcher/base.py | 35 ++++++ limap/line2d/dense/dense_matcher/roma.py | 21 ++++ limap/line2d/dense/extractor.py | 40 +++++++ limap/line2d/dense/matcher.py | 119 +++++++++++++++++++ limap/line2d/register_detector.py | 4 +- limap/line2d/register_matcher.py | 4 +- 8 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 limap/line2d/dense/__init__.py create mode 100644 limap/line2d/dense/dense_matcher/__init__.py create mode 100644 limap/line2d/dense/dense_matcher/base.py create mode 100644 limap/line2d/dense/dense_matcher/roma.py create mode 100644 limap/line2d/dense/extractor.py create mode 100644 limap/line2d/dense/matcher.py diff --git a/limap/line2d/dense/__init__.py b/limap/line2d/dense/__init__.py new file mode 100644 index 00000000..b22b844f --- /dev/null +++ b/limap/line2d/dense/__init__.py @@ -0,0 +1,2 @@ +from .extractor import DenseNaiveExtractor +from .matcher import RoMaLineMatcher diff --git a/limap/line2d/dense/dense_matcher/__init__.py b/limap/line2d/dense/dense_matcher/__init__.py new file mode 100644 index 00000000..8e8108db --- /dev/null +++ b/limap/line2d/dense/dense_matcher/__init__.py @@ -0,0 +1 @@ +from .roma import RoMa diff --git a/limap/line2d/dense/dense_matcher/base.py b/limap/line2d/dense/dense_matcher/base.py new file mode 100644 index 00000000..568affb8 --- /dev/null +++ b/limap/line2d/dense/dense_matcher/base.py @@ -0,0 +1,35 @@ +import os +import torch + +class BaseDenseMatcher(object): + def __init__(self): + pass + + def to_normalized_coordinates(self, coords, h, w): + ''' + coords: (..., 2) in the order x, y + ''' + coords_x = 2 / w * coords[..., 0] - 1; + coords_y = 2 / h * coords[..., 1] - 1; + return torch.stack([coords_x, coords_y], axis=-1) + + def to_unnormalized_coordinates(self, coords, h, w): + ''' + Inverse operation of `to_normalized_coordinates` + ''' + coords_x = (coords[..., 0] + 1) * w / 2 + coords_y = (coords[..., 1] + 1) * h / 2 + return torch.stack([coords_x, coords_y], axis=-1) + + def get_sample_thresh(self): + ''' + return sample threshold + ''' + raise NotImplementedError + + def get_warpping_symmetric(self, img1, img2): + ''' + return warp_1to2 ([-1, 1]), cert_1to2, warp_2to1([-1, 1]), cert_2to1 + ''' + raise NotImplementedError + diff --git a/limap/line2d/dense/dense_matcher/roma.py b/limap/line2d/dense/dense_matcher/roma.py new file mode 100644 index 00000000..3b1be393 --- /dev/null +++ b/limap/line2d/dense/dense_matcher/roma.py @@ -0,0 +1,21 @@ +import os +from .base import BaseDenseMatcher +import romatch +from PIL import Image + +class RoMa(BaseDenseMatcher): + def __init__(self, mode = "outdoor", device="cuda"): + super(RoMa).__init__() + if mode == "outdoor": + self.model = romatch.roma_outdoor(device=device, coarse_res=560) + elif mode == "indoor": + self.model = romatch.roma_indoor(device=device, coarse_res=560) + + def get_sample_thresh(self): + return self.model.sample_thresh + + def get_warpping_symmetric(self, img1, img2): + warp, certainty = self.model.match(Image.fromarray(img1), Image.fromarray(img2)) + N = 864 + return warp[:,:N,2:], certainty[:,:N], warp[:,N:,:2], certainty[:,N:] + diff --git a/limap/line2d/dense/extractor.py b/limap/line2d/dense/extractor.py new file mode 100644 index 00000000..04595cbc --- /dev/null +++ b/limap/line2d/dense/extractor.py @@ -0,0 +1,40 @@ +import os +import numpy as np +import limap.util.io as limapio +from ..base_detector import BaseDetector, BaseDetectorOptions + + +class DenseNaiveExtractor(BaseDetector): + def __init__(self, options=BaseDetectorOptions(), device=None): + super(DenseNaiveExtractor, self).__init__(options) + + def get_module_name(self): + return "dense_naive" + + def get_descinfo_fname(self, descinfo_folder, img_id): + fname = os.path.join(descinfo_folder, "descinfo_{0}.npz".format(img_id)) + return fname + + def save_descinfo(self, descinfo_folder, img_id, descinfo): + limapio.check_makedirs(descinfo_folder) + fname = self.get_descinfo_fname(descinfo_folder, img_id) + limapio.save_npz(fname, descinfo) + + def read_descinfo(self, descinfo_folder, img_id): + fname = self.get_descinfo_fname(descinfo_folder, img_id) + descinfo = limapio.read_npz(fname) + return descinfo + + def extract(self, camview, segs): + img = camview.read_image(set_gray=self.set_gray) + lines = segs[:, :4].reshape(-1, 2, 2) + scores = segs[:, -1] * np.sqrt( + np.linalg.norm(segs[:, :2] - segs[:, 2:4], axis=1) + ) + descinfo = {"camview": camview, + "image_shape": img.shape, + "lines": lines, + "scores": scores, + } + return descinfo + diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py new file mode 100644 index 00000000..f257ea6e --- /dev/null +++ b/limap/line2d/dense/matcher.py @@ -0,0 +1,119 @@ +import os +import numpy as np +import torch +import torch.nn.functional as F +from typing import NamedTuple + +import limap.util.io as limapio +from ..base_matcher import BaseMatcher, BaseMatcherOptions + +class BaseDenseLineMatcherOptions(NamedTuple): + n_samples: int = 21 + segment_percentage_th: float = 0.5 + device = "cuda" + pixel_th: float = 10.0 + + +class BaseDenseLineMatcher(BaseMatcher): + def __init__(self, extractor, dense_matcher, dense_options=BaseDenseLineMatcherOptions(), options=BaseMatcherOptions()): + super(BaseDenseLineMatcher, self).__init__(extractor, options) + assert self.extractor.get_module_name() == "dense_naive" + self.dense_matcher = dense_matcher + self.dense_options = dense_options + assert self.dense_options.n_samples >= 2 + + def get_module_name(self): + raise NotImplementedError + + def match_pair(self, descinfo1, descinfo2): + if self.topk == 0: + return self.match_segs_with_descinfo(descinfo1, descinfo2) + else: + return self.match_segs_with_descinfo_topk( + descinfo1, descinfo2, topk=self.topk + ) + + def compute_distance_one_direction(self, descinfo1, descinfo2, warp_1to2, cert_1to2): + # get point samples along lines + segs1 = torch.from_numpy(descinfo1["lines"]).to(self.dense_options.device) + n_segs1 = segs1.shape[0] + ratio = torch.arange(0, 1 + 0.5 / (self.dense_options.n_samples - 1), 1.0 / (self.dense_options.n_samples - 1)).to(self.dense_options.device) + ratio = ratio[:,None].repeat(1, 2) + coords_1 = ratio * segs1[:,[0],:].repeat(1, self.dense_options.n_samples, 1) + (1 - ratio) * segs1[:,[1],:].repeat(1, self.dense_options.n_samples, 1) + coords_1 = coords_1.reshape(-1, 2) + coords = self.dense_matcher.to_normalized_coordinates(coords_1, descinfo1["image_shape"][0], descinfo1["image_shape"][1]) + coords_to_2 = F.grid_sample(warp_1to2.permute(2, 0, 1)[None], coords[None, None], align_corners=False, mode="bilinear")[0,:,0].mT + coords_to_2 = self.dense_matcher.to_unnormalized_coordinates(coords_to_2, descinfo2["image_shape"][0], descinfo2["image_shape"][1]) + cert_to_2 = F.grid_sample(cert_1to2[None,None,...], coords[None, None], align_corners=False, mode="bilinear")[0,0,0] + cert_to_2 = cert_to_2.reshape(-1, self.dense_options.n_samples) + + # get projections + segs2 = torch.from_numpy(descinfo2["lines"]).to(self.dense_options.device) + n_segs2 = segs2.shape[0] + starts2, ends2 = segs2[:,0,:], segs2[:,1,:] + directions = ends2 - starts2 + directions /= torch.norm(directions, dim=1, keepdim=True) + starts2_proj = (starts2 * directions).sum(1) + ends2_proj = (ends2 * directions).sum(1) + + # get line equations + starts_homo = torch.cat([starts2, torch.ones_like(segs2[:,[0],0])], 1) + ends_homo = torch.cat([ends2, torch.ones_like(segs2[:,[0],0])], 1) + lines2_homo = torch.cross(starts_homo, ends_homo) + lines2_homo /= torch.norm(lines2_homo[:,:2], dim=1)[:,None].repeat(1, 3) + + # compute distance + coords_to_2_homo = torch.cat([coords_to_2, torch.ones_like(coords_to_2[:,[0]])], 1) + coords_proj = torch.matmul(coords_to_2, directions.T) + dists = torch.abs(torch.matmul(coords_to_2_homo, lines2_homo.T)) + overlap = torch.where(coords_proj > starts2_proj, torch.ones_like(dists), torch.zeros_like(dists)) + overlap = torch.where(coords_proj < ends2_proj, overlap, torch.zeros_like(dists)) + dists = dists.reshape(n_segs1, self.dense_options.n_samples, n_segs2).permute(0, 2, 1) + overlap = overlap.reshape(n_segs1, self.dense_options.n_samples, n_segs2).permute(0, 2, 1).to(torch.bool) + + # get active lines for each target + sample_thresh = self.dense_matcher.get_sample_thresh() + good_sample = cert_to_2 > sample_thresh + good_sample = torch.logical_and(good_sample[:,None,:].repeat(1, overlap.shape[1], 1), overlap) + sample_weight = good_sample.to(torch.float) + sample_weight_sum = sample_weight.sum(2) + sample_weight[sample_weight_sum > 0] /= sample_weight_sum[sample_weight_sum > 0][:,None].repeat(1, sample_weight.shape[2]) + is_active = sample_weight_sum > self.dense_options.segment_percentage_th * self.dense_options.n_samples + + # get weighted dists + weighted_dists = (dists * sample_weight).sum(2) + weighted_dists[weighted_dists == 0] = 10000. + return weighted_dists, sample_weight_sum / self.dense_options.n_samples + + def match_segs_with_descinfo(self, descinfo1, descinfo2): + img1 = descinfo1["camview"].read_image() + img2 = descinfo2["camview"].read_image() + warp_1to2, cert_1to2, warp_2to1, cert_2to1 = self.dense_matcher.get_warpping_symmetric(img1, img2) + + # compute distance and overlap + dists_1to2, overlap_1to2 = self.compute_distance_one_direction(descinfo1, descinfo2, warp_1to2, cert_1to2) + dists_2to1, overlap_2to1 = self.compute_distance_one_direction(descinfo2, descinfo1, warp_2to1, cert_2to1) + overlap = torch.maximum(overlap_1to2, overlap_2to1.T) + dists = torch.where(overlap_1to2 > overlap_2to1.T, dists_1to2, dists_2to1.T) + + # match: one-way nearest neighbor + # TODO: one-to-many matching + inds_1, inds_2 = torch.nonzero(dists == dists.min(dim=-1, keepdim=True).values * (dists <= self.dense_options.pixel_th), as_tuple=True) + inds_1 = inds_1.detach().cpu().numpy() + inds_2 = inds_2.detach().cpu().numpy() + matches_t = np.stack([inds_1, inds_2], axis=1) + return matches_t + + def match_segs_with_descinfo_topk(self, descinfo1, descinfo2, topk=10): + raise NotImplementedError + + +class RoMaLineMatcher(BaseDenseLineMatcher): + def __init__(self, extractor, mode="outdoor", dense_options=BaseDenseLineMatcherOptions(), options=BaseMatcherOptions()): + from .dense_matcher import RoMa + roma_matcher = RoMa(mode=mode, device=dense_options.device) + super(RoMaLineMatcher, self).__init__(extractor, roma_matcher, dense_options=dense_options, options=options) + + def get_module_name(self): + return "dense_roma" + diff --git a/limap/line2d/register_detector.py b/limap/line2d/register_detector.py index abf13582..256a0300 100644 --- a/limap/line2d/register_detector.py +++ b/limap/line2d/register_detector.py @@ -81,7 +81,9 @@ def get_extractor(cfg_extractor, weight_path=None): return SuperPointEndpointsExtractor(options) elif method == "wireframe": from .GlueStick import WireframeExtractor - return WireframeExtractor(options) + elif method == "dense_naive": + from .dense import DenseNaiveExtractor + return DenseNaiveExtractor(options) else: raise NotImplementedError diff --git a/limap/line2d/register_matcher.py b/limap/line2d/register_matcher.py index e80bd5f3..014732b7 100644 --- a/limap/line2d/register_matcher.py +++ b/limap/line2d/register_matcher.py @@ -46,7 +46,9 @@ def get_matcher(cfg_matcher, extractor, n_neighbors=20, weight_path=None): ) elif method == "gluestick": from .GlueStick import GlueStickMatcher - return GlueStickMatcher(extractor, options) + elif method == "dense_roma": + from .dense import RoMaLineMatcher + return RoMaLineMatcher(extractor, options=options) else: raise NotImplementedError From 237e215da72e347354d7190212be8a9c154e700b Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Sat, 19 Oct 2024 14:40:38 +0200 Subject: [PATCH 02/20] formatting. --- limap/line2d/dense/dense_matcher/base.py | 22 ++-- limap/line2d/dense/dense_matcher/roma.py | 15 ++- limap/line2d/dense/extractor.py | 12 +- limap/line2d/dense/matcher.py | 152 ++++++++++++++++++----- limap/line2d/register_detector.py | 2 + limap/line2d/register_matcher.py | 2 + 6 files changed, 152 insertions(+), 53 deletions(-) diff --git a/limap/line2d/dense/dense_matcher/base.py b/limap/line2d/dense/dense_matcher/base.py index 568affb8..9add74cc 100644 --- a/limap/line2d/dense/dense_matcher/base.py +++ b/limap/line2d/dense/dense_matcher/base.py @@ -1,35 +1,35 @@ import os import torch + class BaseDenseMatcher(object): def __init__(self): pass def to_normalized_coordinates(self, coords, h, w): - ''' + """ coords: (..., 2) in the order x, y - ''' - coords_x = 2 / w * coords[..., 0] - 1; - coords_y = 2 / h * coords[..., 1] - 1; + """ + coords_x = 2 / w * coords[..., 0] - 1 + coords_y = 2 / h * coords[..., 1] - 1 return torch.stack([coords_x, coords_y], axis=-1) def to_unnormalized_coordinates(self, coords, h, w): - ''' + """ Inverse operation of `to_normalized_coordinates` - ''' + """ coords_x = (coords[..., 0] + 1) * w / 2 coords_y = (coords[..., 1] + 1) * h / 2 return torch.stack([coords_x, coords_y], axis=-1) def get_sample_thresh(self): - ''' + """ return sample threshold - ''' + """ raise NotImplementedError def get_warpping_symmetric(self, img1, img2): - ''' + """ return warp_1to2 ([-1, 1]), cert_1to2, warp_2to1([-1, 1]), cert_2to1 - ''' + """ raise NotImplementedError - diff --git a/limap/line2d/dense/dense_matcher/roma.py b/limap/line2d/dense/dense_matcher/roma.py index 3b1be393..52a5a981 100644 --- a/limap/line2d/dense/dense_matcher/roma.py +++ b/limap/line2d/dense/dense_matcher/roma.py @@ -3,8 +3,9 @@ import romatch from PIL import Image + class RoMa(BaseDenseMatcher): - def __init__(self, mode = "outdoor", device="cuda"): + def __init__(self, mode="outdoor", device="cuda"): super(RoMa).__init__() if mode == "outdoor": self.model = romatch.roma_outdoor(device=device, coarse_res=560) @@ -15,7 +16,13 @@ def get_sample_thresh(self): return self.model.sample_thresh def get_warpping_symmetric(self, img1, img2): - warp, certainty = self.model.match(Image.fromarray(img1), Image.fromarray(img2)) + warp, certainty = self.model.match( + Image.fromarray(img1), Image.fromarray(img2) + ) N = 864 - return warp[:,:N,2:], certainty[:,:N], warp[:,N:,:2], certainty[:,N:] - + return ( + warp[:, :N, 2:], + certainty[:, :N], + warp[:, N:, :2], + certainty[:, N:], + ) diff --git a/limap/line2d/dense/extractor.py b/limap/line2d/dense/extractor.py index 04595cbc..6f344bc8 100644 --- a/limap/line2d/dense/extractor.py +++ b/limap/line2d/dense/extractor.py @@ -31,10 +31,10 @@ def extract(self, camview, segs): scores = segs[:, -1] * np.sqrt( np.linalg.norm(segs[:, :2] - segs[:, 2:4], axis=1) ) - descinfo = {"camview": camview, - "image_shape": img.shape, - "lines": lines, - "scores": scores, - } + descinfo = { + "camview": camview, + "image_shape": img.shape, + "lines": lines, + "scores": scores, + } return descinfo - diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py index f257ea6e..bf3146ad 100644 --- a/limap/line2d/dense/matcher.py +++ b/limap/line2d/dense/matcher.py @@ -7,6 +7,7 @@ import limap.util.io as limapio from ..base_matcher import BaseMatcher, BaseMatcherOptions + class BaseDenseLineMatcherOptions(NamedTuple): n_samples: int = 21 segment_percentage_th: float = 0.5 @@ -15,7 +16,13 @@ class BaseDenseLineMatcherOptions(NamedTuple): class BaseDenseLineMatcher(BaseMatcher): - def __init__(self, extractor, dense_matcher, dense_options=BaseDenseLineMatcherOptions(), options=BaseMatcherOptions()): + def __init__( + self, + extractor, + dense_matcher, + dense_options=BaseDenseLineMatcherOptions(), + options=BaseMatcherOptions(), + ): super(BaseDenseLineMatcher, self).__init__(extractor, options) assert self.extractor.get_module_name() == "dense_naive" self.dense_matcher = dense_matcher @@ -33,72 +40,142 @@ def match_pair(self, descinfo1, descinfo2): descinfo1, descinfo2, topk=self.topk ) - def compute_distance_one_direction(self, descinfo1, descinfo2, warp_1to2, cert_1to2): + def compute_distance_one_direction( + self, descinfo1, descinfo2, warp_1to2, cert_1to2 + ): # get point samples along lines - segs1 = torch.from_numpy(descinfo1["lines"]).to(self.dense_options.device) + segs1 = torch.from_numpy(descinfo1["lines"]).to( + self.dense_options.device + ) n_segs1 = segs1.shape[0] - ratio = torch.arange(0, 1 + 0.5 / (self.dense_options.n_samples - 1), 1.0 / (self.dense_options.n_samples - 1)).to(self.dense_options.device) - ratio = ratio[:,None].repeat(1, 2) - coords_1 = ratio * segs1[:,[0],:].repeat(1, self.dense_options.n_samples, 1) + (1 - ratio) * segs1[:,[1],:].repeat(1, self.dense_options.n_samples, 1) + ratio = torch.arange( + 0, + 1 + 0.5 / (self.dense_options.n_samples - 1), + 1.0 / (self.dense_options.n_samples - 1), + ).to(self.dense_options.device) + ratio = ratio[:, None].repeat(1, 2) + coords_1 = ratio * segs1[:, [0], :].repeat( + 1, self.dense_options.n_samples, 1 + ) + (1 - ratio) * segs1[:, [1], :].repeat( + 1, self.dense_options.n_samples, 1 + ) coords_1 = coords_1.reshape(-1, 2) - coords = self.dense_matcher.to_normalized_coordinates(coords_1, descinfo1["image_shape"][0], descinfo1["image_shape"][1]) - coords_to_2 = F.grid_sample(warp_1to2.permute(2, 0, 1)[None], coords[None, None], align_corners=False, mode="bilinear")[0,:,0].mT - coords_to_2 = self.dense_matcher.to_unnormalized_coordinates(coords_to_2, descinfo2["image_shape"][0], descinfo2["image_shape"][1]) - cert_to_2 = F.grid_sample(cert_1to2[None,None,...], coords[None, None], align_corners=False, mode="bilinear")[0,0,0] + coords = self.dense_matcher.to_normalized_coordinates( + coords_1, descinfo1["image_shape"][0], descinfo1["image_shape"][1] + ) + coords_to_2 = F.grid_sample( + warp_1to2.permute(2, 0, 1)[None], + coords[None, None], + align_corners=False, + mode="bilinear", + )[0, :, 0].mT + coords_to_2 = self.dense_matcher.to_unnormalized_coordinates( + coords_to_2, + descinfo2["image_shape"][0], + descinfo2["image_shape"][1], + ) + cert_to_2 = F.grid_sample( + cert_1to2[None, None, ...], + coords[None, None], + align_corners=False, + mode="bilinear", + )[0, 0, 0] cert_to_2 = cert_to_2.reshape(-1, self.dense_options.n_samples) # get projections - segs2 = torch.from_numpy(descinfo2["lines"]).to(self.dense_options.device) + segs2 = torch.from_numpy(descinfo2["lines"]).to( + self.dense_options.device + ) n_segs2 = segs2.shape[0] - starts2, ends2 = segs2[:,0,:], segs2[:,1,:] + starts2, ends2 = segs2[:, 0, :], segs2[:, 1, :] directions = ends2 - starts2 directions /= torch.norm(directions, dim=1, keepdim=True) starts2_proj = (starts2 * directions).sum(1) ends2_proj = (ends2 * directions).sum(1) # get line equations - starts_homo = torch.cat([starts2, torch.ones_like(segs2[:,[0],0])], 1) - ends_homo = torch.cat([ends2, torch.ones_like(segs2[:,[0],0])], 1) + starts_homo = torch.cat([starts2, torch.ones_like(segs2[:, [0], 0])], 1) + ends_homo = torch.cat([ends2, torch.ones_like(segs2[:, [0], 0])], 1) lines2_homo = torch.cross(starts_homo, ends_homo) - lines2_homo /= torch.norm(lines2_homo[:,:2], dim=1)[:,None].repeat(1, 3) + lines2_homo /= torch.norm(lines2_homo[:, :2], dim=1)[:, None].repeat( + 1, 3 + ) # compute distance - coords_to_2_homo = torch.cat([coords_to_2, torch.ones_like(coords_to_2[:,[0]])], 1) + coords_to_2_homo = torch.cat( + [coords_to_2, torch.ones_like(coords_to_2[:, [0]])], 1 + ) coords_proj = torch.matmul(coords_to_2, directions.T) dists = torch.abs(torch.matmul(coords_to_2_homo, lines2_homo.T)) - overlap = torch.where(coords_proj > starts2_proj, torch.ones_like(dists), torch.zeros_like(dists)) - overlap = torch.where(coords_proj < ends2_proj, overlap, torch.zeros_like(dists)) - dists = dists.reshape(n_segs1, self.dense_options.n_samples, n_segs2).permute(0, 2, 1) - overlap = overlap.reshape(n_segs1, self.dense_options.n_samples, n_segs2).permute(0, 2, 1).to(torch.bool) + overlap = torch.where( + coords_proj > starts2_proj, + torch.ones_like(dists), + torch.zeros_like(dists), + ) + overlap = torch.where( + coords_proj < ends2_proj, overlap, torch.zeros_like(dists) + ) + dists = dists.reshape( + n_segs1, self.dense_options.n_samples, n_segs2 + ).permute(0, 2, 1) + overlap = ( + overlap.reshape(n_segs1, self.dense_options.n_samples, n_segs2) + .permute(0, 2, 1) + .to(torch.bool) + ) # get active lines for each target sample_thresh = self.dense_matcher.get_sample_thresh() good_sample = cert_to_2 > sample_thresh - good_sample = torch.logical_and(good_sample[:,None,:].repeat(1, overlap.shape[1], 1), overlap) + good_sample = torch.logical_and( + good_sample[:, None, :].repeat(1, overlap.shape[1], 1), overlap + ) sample_weight = good_sample.to(torch.float) sample_weight_sum = sample_weight.sum(2) - sample_weight[sample_weight_sum > 0] /= sample_weight_sum[sample_weight_sum > 0][:,None].repeat(1, sample_weight.shape[2]) - is_active = sample_weight_sum > self.dense_options.segment_percentage_th * self.dense_options.n_samples + sample_weight[sample_weight_sum > 0] /= sample_weight_sum[ + sample_weight_sum > 0 + ][:, None].repeat(1, sample_weight.shape[2]) + is_active = ( + sample_weight_sum + > self.dense_options.segment_percentage_th + * self.dense_options.n_samples + ) # get weighted dists weighted_dists = (dists * sample_weight).sum(2) - weighted_dists[weighted_dists == 0] = 10000. + weighted_dists[weighted_dists == 0] = 10000.0 return weighted_dists, sample_weight_sum / self.dense_options.n_samples def match_segs_with_descinfo(self, descinfo1, descinfo2): img1 = descinfo1["camview"].read_image() img2 = descinfo2["camview"].read_image() - warp_1to2, cert_1to2, warp_2to1, cert_2to1 = self.dense_matcher.get_warpping_symmetric(img1, img2) + ( + warp_1to2, + cert_1to2, + warp_2to1, + cert_2to1, + ) = self.dense_matcher.get_warpping_symmetric(img1, img2) # compute distance and overlap - dists_1to2, overlap_1to2 = self.compute_distance_one_direction(descinfo1, descinfo2, warp_1to2, cert_1to2) - dists_2to1, overlap_2to1 = self.compute_distance_one_direction(descinfo2, descinfo1, warp_2to1, cert_2to1) + dists_1to2, overlap_1to2 = self.compute_distance_one_direction( + descinfo1, descinfo2, warp_1to2, cert_1to2 + ) + dists_2to1, overlap_2to1 = self.compute_distance_one_direction( + descinfo2, descinfo1, warp_2to1, cert_2to1 + ) overlap = torch.maximum(overlap_1to2, overlap_2to1.T) - dists = torch.where(overlap_1to2 > overlap_2to1.T, dists_1to2, dists_2to1.T) + dists = torch.where( + overlap_1to2 > overlap_2to1.T, dists_1to2, dists_2to1.T + ) # match: one-way nearest neighbor # TODO: one-to-many matching - inds_1, inds_2 = torch.nonzero(dists == dists.min(dim=-1, keepdim=True).values * (dists <= self.dense_options.pixel_th), as_tuple=True) + inds_1, inds_2 = torch.nonzero( + dists + == dists.min(dim=-1, keepdim=True).values + * (dists <= self.dense_options.pixel_th), + as_tuple=True, + ) inds_1 = inds_1.detach().cpu().numpy() inds_2 = inds_2.detach().cpu().numpy() matches_t = np.stack([inds_1, inds_2], axis=1) @@ -109,11 +186,22 @@ def match_segs_with_descinfo_topk(self, descinfo1, descinfo2, topk=10): class RoMaLineMatcher(BaseDenseLineMatcher): - def __init__(self, extractor, mode="outdoor", dense_options=BaseDenseLineMatcherOptions(), options=BaseMatcherOptions()): + def __init__( + self, + extractor, + mode="outdoor", + dense_options=BaseDenseLineMatcherOptions(), + options=BaseMatcherOptions(), + ): from .dense_matcher import RoMa + roma_matcher = RoMa(mode=mode, device=dense_options.device) - super(RoMaLineMatcher, self).__init__(extractor, roma_matcher, dense_options=dense_options, options=options) + super(RoMaLineMatcher, self).__init__( + extractor, + roma_matcher, + dense_options=dense_options, + options=options, + ) def get_module_name(self): return "dense_roma" - diff --git a/limap/line2d/register_detector.py b/limap/line2d/register_detector.py index 256a0300..8394b6d5 100644 --- a/limap/line2d/register_detector.py +++ b/limap/line2d/register_detector.py @@ -81,9 +81,11 @@ def get_extractor(cfg_extractor, weight_path=None): return SuperPointEndpointsExtractor(options) elif method == "wireframe": from .GlueStick import WireframeExtractor + return WireframeExtractor(options) elif method == "dense_naive": from .dense import DenseNaiveExtractor + return DenseNaiveExtractor(options) else: raise NotImplementedError diff --git a/limap/line2d/register_matcher.py b/limap/line2d/register_matcher.py index 014732b7..f4ace620 100644 --- a/limap/line2d/register_matcher.py +++ b/limap/line2d/register_matcher.py @@ -46,9 +46,11 @@ def get_matcher(cfg_matcher, extractor, n_neighbors=20, weight_path=None): ) elif method == "gluestick": from .GlueStick import GlueStickMatcher + return GlueStickMatcher(extractor, options) elif method == "dense_roma": from .dense import RoMaLineMatcher + return RoMaLineMatcher(extractor, options=options) else: raise NotImplementedError From 009b0db5c2f54d1e1c70f83e59aa60b4a3a8bc43 Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Sat, 19 Oct 2024 14:45:47 +0200 Subject: [PATCH 03/20] add a test script. --- test_matching.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test_matching.py diff --git a/test_matching.py b/test_matching.py new file mode 100644 index 00000000..34b00aa7 --- /dev/null +++ b/test_matching.py @@ -0,0 +1,56 @@ +import limap.util.config +import limap.base +import limap.line2d +import cv2 + +detector = limap.line2d.get_detector({"method": "lsd", "skip_exists": False}) # get a line detector +extractor = limap.line2d.get_extractor({"method": "dense_naive", "skip_exists": False}) # get a line extractor +matcher = limap.line2d.get_matcher({"method": "dense_roma", "skip_exists": False, "n_jobs": 1, "topk": 0}, extractor) # initiate a line matcher + +view1 = limap.base.CameraView("/home/shaoliu/workspace/GlueStick/resources/img1.jpg") # initiate an limap.base.CameraView instance for detection. +view2 = limap.base.CameraView("/home/shaoliu/workspace/GlueStick/resources/img2.jpg") # initiate an limap.base.CameraView instance for detection. + +segs1 = detector.detect(view1) # detection +desc1 = extractor.extract(view1, segs1) # description +segs2 = detector.detect(view2) # detection +desc2 = extractor.extract(view2, segs2) # description +matches = matcher.match_pair(desc1, desc2) # matching + +def vis_detections(img, segs): + from limap.visualize.vis_utils import draw_segments + import copy + img_draw = copy.deepcopy(img) + img_draw = draw_segments(img_draw, segs, color=[0, 255, 0]) + return img_draw + +def vis_matches(img1, img2, segs1, segs2, matches): + import cv2 + import numpy as np + import seaborn as sns + import copy + matched_seg1 = segs1[matches[:,0]] + matched_seg2 = segs2[matches[:,1]] + n_lines = matched_seg1.shape[0] + colors = sns.color_palette("husl", n_colors=n_lines) + img1_draw = copy.deepcopy(img1) + img2_draw = copy.deepcopy(img2) + for idx in range(n_lines): + color = np.array(colors[idx]) * 255. + color = color.astype(int).tolist() + cv2.line(img1_draw, (int(matched_seg1[idx, 0]), int(matched_seg1[idx, 1])), (int(matched_seg1[idx, 2]), int(matched_seg1[idx, 3])), color, 4) + cv2.line(img2_draw, (int(matched_seg2[idx, 0]), int(matched_seg2[idx, 1])), (int(matched_seg2[idx, 2]), int(matched_seg2[idx, 3])), color, 4) + return img1_draw, img2_draw + +img1 = view1.read_image() +img2 = view2.read_image() +img1_det = vis_detections(img1, segs1) +cv2.imwrite("tmp/img1_det.png", img1_det) +img2_det = vis_detections(img2, segs2) +cv2.imwrite("tmp/img2_det.png", img2_det) +img1_draw, img2_draw = vis_matches(img1, img2, segs1, segs2, matches) +cv2.imwrite("tmp/img1_draw.png", img1_draw) +cv2.imwrite("tmp/img2_draw.png", img2_draw) + +import pdb +pdb.set_trace() + From b95f8c598193eae119f486cf5684870af897f722 Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Sat, 19 Oct 2024 14:46:38 +0200 Subject: [PATCH 04/20] minor. --- test_matching.py => scripts/test_matching.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test_matching.py => scripts/test_matching.py (100%) diff --git a/test_matching.py b/scripts/test_matching.py similarity index 100% rename from test_matching.py rename to scripts/test_matching.py From 15e88b33561094f3d8b7148e6134d1a4ad8d7fa5 Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Sat, 19 Oct 2024 14:48:18 +0200 Subject: [PATCH 05/20] formatting. --- scripts/test_matching.py | 60 +++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/scripts/test_matching.py b/scripts/test_matching.py index 34b00aa7..8f21f9ac 100644 --- a/scripts/test_matching.py +++ b/scripts/test_matching.py @@ -3,44 +3,72 @@ import limap.line2d import cv2 -detector = limap.line2d.get_detector({"method": "lsd", "skip_exists": False}) # get a line detector -extractor = limap.line2d.get_extractor({"method": "dense_naive", "skip_exists": False}) # get a line extractor -matcher = limap.line2d.get_matcher({"method": "dense_roma", "skip_exists": False, "n_jobs": 1, "topk": 0}, extractor) # initiate a line matcher +detector = limap.line2d.get_detector( + {"method": "lsd", "skip_exists": False} +) # get a line detector +extractor = limap.line2d.get_extractor( + {"method": "dense_naive", "skip_exists": False} +) # get a line extractor +matcher = limap.line2d.get_matcher( + {"method": "dense_roma", "skip_exists": False, "n_jobs": 1, "topk": 0}, + extractor, +) # initiate a line matcher -view1 = limap.base.CameraView("/home/shaoliu/workspace/GlueStick/resources/img1.jpg") # initiate an limap.base.CameraView instance for detection. -view2 = limap.base.CameraView("/home/shaoliu/workspace/GlueStick/resources/img2.jpg") # initiate an limap.base.CameraView instance for detection. +view1 = limap.base.CameraView( + "/home/shaoliu/workspace/GlueStick/resources/img1.jpg" +) # initiate an limap.base.CameraView instance for detection. +view2 = limap.base.CameraView( + "/home/shaoliu/workspace/GlueStick/resources/img2.jpg" +) # initiate an limap.base.CameraView instance for detection. + +segs1 = detector.detect(view1) # detection +desc1 = extractor.extract(view1, segs1) # description +segs2 = detector.detect(view2) # detection +desc2 = extractor.extract(view2, segs2) # description +matches = matcher.match_pair(desc1, desc2) # matching -segs1 = detector.detect(view1) # detection -desc1 = extractor.extract(view1, segs1) # description -segs2 = detector.detect(view2) # detection -desc2 = extractor.extract(view2, segs2) # description -matches = matcher.match_pair(desc1, desc2) # matching def vis_detections(img, segs): from limap.visualize.vis_utils import draw_segments import copy + img_draw = copy.deepcopy(img) img_draw = draw_segments(img_draw, segs, color=[0, 255, 0]) return img_draw + def vis_matches(img1, img2, segs1, segs2, matches): import cv2 import numpy as np import seaborn as sns import copy - matched_seg1 = segs1[matches[:,0]] - matched_seg2 = segs2[matches[:,1]] + + matched_seg1 = segs1[matches[:, 0]] + matched_seg2 = segs2[matches[:, 1]] n_lines = matched_seg1.shape[0] colors = sns.color_palette("husl", n_colors=n_lines) img1_draw = copy.deepcopy(img1) img2_draw = copy.deepcopy(img2) for idx in range(n_lines): - color = np.array(colors[idx]) * 255. + color = np.array(colors[idx]) * 255.0 color = color.astype(int).tolist() - cv2.line(img1_draw, (int(matched_seg1[idx, 0]), int(matched_seg1[idx, 1])), (int(matched_seg1[idx, 2]), int(matched_seg1[idx, 3])), color, 4) - cv2.line(img2_draw, (int(matched_seg2[idx, 0]), int(matched_seg2[idx, 1])), (int(matched_seg2[idx, 2]), int(matched_seg2[idx, 3])), color, 4) + cv2.line( + img1_draw, + (int(matched_seg1[idx, 0]), int(matched_seg1[idx, 1])), + (int(matched_seg1[idx, 2]), int(matched_seg1[idx, 3])), + color, + 4, + ) + cv2.line( + img2_draw, + (int(matched_seg2[idx, 0]), int(matched_seg2[idx, 1])), + (int(matched_seg2[idx, 2]), int(matched_seg2[idx, 3])), + color, + 4, + ) return img1_draw, img2_draw + img1 = view1.read_image() img2 = view2.read_image() img1_det = vis_detections(img1, segs1) @@ -52,5 +80,5 @@ def vis_matches(img1, img2, segs1, segs2, matches): cv2.imwrite("tmp/img2_draw.png", img2_draw) import pdb -pdb.set_trace() +pdb.set_trace() From 4acbf88fa2dd28952306d35f185e700bdfea8c20 Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Sat, 19 Oct 2024 16:55:29 +0200 Subject: [PATCH 06/20] fix formatting with ruff. --- limap/line2d/dense/dense_matcher/base.py | 3 ++- limap/line2d/dense/dense_matcher/roma.py | 4 +++- limap/line2d/dense/extractor.py | 7 +++++-- limap/line2d/dense/matcher.py | 20 +++++++++++--------- scripts/test_matching.py | 11 +++++++---- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/limap/line2d/dense/dense_matcher/base.py b/limap/line2d/dense/dense_matcher/base.py index 9add74cc..d751db8e 100644 --- a/limap/line2d/dense/dense_matcher/base.py +++ b/limap/line2d/dense/dense_matcher/base.py @@ -1,8 +1,9 @@ import os + import torch -class BaseDenseMatcher(object): +class BaseDenseMatcher: def __init__(self): pass diff --git a/limap/line2d/dense/dense_matcher/roma.py b/limap/line2d/dense/dense_matcher/roma.py index 52a5a981..a2ff710e 100644 --- a/limap/line2d/dense/dense_matcher/roma.py +++ b/limap/line2d/dense/dense_matcher/roma.py @@ -1,8 +1,10 @@ import os -from .base import BaseDenseMatcher + import romatch from PIL import Image +from .base import BaseDenseMatcher + class RoMa(BaseDenseMatcher): def __init__(self, mode="outdoor", device="cuda"): diff --git a/limap/line2d/dense/extractor.py b/limap/line2d/dense/extractor.py index 6f344bc8..2668ce38 100644 --- a/limap/line2d/dense/extractor.py +++ b/limap/line2d/dense/extractor.py @@ -1,18 +1,21 @@ import os + import numpy as np + import limap.util.io as limapio + from ..base_detector import BaseDetector, BaseDetectorOptions class DenseNaiveExtractor(BaseDetector): def __init__(self, options=BaseDetectorOptions(), device=None): - super(DenseNaiveExtractor, self).__init__(options) + super().__init__(options) def get_module_name(self): return "dense_naive" def get_descinfo_fname(self, descinfo_folder, img_id): - fname = os.path.join(descinfo_folder, "descinfo_{0}.npz".format(img_id)) + fname = os.path.join(descinfo_folder, f"descinfo_{img_id}.npz") return fname def save_descinfo(self, descinfo_folder, img_id, descinfo): diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py index bf3146ad..2d213a85 100644 --- a/limap/line2d/dense/matcher.py +++ b/limap/line2d/dense/matcher.py @@ -1,10 +1,12 @@ import os +from typing import NamedTuple + import numpy as np import torch import torch.nn.functional as F -from typing import NamedTuple import limap.util.io as limapio + from ..base_matcher import BaseMatcher, BaseMatcherOptions @@ -23,7 +25,7 @@ def __init__( dense_options=BaseDenseLineMatcherOptions(), options=BaseMatcherOptions(), ): - super(BaseDenseLineMatcher, self).__init__(extractor, options) + super().__init__(extractor, options) assert self.extractor.get_module_name() == "dense_naive" self.dense_matcher = dense_matcher self.dense_options = dense_options @@ -135,11 +137,11 @@ def compute_distance_one_direction( sample_weight[sample_weight_sum > 0] /= sample_weight_sum[ sample_weight_sum > 0 ][:, None].repeat(1, sample_weight.shape[2]) - is_active = ( - sample_weight_sum - > self.dense_options.segment_percentage_th - * self.dense_options.n_samples - ) + # is_active = ( + # sample_weight_sum + # > self.dense_options.segment_percentage_th + # * self.dense_options.n_samples + # ) # get weighted dists weighted_dists = (dists * sample_weight).sum(2) @@ -163,7 +165,7 @@ def match_segs_with_descinfo(self, descinfo1, descinfo2): dists_2to1, overlap_2to1 = self.compute_distance_one_direction( descinfo2, descinfo1, warp_2to1, cert_2to1 ) - overlap = torch.maximum(overlap_1to2, overlap_2to1.T) + # overlap = torch.maximum(overlap_1to2, overlap_2to1.T) dists = torch.where( overlap_1to2 > overlap_2to1.T, dists_1to2, dists_2to1.T ) @@ -196,7 +198,7 @@ def __init__( from .dense_matcher import RoMa roma_matcher = RoMa(mode=mode, device=dense_options.device) - super(RoMaLineMatcher, self).__init__( + super().__init__( extractor, roma_matcher, dense_options=dense_options, diff --git a/scripts/test_matching.py b/scripts/test_matching.py index 8f21f9ac..ec417b5c 100644 --- a/scripts/test_matching.py +++ b/scripts/test_matching.py @@ -1,7 +1,8 @@ -import limap.util.config +import cv2 + import limap.base import limap.line2d -import cv2 +import limap.util.config detector = limap.line2d.get_detector( {"method": "lsd", "skip_exists": False} @@ -29,19 +30,21 @@ def vis_detections(img, segs): - from limap.visualize.vis_utils import draw_segments import copy + from limap.visualize.vis_utils import draw_segments + img_draw = copy.deepcopy(img) img_draw = draw_segments(img_draw, segs, color=[0, 255, 0]) return img_draw def vis_matches(img1, img2, segs1, segs2, matches): + import copy + import cv2 import numpy as np import seaborn as sns - import copy matched_seg1 = segs1[matches[:, 0]] matched_seg2 = segs2[matches[:, 1]] From 786211ce50c0883ea2ce34b433e647bfa3c5acf1 Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Mon, 21 Oct 2024 22:29:03 +0200 Subject: [PATCH 07/20] add tiny roma. --- limap/line2d/dense/dense_matcher/__init__.py | 1 + limap/line2d/dense/dense_matcher/roma.py | 2 ++ limap/line2d/dense/matcher.py | 6 ++++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/limap/line2d/dense/dense_matcher/__init__.py b/limap/line2d/dense/dense_matcher/__init__.py index 8e8108db..c9a48134 100644 --- a/limap/line2d/dense/dense_matcher/__init__.py +++ b/limap/line2d/dense/dense_matcher/__init__.py @@ -1 +1,2 @@ +from .base import BaseDenseMatcher from .roma import RoMa diff --git a/limap/line2d/dense/dense_matcher/roma.py b/limap/line2d/dense/dense_matcher/roma.py index a2ff710e..40923b8f 100644 --- a/limap/line2d/dense/dense_matcher/roma.py +++ b/limap/line2d/dense/dense_matcher/roma.py @@ -13,6 +13,8 @@ def __init__(self, mode="outdoor", device="cuda"): self.model = romatch.roma_outdoor(device=device, coarse_res=560) elif mode == "indoor": self.model = romatch.roma_indoor(device=device, coarse_res=560) + elif mode == "tiny_outdoor": + self.model = romatch.tiny_roma_v1_outdoor(device=device) def get_sample_thresh(self): return self.model.sample_thresh diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py index 2d213a85..fb2548d2 100644 --- a/limap/line2d/dense/matcher.py +++ b/limap/line2d/dense/matcher.py @@ -8,6 +8,7 @@ import limap.util.io as limapio from ..base_matcher import BaseMatcher, BaseMatcherOptions +from .dense_matcher import BaseDenseMatcher class BaseDenseLineMatcherOptions(NamedTuple): @@ -27,9 +28,10 @@ def __init__( ): super().__init__(extractor, options) assert self.extractor.get_module_name() == "dense_naive" - self.dense_matcher = dense_matcher + assert dense_options.n_samples >= 2 self.dense_options = dense_options - assert self.dense_options.n_samples >= 2 + assert isinstance(dense_matcher, BaseDenseMatcher) + self.dense_matcher = dense_matcher def get_module_name(self): raise NotImplementedError From 7248ddd00b51f5769492603f1b9422d2ee4d2e07 Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Mon, 21 Oct 2024 22:31:34 +0200 Subject: [PATCH 08/20] minor. --- scripts/test_matching.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/test_matching.py b/scripts/test_matching.py index ec417b5c..abd042ea 100644 --- a/scripts/test_matching.py +++ b/scripts/test_matching.py @@ -81,7 +81,3 @@ def vis_matches(img1, img2, segs1, segs2, matches): img1_draw, img2_draw = vis_matches(img1, img2, segs1, segs2, matches) cv2.imwrite("tmp/img1_draw.png", img1_draw) cv2.imwrite("tmp/img2_draw.png", img2_draw) - -import pdb - -pdb.set_trace() From 1207f5f8a64e61fadcf8049726bc5cd1121800dc Mon Sep 17 00:00:00 2001 From: pautratrmi Date: Wed, 23 Oct 2024 17:54:29 +0200 Subject: [PATCH 09/20] Fix different input/output conventions for tiny RoMa --- limap/line2d/dense/dense_matcher/base.py | 2 +- limap/line2d/dense/dense_matcher/roma.py | 28 ++++++++++++++++-------- limap/line2d/dense/matcher.py | 2 +- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/limap/line2d/dense/dense_matcher/base.py b/limap/line2d/dense/dense_matcher/base.py index d751db8e..20309786 100644 --- a/limap/line2d/dense/dense_matcher/base.py +++ b/limap/line2d/dense/dense_matcher/base.py @@ -29,7 +29,7 @@ def get_sample_thresh(self): """ raise NotImplementedError - def get_warpping_symmetric(self, img1, img2): + def get_warping_symmetric(self, img1, img2): """ return warp_1to2 ([-1, 1]), cert_1to2, warp_2to1([-1, 1]), cert_2to1 """ diff --git a/limap/line2d/dense/dense_matcher/roma.py b/limap/line2d/dense/dense_matcher/roma.py index 40923b8f..74f36ec2 100644 --- a/limap/line2d/dense/dense_matcher/roma.py +++ b/limap/line2d/dense/dense_matcher/roma.py @@ -9,24 +9,34 @@ class RoMa(BaseDenseMatcher): def __init__(self, mode="outdoor", device="cuda"): super(RoMa).__init__() + self.output_res = 864 + self.mode = mode if mode == "outdoor": - self.model = romatch.roma_outdoor(device=device, coarse_res=560) + self.model = romatch.roma_outdoor( + device=device, coarse_res=560, upsample_res=self.output_res + ) elif mode == "indoor": - self.model = romatch.roma_indoor(device=device, coarse_res=560) + self.model = romatch.roma_indoor( + device=device, coarse_res=560, upsample_res=self.output_res + ) elif mode == "tiny_outdoor": self.model = romatch.tiny_roma_v1_outdoor(device=device) def get_sample_thresh(self): return self.model.sample_thresh - def get_warpping_symmetric(self, img1, img2): + def get_warping_symmetric(self, img1, img2): warp, certainty = self.model.match( - Image.fromarray(img1), Image.fromarray(img2) + Image.fromarray(img1), Image.fromarray(img2), batched=False ) - N = 864 + if self.mode.startswith("tiny"): + warp2_to_1, certainty2_to_1 = self.model.match( + Image.fromarray(img2), Image.fromarray(img1), batched=False + ) + return warp[:, :, 2:], certainty, warp2_to_1[:, :, 2:], certainty2_to_1 return ( - warp[:, :N, 2:], - certainty[:, :N], - warp[:, N:, :2], - certainty[:, N:], + warp[:, : self.output_res, 2:], + certainty[:, : self.output_res], + warp[:, self.output_res :, :2], + certainty[:, self.output_res :], ) diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py index fb2548d2..10aec815 100644 --- a/limap/line2d/dense/matcher.py +++ b/limap/line2d/dense/matcher.py @@ -158,7 +158,7 @@ def match_segs_with_descinfo(self, descinfo1, descinfo2): cert_1to2, warp_2to1, cert_2to1, - ) = self.dense_matcher.get_warpping_symmetric(img1, img2) + ) = self.dense_matcher.get_warping_symmetric(img1, img2) # compute distance and overlap dists_1to2, overlap_1to2 = self.compute_distance_one_direction( From 71b9a07039dae50f76748f0980922356103cddda Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Thu, 24 Oct 2024 15:17:32 +0200 Subject: [PATCH 10/20] formattin. --- limap/line2d/dense/dense_matcher/roma.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/limap/line2d/dense/dense_matcher/roma.py b/limap/line2d/dense/dense_matcher/roma.py index 74f36ec2..1b881505 100644 --- a/limap/line2d/dense/dense_matcher/roma.py +++ b/limap/line2d/dense/dense_matcher/roma.py @@ -33,10 +33,16 @@ def get_warping_symmetric(self, img1, img2): warp2_to_1, certainty2_to_1 = self.model.match( Image.fromarray(img2), Image.fromarray(img1), batched=False ) - return warp[:, :, 2:], certainty, warp2_to_1[:, :, 2:], certainty2_to_1 - return ( - warp[:, : self.output_res, 2:], - certainty[:, : self.output_res], - warp[:, self.output_res :, :2], - certainty[:, self.output_res :], - ) + return ( + warp[:, :, 2:], + certainty, + warp2_to_1[:, :, 2:], + certainty2_to_1, + ) + else: + return ( + warp[:, : self.output_res, 2:], + certainty[:, : self.output_res], + warp[:, self.output_res :, :2], + certainty[:, self.output_res :], + ) From 9df66fc4b0bfe5b14cc12116dd6b36d05e10ffcd Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Thu, 24 Oct 2024 15:24:59 +0200 Subject: [PATCH 11/20] refactor. set overlap threshold to 0.2 --- limap/line2d/dense/matcher.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py index 10aec815..47d067f9 100644 --- a/limap/line2d/dense/matcher.py +++ b/limap/line2d/dense/matcher.py @@ -13,7 +13,7 @@ class BaseDenseLineMatcherOptions(NamedTuple): n_samples: int = 21 - segment_percentage_th: float = 0.5 + segment_percentage_th: float = 0.2 device = "cuda" pixel_th: float = 10.0 @@ -111,19 +111,19 @@ def compute_distance_one_direction( ) coords_proj = torch.matmul(coords_to_2, directions.T) dists = torch.abs(torch.matmul(coords_to_2_homo, lines2_homo.T)) - overlap = torch.where( + has_overlap = torch.where( coords_proj > starts2_proj, torch.ones_like(dists), torch.zeros_like(dists), ) - overlap = torch.where( - coords_proj < ends2_proj, overlap, torch.zeros_like(dists) + has_overlap = torch.where( + coords_proj < ends2_proj, has_overlap, torch.zeros_like(dists) ) dists = dists.reshape( n_segs1, self.dense_options.n_samples, n_segs2 ).permute(0, 2, 1) - overlap = ( - overlap.reshape(n_segs1, self.dense_options.n_samples, n_segs2) + has_overlap = ( + has_overlap.reshape(n_segs1, self.dense_options.n_samples, n_segs2) .permute(0, 2, 1) .to(torch.bool) ) @@ -132,23 +132,22 @@ def compute_distance_one_direction( sample_thresh = self.dense_matcher.get_sample_thresh() good_sample = cert_to_2 > sample_thresh good_sample = torch.logical_and( - good_sample[:, None, :].repeat(1, overlap.shape[1], 1), overlap + good_sample[:, None, :].repeat(1, has_overlap.shape[1], 1), + has_overlap, ) sample_weight = good_sample.to(torch.float) sample_weight_sum = sample_weight.sum(2) + overlap = sample_weight_sum / self.dense_options.n_samples sample_weight[sample_weight_sum > 0] /= sample_weight_sum[ sample_weight_sum > 0 ][:, None].repeat(1, sample_weight.shape[2]) - # is_active = ( - # sample_weight_sum - # > self.dense_options.segment_percentage_th - # * self.dense_options.n_samples - # ) # get weighted dists weighted_dists = (dists * sample_weight).sum(2) - weighted_dists[weighted_dists == 0] = 10000.0 - return weighted_dists, sample_weight_sum / self.dense_options.n_samples + weighted_dists[overlap < self.dense_options.segment_percentage_th] = ( + 10000.0 # ensure that there is overlap + ) + return weighted_dists, overlap def match_segs_with_descinfo(self, descinfo1, descinfo2): img1 = descinfo1["camview"].read_image() From 8f2b91ada620505836a87d0c333042da94667c10 Mon Sep 17 00:00:00 2001 From: Remi Pautrat Date: Sat, 26 Oct 2024 11:55:13 +0200 Subject: [PATCH 12/20] Minor simplifications --- limap/line2d/dense/matcher.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py index 47d067f9..09548d6d 100644 --- a/limap/line2d/dense/matcher.py +++ b/limap/line2d/dense/matcher.py @@ -52,15 +52,13 @@ def compute_distance_one_direction( self.dense_options.device ) n_segs1 = segs1.shape[0] - ratio = torch.arange( - 0, - 1 + 0.5 / (self.dense_options.n_samples - 1), - 1.0 / (self.dense_options.n_samples - 1), - ).to(self.dense_options.device) + ratio = torch.linspace( + 0, 1, self.dense_options.n_samples, device=self.dense_options.device + ) ratio = ratio[:, None].repeat(1, 2) - coords_1 = ratio * segs1[:, [0], :].repeat( + coords_1 = ratio * segs1[:, [0]].repeat( 1, self.dense_options.n_samples, 1 - ) + (1 - ratio) * segs1[:, [1], :].repeat( + ) + (1 - ratio) * segs1[:, [1]].repeat( 1, self.dense_options.n_samples, 1 ) coords_1 = coords_1.reshape(-1, 2) @@ -79,7 +77,7 @@ def compute_distance_one_direction( descinfo2["image_shape"][1], ) cert_to_2 = F.grid_sample( - cert_1to2[None, None, ...], + cert_1to2[None, None], coords[None, None], align_corners=False, mode="bilinear", @@ -91,7 +89,7 @@ def compute_distance_one_direction( self.dense_options.device ) n_segs2 = segs2.shape[0] - starts2, ends2 = segs2[:, 0, :], segs2[:, 1, :] + starts2, ends2 = segs2[:, 0], segs2[:, 1] directions = ends2 - starts2 directions /= torch.norm(directions, dim=1, keepdim=True) starts2_proj = (starts2 * directions).sum(1) @@ -132,7 +130,7 @@ def compute_distance_one_direction( sample_thresh = self.dense_matcher.get_sample_thresh() good_sample = cert_to_2 > sample_thresh good_sample = torch.logical_and( - good_sample[:, None, :].repeat(1, has_overlap.shape[1], 1), + good_sample[:, None].repeat(1, has_overlap.shape[1], 1), has_overlap, ) sample_weight = good_sample.to(torch.float) From b98ade9a8f429b226536473cf42df69e7fd8cdd6 Mon Sep 17 00:00:00 2001 From: Remi Pautrat Date: Mon, 28 Oct 2024 08:25:35 +0100 Subject: [PATCH 13/20] RoMa mode in config --- limap/line2d/dense/dense_matcher/roma.py | 2 ++ limap/line2d/register_matcher.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/limap/line2d/dense/dense_matcher/roma.py b/limap/line2d/dense/dense_matcher/roma.py index 1b881505..5842e4e7 100644 --- a/limap/line2d/dense/dense_matcher/roma.py +++ b/limap/line2d/dense/dense_matcher/roma.py @@ -21,6 +21,8 @@ def __init__(self, mode="outdoor", device="cuda"): ) elif mode == "tiny_outdoor": self.model = romatch.tiny_roma_v1_outdoor(device=device) + else: + raise ValueError(f"Unknown mode for RoMa: {mode}") def get_sample_thresh(self): return self.model.sample_thresh diff --git a/limap/line2d/register_matcher.py b/limap/line2d/register_matcher.py index f4ace620..6c55bf98 100644 --- a/limap/line2d/register_matcher.py +++ b/limap/line2d/register_matcher.py @@ -51,6 +51,8 @@ def get_matcher(cfg_matcher, extractor, n_neighbors=20, weight_path=None): elif method == "dense_roma": from .dense import RoMaLineMatcher - return RoMaLineMatcher(extractor, options=options) + return RoMaLineMatcher( + extractor, options=options, mode=cfg_matcher["dense_roma"]["mode"] + ) else: raise NotImplementedError From 2e967afc11c88708f8a58f8e8fb6428072a1ffd8 Mon Sep 17 00:00:00 2001 From: Remi Pautrat Date: Mon, 28 Oct 2024 09:11:56 +0100 Subject: [PATCH 14/20] One-to-many matching --- limap/line2d/dense/__init__.py | 2 +- limap/line2d/dense/dense_matcher/roma.py | 8 +++---- limap/line2d/dense/matcher.py | 16 ++++++++----- limap/line2d/register_matcher.py | 12 ++++++++-- scripts/test_matching.py | 30 ++++++++++++++++++------ 5 files changed, 48 insertions(+), 20 deletions(-) diff --git a/limap/line2d/dense/__init__.py b/limap/line2d/dense/__init__.py index b22b844f..d14f2016 100644 --- a/limap/line2d/dense/__init__.py +++ b/limap/line2d/dense/__init__.py @@ -1,2 +1,2 @@ from .extractor import DenseNaiveExtractor -from .matcher import RoMaLineMatcher +from .matcher import BaseDenseLineMatcherOptions, RoMaLineMatcher diff --git a/limap/line2d/dense/dense_matcher/roma.py b/limap/line2d/dense/dense_matcher/roma.py index 5842e4e7..da556155 100644 --- a/limap/line2d/dense/dense_matcher/roma.py +++ b/limap/line2d/dense/dense_matcher/roma.py @@ -28,12 +28,12 @@ def get_sample_thresh(self): return self.model.sample_thresh def get_warping_symmetric(self, img1, img2): - warp, certainty = self.model.match( - Image.fromarray(img1), Image.fromarray(img2), batched=False - ) + pil_img1 = Image.fromarray(img1) + pil_img2 = Image.fromarray(img2) + warp, certainty = self.model.match(pil_img1, pil_img2, batched=False) if self.mode.startswith("tiny"): warp2_to_1, certainty2_to_1 = self.model.match( - Image.fromarray(img2), Image.fromarray(img1), batched=False + pil_img2, pil_img1, batched=False ) return ( warp[:, :, 2:], diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py index 09548d6d..69dfa054 100644 --- a/limap/line2d/dense/matcher.py +++ b/limap/line2d/dense/matcher.py @@ -16,6 +16,7 @@ class BaseDenseLineMatcherOptions(NamedTuple): segment_percentage_th: float = 0.2 device = "cuda" pixel_th: float = 10.0 + one_to_many: bool = False class BaseDenseLineMatcher(BaseMatcher): @@ -98,7 +99,7 @@ def compute_distance_one_direction( # get line equations starts_homo = torch.cat([starts2, torch.ones_like(segs2[:, [0], 0])], 1) ends_homo = torch.cat([ends2, torch.ones_like(segs2[:, [0], 0])], 1) - lines2_homo = torch.cross(starts_homo, ends_homo) + lines2_homo = torch.cross(starts_homo, ends_homo, dim=1) lines2_homo /= torch.norm(lines2_homo[:, :2], dim=1)[:, None].repeat( 1, 3 ) @@ -169,12 +170,15 @@ def match_segs_with_descinfo(self, descinfo1, descinfo2): overlap_1to2 > overlap_2to1.T, dists_1to2, dists_2to1.T ) - # match: one-way nearest neighbor - # TODO: one-to-many matching + # match + best_matches = dists <= self.dense_options.pixel_th + if not self.dense_options.one_to_many: + # one-to-one matching + best_matches = best_matches * ( + dists == dists.min(dim=-1, keepdim=True).values + ) inds_1, inds_2 = torch.nonzero( - dists - == dists.min(dim=-1, keepdim=True).values - * (dists <= self.dense_options.pixel_th), + best_matches, as_tuple=True, ) inds_1 = inds_1.detach().cpu().numpy() diff --git a/limap/line2d/register_matcher.py b/limap/line2d/register_matcher.py index 6c55bf98..174b6297 100644 --- a/limap/line2d/register_matcher.py +++ b/limap/line2d/register_matcher.py @@ -49,10 +49,18 @@ def get_matcher(cfg_matcher, extractor, n_neighbors=20, weight_path=None): return GlueStickMatcher(extractor, options) elif method == "dense_roma": - from .dense import RoMaLineMatcher + from .dense import BaseDenseLineMatcherOptions, RoMaLineMatcher + dense_options = BaseDenseLineMatcherOptions() + if "one_to_many" in cfg_matcher: + dense_options = dense_options._replace( + one_to_many=cfg_matcher["one_to_many"] + ) return RoMaLineMatcher( - extractor, options=options, mode=cfg_matcher["dense_roma"]["mode"] + extractor, + options=options, + dense_options=dense_options, + mode=cfg_matcher["dense_roma"]["mode"], ) else: raise NotImplementedError diff --git a/scripts/test_matching.py b/scripts/test_matching.py index abd042ea..c3356002 100644 --- a/scripts/test_matching.py +++ b/scripts/test_matching.py @@ -1,4 +1,8 @@ +import os +import time + import cv2 +import torch import limap.base import limap.line2d @@ -11,22 +15,34 @@ {"method": "dense_naive", "skip_exists": False} ) # get a line extractor matcher = limap.line2d.get_matcher( - {"method": "dense_roma", "skip_exists": False, "n_jobs": 1, "topk": 0}, + { + "method": "dense_roma", + "dense_roma": {"mode": "outdoor"}, + "one_to_many": False, + "skip_exists": False, + "n_jobs": 1, + "topk": 0, + }, extractor, ) # initiate a line matcher +current_dir = os.path.abspath(os.path.dirname(__file__)) view1 = limap.base.CameraView( - "/home/shaoliu/workspace/GlueStick/resources/img1.jpg" + os.path.join(current_dir, "../third-party/GlueStick/resources/img1.jpg") ) # initiate an limap.base.CameraView instance for detection. view2 = limap.base.CameraView( - "/home/shaoliu/workspace/GlueStick/resources/img2.jpg" + os.path.join(current_dir, "../third-party/GlueStick/resources/img2.jpg") ) # initiate an limap.base.CameraView instance for detection. segs1 = detector.detect(view1) # detection desc1 = extractor.extract(view1, segs1) # description segs2 = detector.detect(view2) # detection desc2 = extractor.extract(view2, segs2) # description +torch.cuda.synchronize() +start = time.time() matches = matcher.match_pair(desc1, desc2) # matching +torch.cuda.synchronize() +print(f"Matching time: {time.time() - start:.3f}s") def vis_detections(img, segs): @@ -75,9 +91,9 @@ def vis_matches(img1, img2, segs1, segs2, matches): img1 = view1.read_image() img2 = view2.read_image() img1_det = vis_detections(img1, segs1) -cv2.imwrite("tmp/img1_det.png", img1_det) +cv2.imwrite("/tmp/img1_det.png", img1_det) img2_det = vis_detections(img2, segs2) -cv2.imwrite("tmp/img2_det.png", img2_det) +cv2.imwrite("/tmp/img2_det.png", img2_det) img1_draw, img2_draw = vis_matches(img1, img2, segs1, segs2, matches) -cv2.imwrite("tmp/img1_draw.png", img1_draw) -cv2.imwrite("tmp/img2_draw.png", img2_draw) +cv2.imwrite("/tmp/img1_draw.png", img1_draw) +cv2.imwrite("/tmp/img2_draw.png", img2_draw) From f20ed216a25582247136fc41eced094b1c8264c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Pautrat?= Date: Tue, 5 Nov 2024 08:39:06 +0100 Subject: [PATCH 15/20] Mutual nearest neighbors + small fixes --- limap/line2d/dense/extractor.py | 4 +- limap/line2d/dense/matcher.py | 67 ++++++++++++++++++++++++++------ limap/line2d/register_matcher.py | 2 +- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/limap/line2d/dense/extractor.py b/limap/line2d/dense/extractor.py index 2668ce38..dd48b51e 100644 --- a/limap/line2d/dense/extractor.py +++ b/limap/line2d/dense/extractor.py @@ -29,13 +29,13 @@ def read_descinfo(self, descinfo_folder, img_id): return descinfo def extract(self, camview, segs): - img = camview.read_image(set_gray=self.set_gray) + img = camview.read_image() lines = segs[:, :4].reshape(-1, 2, 2) scores = segs[:, -1] * np.sqrt( np.linalg.norm(segs[:, :2] - segs[:, 2:4], axis=1) ) descinfo = { - "camview": camview, + "image": img, "image_shape": img.shape, "lines": lines, "scores": scores, diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py index 69dfa054..1161ea1e 100644 --- a/limap/line2d/dense/matcher.py +++ b/limap/line2d/dense/matcher.py @@ -1,12 +1,9 @@ -import os from typing import NamedTuple import numpy as np import torch import torch.nn.functional as F -import limap.util.io as limapio - from ..base_matcher import BaseMatcher, BaseMatcherOptions from .dense_matcher import BaseDenseMatcher @@ -49,8 +46,10 @@ def compute_distance_one_direction( self, descinfo1, descinfo2, warp_1to2, cert_1to2 ): # get point samples along lines - segs1 = torch.from_numpy(descinfo1["lines"]).to( - self.dense_options.device + segs1 = ( + torch.from_numpy(descinfo1["lines"]) + .to(self.dense_options.device) + .float() ) n_segs1 = segs1.shape[0] ratio = torch.linspace( @@ -86,8 +85,10 @@ def compute_distance_one_direction( cert_to_2 = cert_to_2.reshape(-1, self.dense_options.n_samples) # get projections - segs2 = torch.from_numpy(descinfo2["lines"]).to( - self.dense_options.device + segs2 = ( + torch.from_numpy(descinfo2["lines"]) + .to(self.dense_options.device) + .float() ) n_segs2 = segs2.shape[0] starts2, ends2 = segs2[:, 0], segs2[:, 1] @@ -149,14 +150,14 @@ def compute_distance_one_direction( return weighted_dists, overlap def match_segs_with_descinfo(self, descinfo1, descinfo2): - img1 = descinfo1["camview"].read_image() - img2 = descinfo2["camview"].read_image() ( warp_1to2, cert_1to2, warp_2to1, cert_2to1, - ) = self.dense_matcher.get_warping_symmetric(img1, img2) + ) = self.dense_matcher.get_warping_symmetric( + descinfo1["image"], descinfo2["image"] + ) # compute distance and overlap dists_1to2, overlap_1to2 = self.compute_distance_one_direction( @@ -165,10 +166,16 @@ def match_segs_with_descinfo(self, descinfo1, descinfo2): dists_2to1, overlap_2to1 = self.compute_distance_one_direction( descinfo2, descinfo1, warp_2to1, cert_2to1 ) - # overlap = torch.maximum(overlap_1to2, overlap_2to1.T) dists = torch.where( overlap_1to2 > overlap_2to1.T, dists_1to2, dists_2to1.T ) + # Both warps must have a minimum overlap and close distance (= equivalent of mutual nearest neighbor) + overlap = torch.minimum(overlap_1to2, overlap_2to1.T) + dists[overlap < self.dense_options.segment_percentage_th] = 10000 + dists[ + torch.maximum(dists_1to2, dists_2to1.T) + > self.dense_options.pixel_th + ] = 10000 # match best_matches = dists <= self.dense_options.pixel_th @@ -187,7 +194,43 @@ def match_segs_with_descinfo(self, descinfo1, descinfo2): return matches_t def match_segs_with_descinfo_topk(self, descinfo1, descinfo2, topk=10): - raise NotImplementedError + ( + warp_1to2, + cert_1to2, + warp_2to1, + cert_2to1, + ) = self.dense_matcher.get_warping_symmetric( + descinfo1["image"], descinfo2["image"] + ) + + # compute distance and overlap + dists_1to2, overlap_1to2 = self.compute_distance_one_direction( + descinfo1, descinfo2, warp_1to2, cert_1to2 + ) + dists_2to1, overlap_2to1 = self.compute_distance_one_direction( + descinfo2, descinfo1, warp_2to1, cert_2to1 + ) + dists = torch.where( + overlap_1to2 > overlap_2to1.T, dists_1to2, dists_2to1.T + ) + # Both warps must have a minimum overlap and close distance (= equivalent of mutual nearest neighbor) + overlap = torch.minimum(overlap_1to2, overlap_2to1.T) + dists[overlap < self.dense_options.segment_percentage_th] = 10000 + dists[ + torch.maximum(dists_1to2, dists_2to1.T) + > self.dense_options.pixel_th + ] = 10000 + + # match + best_matches = dists <= self.dense_options.pixel_th + inds_1, inds_2 = torch.nonzero( + best_matches, + as_tuple=True, + ) + inds_1 = inds_1.detach().cpu().numpy() + inds_2 = inds_2.detach().cpu().numpy() + matches_t = np.stack([inds_1, inds_2], axis=1) + return matches_t class RoMaLineMatcher(BaseDenseLineMatcher): diff --git a/limap/line2d/register_matcher.py b/limap/line2d/register_matcher.py index 174b6297..a9d493c5 100644 --- a/limap/line2d/register_matcher.py +++ b/limap/line2d/register_matcher.py @@ -60,7 +60,7 @@ def get_matcher(cfg_matcher, extractor, n_neighbors=20, weight_path=None): extractor, options=options, dense_options=dense_options, - mode=cfg_matcher["dense_roma"]["mode"], + mode=cfg_matcher["mode"], ) else: raise NotImplementedError From 9141712bf9a24ec8c2c57bb3ffdd8d29c24dbabb Mon Sep 17 00:00:00 2001 From: B1ueber2y Date: Sun, 24 Nov 2024 22:59:26 +0100 Subject: [PATCH 16/20] merge and fix linting issues. --- limap/line2d/dense/__init__.py | 4 +++- limap/line2d/dense/dense_matcher/__init__.py | 2 ++ limap/line2d/dense/dense_matcher/base.py | 2 -- limap/line2d/dense/dense_matcher/roma.py | 2 -- limap/line2d/dense/extractor.py | 4 ++-- limap/line2d/dense/matcher.py | 19 ++++++++++++------- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/limap/line2d/dense/__init__.py b/limap/line2d/dense/__init__.py index d14f2016..a54de7e5 100644 --- a/limap/line2d/dense/__init__.py +++ b/limap/line2d/dense/__init__.py @@ -1,2 +1,4 @@ from .extractor import DenseNaiveExtractor -from .matcher import BaseDenseLineMatcherOptions, RoMaLineMatcher +from .matcher import RoMaLineMatcher + +__all__ = ["DenseNaiveExtractor", "RoMaLineMatcher"] diff --git a/limap/line2d/dense/dense_matcher/__init__.py b/limap/line2d/dense/dense_matcher/__init__.py index c9a48134..7683afaa 100644 --- a/limap/line2d/dense/dense_matcher/__init__.py +++ b/limap/line2d/dense/dense_matcher/__init__.py @@ -1,2 +1,4 @@ from .base import BaseDenseMatcher from .roma import RoMa + +__all__ = ["BaseDenseMatcher", "RoMa"] diff --git a/limap/line2d/dense/dense_matcher/base.py b/limap/line2d/dense/dense_matcher/base.py index 20309786..6a74acce 100644 --- a/limap/line2d/dense/dense_matcher/base.py +++ b/limap/line2d/dense/dense_matcher/base.py @@ -1,5 +1,3 @@ -import os - import torch diff --git a/limap/line2d/dense/dense_matcher/roma.py b/limap/line2d/dense/dense_matcher/roma.py index da556155..0a327ae8 100644 --- a/limap/line2d/dense/dense_matcher/roma.py +++ b/limap/line2d/dense/dense_matcher/roma.py @@ -1,5 +1,3 @@ -import os - import romatch from PIL import Image diff --git a/limap/line2d/dense/extractor.py b/limap/line2d/dense/extractor.py index dd48b51e..5fff8818 100644 --- a/limap/line2d/dense/extractor.py +++ b/limap/line2d/dense/extractor.py @@ -4,11 +4,11 @@ import limap.util.io as limapio -from ..base_detector import BaseDetector, BaseDetectorOptions +from ..base_detector import BaseDetector, DefaultDetectorOptions class DenseNaiveExtractor(BaseDetector): - def __init__(self, options=BaseDetectorOptions(), device=None): + def __init__(self, options=DefaultDetectorOptions, device=None): super().__init__(options) def get_module_name(self): diff --git a/limap/line2d/dense/matcher.py b/limap/line2d/dense/matcher.py index 1161ea1e..256f51d0 100644 --- a/limap/line2d/dense/matcher.py +++ b/limap/line2d/dense/matcher.py @@ -4,7 +4,7 @@ import torch import torch.nn.functional as F -from ..base_matcher import BaseMatcher, BaseMatcherOptions +from ..base_matcher import BaseMatcher, DefaultMatcherOptions from .dense_matcher import BaseDenseMatcher @@ -16,13 +16,16 @@ class BaseDenseLineMatcherOptions(NamedTuple): one_to_many: bool = False +DefaultDenseLineMatcherOptions = BaseDenseLineMatcherOptions() + + class BaseDenseLineMatcher(BaseMatcher): def __init__( self, extractor, dense_matcher, - dense_options=BaseDenseLineMatcherOptions(), - options=BaseMatcherOptions(), + dense_options=DefaultDenseLineMatcherOptions, + options=DefaultMatcherOptions, ): super().__init__(extractor, options) assert self.extractor.get_module_name() == "dense_naive" @@ -169,7 +172,8 @@ def match_segs_with_descinfo(self, descinfo1, descinfo2): dists = torch.where( overlap_1to2 > overlap_2to1.T, dists_1to2, dists_2to1.T ) - # Both warps must have a minimum overlap and close distance (= equivalent of mutual nearest neighbor) + # Both warps must have a minimum overlap and close distance + # (= equivalent of mutual nearest neighbor) overlap = torch.minimum(overlap_1to2, overlap_2to1.T) dists[overlap < self.dense_options.segment_percentage_th] = 10000 dists[ @@ -213,7 +217,8 @@ def match_segs_with_descinfo_topk(self, descinfo1, descinfo2, topk=10): dists = torch.where( overlap_1to2 > overlap_2to1.T, dists_1to2, dists_2to1.T ) - # Both warps must have a minimum overlap and close distance (= equivalent of mutual nearest neighbor) + # Both warps must have a minimum overlap and close distance + # (= equivalent of mutual nearest neighbor) overlap = torch.minimum(overlap_1to2, overlap_2to1.T) dists[overlap < self.dense_options.segment_percentage_th] = 10000 dists[ @@ -238,8 +243,8 @@ def __init__( self, extractor, mode="outdoor", - dense_options=BaseDenseLineMatcherOptions(), - options=BaseMatcherOptions(), + dense_options=DefaultDenseLineMatcherOptions, + options=DefaultMatcherOptions, ): from .dense_matcher import RoMa From 9402743af120715234a4473f48169bc108bab517 Mon Sep 17 00:00:00 2001 From: pautratrmi Date: Sun, 15 Dec 2024 18:18:31 +0100 Subject: [PATCH 17/20] Make Gluestick install editable --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 70c6c17f..7fd51748 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,5 +27,5 @@ clang-format==19.1.0 pytlsd@git+https://github.com/iago-suarez/pytlsd.git@37ac583 deeplsd@git+https://github.com/cvg/DeepLSD.git@88c589d -gluestick@git+https://github.com/cvg/GlueStick.git@0f28efd +-e git+https://github.com/cvg/GlueStick.git@0f28efd#egg=gluestick -e git+https://github.com/B1ueber2y/Hierarchical-Localization.git@f91076b#egg=hloc From b7a7b0f2507c28792daa73b03f570716fcbc237a Mon Sep 17 00:00:00 2001 From: pautratrmi Date: Sun, 15 Dec 2024 18:18:51 +0100 Subject: [PATCH 18/20] Update the dense matcher configuration --- cfgs/localization/default.yaml | 3 +++ cfgs/triangulation/default.yaml | 3 +++ limap/line2d/dense/__init__.py | 4 ++-- limap/line2d/register_matcher.py | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cfgs/localization/default.yaml b/cfgs/localization/default.yaml index 686c5638..2b3ea25d 100644 --- a/cfgs/localization/default.yaml +++ b/cfgs/localization/default.yaml @@ -123,6 +123,9 @@ line2d: skip_exists: False superglue: weights: "outdoor" # ["indoor", "outdoor"] for selecting superglue models + dense: + one_to_many: False + weights: "outdoor" # ["indoor", "outdoor", "tiny_outdoor"] for selecting RoMa models var2d: # in pixels sold2: 5.0 lsd: 2.0 diff --git a/cfgs/triangulation/default.yaml b/cfgs/triangulation/default.yaml index 0524c674..b9b2da74 100644 --- a/cfgs/triangulation/default.yaml +++ b/cfgs/triangulation/default.yaml @@ -55,6 +55,9 @@ line2d: skip_exists: False superglue: weights: "outdoor" # ["indoor", "outdoor"] for selecting superglue models + dense: + one_to_many: False + weights: "outdoor" # ["indoor", "outdoor", "tiny_outdoor"] for selecting RoMa models var2d: # in pixels sold2: 5.0 lsd: 2.0 diff --git a/limap/line2d/dense/__init__.py b/limap/line2d/dense/__init__.py index a54de7e5..c9d9e17d 100644 --- a/limap/line2d/dense/__init__.py +++ b/limap/line2d/dense/__init__.py @@ -1,4 +1,4 @@ from .extractor import DenseNaiveExtractor -from .matcher import RoMaLineMatcher +from .matcher import BaseDenseLineMatcherOptions, RoMaLineMatcher -__all__ = ["DenseNaiveExtractor", "RoMaLineMatcher"] +__all__ = ["BaseDenseLineMatcherOptions", "DenseNaiveExtractor", "RoMaLineMatcher"] diff --git a/limap/line2d/register_matcher.py b/limap/line2d/register_matcher.py index e68bf68f..2b86f5d0 100644 --- a/limap/line2d/register_matcher.py +++ b/limap/line2d/register_matcher.py @@ -55,13 +55,13 @@ def get_matcher(cfg_matcher, extractor, n_neighbors=20, weight_path=None): dense_options = BaseDenseLineMatcherOptions() if "one_to_many" in cfg_matcher: dense_options = dense_options._replace( - one_to_many=cfg_matcher["one_to_many"] + one_to_many=cfg_matcher["dense"]["one_to_many"] ) return RoMaLineMatcher( extractor, options=options, dense_options=dense_options, - mode=cfg_matcher["mode"], + mode=cfg_matcher["dense"]["weights"], ) else: raise NotImplementedError From 8b5af66adfed95052feedf60ecbbdbc205e4dd6d Mon Sep 17 00:00:00 2001 From: Remi Pautrat Date: Sun, 15 Dec 2024 18:28:02 +0100 Subject: [PATCH 19/20] Format --- limap/line2d/dense/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/limap/line2d/dense/__init__.py b/limap/line2d/dense/__init__.py index c9d9e17d..7bca4416 100644 --- a/limap/line2d/dense/__init__.py +++ b/limap/line2d/dense/__init__.py @@ -1,4 +1,8 @@ from .extractor import DenseNaiveExtractor from .matcher import BaseDenseLineMatcherOptions, RoMaLineMatcher -__all__ = ["BaseDenseLineMatcherOptions", "DenseNaiveExtractor", "RoMaLineMatcher"] +__all__ = [ + "BaseDenseLineMatcherOptions", + "DenseNaiveExtractor", + "RoMaLineMatcher", +] From 12c9684726558c5fa9e99acc15bc6a86ed01298f Mon Sep 17 00:00:00 2001 From: Remi Pautrat Date: Mon, 16 Dec 2024 09:09:13 +0100 Subject: [PATCH 20/20] Revert editable GlueStick --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fd51748..70c6c17f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,5 +27,5 @@ clang-format==19.1.0 pytlsd@git+https://github.com/iago-suarez/pytlsd.git@37ac583 deeplsd@git+https://github.com/cvg/DeepLSD.git@88c589d --e git+https://github.com/cvg/GlueStick.git@0f28efd#egg=gluestick +gluestick@git+https://github.com/cvg/GlueStick.git@0f28efd -e git+https://github.com/B1ueber2y/Hierarchical-Localization.git@f91076b#egg=hloc