How can I use SIFT with LightGlue?

[RoyAmoyal]

Hey, I want to use SIFT features with LightGlue matching, how can I do that?

Thanks!

[edgarriba]

@ducha-aiki

[ducha-aiki]

@RoyAmoyal

Hi, here is a slightly modified DISK-LightGlue example to use with SIFT:

import io

import requests


def download_image(url: str, filename: str = "") -> str:
    filename = url.split("/")[-1] if len(filename) == 0 else filename
    # Download
    bytesio = io.BytesIO(requests.get(url).content)
    # Save file
    with open(filename, "wb") as outfile:
        outfile.write(bytesio.getbuffer())

    return filename


url_a = "https://github.com/kornia/data/raw/main/matching/kn_church-2.jpg"
url_b = "https://github.com/kornia/data/raw/main/matching/kn_church-8.jpg"
download_image(url_a)
download_image(url_b)


import kornia as K
import kornia.feature as KF
import cv2
from kornia_moons.feature import OpenCVFeatureKornia
from kornia_moons.viz import draw_LAF_matches

import torch
import torch.nn.functional as F



fname1 = "kn_church-2.jpg"
fname2 = "kn_church-8.jpg"

device = K.utils.get_cuda_or_mps_device_if_available()
device = torch.device('cpu') 



img1 = K.io.load_image(fname1, K.io.ImageLoadType.RGB32, device=device)[None, ...]
img2 = K.io.load_image(fname2, K.io.ImageLoadType.RGB32, device=device)[None, ...]


hw1 = torch.tensor(img1.shape[2:], device=device)
hw2 = torch.tensor(img2.shape[2:], device=device)


opencv_sift = OpenCVFeatureKornia(cv2.SIFT_create(4096), mrSize=1.0)

lg_matcher = KF.LightGlueMatcher("sift").eval().to(device)


use_opencv_sift = False
with torch.inference_mode():
    lafs1, r1, descs1 = opencv_sift(img1.to(device))
    lafs2, r2, descs2 = opencv_sift(img2.to(device))
    # LightGlue SIFT is RootSIFT in reality, so we have to convert to RootSIFT
    descs1 = F.normalize(descs1, dim=-1, p=1).sqrt()
    descs2 = F.normalize(descs2, dim=-1, p=1).sqrt()
    dists, idxs = lg_matcher(descs1[0], descs2[0], 
                             lafs1, lafs2, hw1=hw1, hw2=hw2)
    

print(f"{idxs.shape[0]} tentative matches with SIFT LightGlue")

def get_matching_keypoints(kp1, kp2, idxs):
    mkpts1 = kp1[idxs[:, 0]]
    mkpts2 = kp2[idxs[:, 1]]
    return mkpts1, mkpts2


kps1 = KF.get_laf_center(lafs1)[0]
kps2 = KF.get_laf_center(lafs2)[0]

mkpts1, mkpts2 = get_matching_keypoints(KF.get_laf_center(lafs1)[0],
                                        KF.get_laf_center(lafs2)[0], idxs)

Fm, inliers = cv2.findFundamentalMat(
    mkpts1.detach().cpu().numpy(), mkpts2.detach().cpu().numpy(), cv2.USAC_MAGSAC, 1.0, 0.999, 100000
)
inliers = inliers > 0
print(f"{inliers.sum()} inliers with SIFT")

draw_LAF_matches(
    lafs1,
    lafs2,
    idxs.cpu(),
    K.tensor_to_image(img1.cpu()),
    K.tensor_to_image(img2.cpu()),
    inliers,
    draw_dict={"inlier_color": (0.2, 1, 0.2), 
               "tentative_color": (1, 1, 0.2, 0.3), 
               "feature_color": None, "vertical": False},
)