add: minima (#112)

README.md

@@ -5,7 +5,7 @@
 
 <p align="center">
   <h1 align="center"><br><ins>$\color{red}{\textnormal{Image\ Matching\ WebUI}}$
-</ins><br>Identify matching points between two images</h1>
+</ins><br>Matching Keypoints between two images</h1>
 </p>
 
 ## Description
@@ -23,6 +23,7 @@ Here is a demo of the tool:
 https://github.com/Vincentqyw/image-matching-webui/assets/18531182/263534692-c3484d1b-cc00-4fdc-9b31-e5b7af07ecd9
 
 The tool currently supports various popular image matching algorithms, namely:
+- [x] [MINIMA](https://github.com/LSXI7/MINIMA), ARXIV 2024
 - [x] [XoFTR](https://github.com/OnderT/XoFTR), CVPR 2024
 - [x] [EfficientLoFTR](https://github.com/zju3dv/EfficientLoFTR), CVPR 2024
 - [x] [MASt3R](https://github.com/naver/mast3r), CVPR 2024
@@ -80,11 +81,25 @@ Just try it on <a href='https://huggingface.co/spaces/Realcat/image-matching-web
 or deploy it locally following the instructions below.
 
 ### Requirements
+
+- [Python 3.9+](https://www.python.org/downloads/)
+
+#### Install from pip [NEW]
+
+Update: now support install from [pip](https://pypi.org/project/imcui), just run:
+
+```bash
+pip install imcui
+```
+
+#### Install from source
+
 ``` bash
 git clone --recursive https://github.com/Vincentqyw/image-matching-webui.git
 cd image-matching-webui
 conda env create -f environment.yaml
 conda activate imw
+pip install -e .
 ```
 
 or using [docker](https://hub.docker.com/r/vincentqin/image-matching-webui):
@@ -104,7 +119,7 @@ python -m imcui.api.server
 
 ### Run demo
 ``` bash
-python ./app.py --config ./config/config.yaml
+python app.py --config ./config/config.yaml
 ```
 then open http://localhost:7860 in your browser.
 
@@ -114,19 +129,25 @@ then open http://localhost:7860 in your browser.
 
 I provide an example to add local feature in [imcui/hloc/extractors/example.py](imcui/hloc/extractors/example.py). Then add feature settings in `confs` in file [imcui/hloc/extract_features.py](imcui/hloc/extract_features.py). Last step is adding some settings to `matcher_zoo` in file [imcui/ui/config.yaml](imcui/ui/config.yaml).
 
+### Upload models
+
+IMCUI hosts all models on [Huggingface](https://huggingface.co/Realcat/imcui_checkpoints).  You can upload your model to Huggingface and add it to the [Realcat/imcui_checkpoints](https://huggingface.co/Realcat/imcui_checkpoints) repository.
+
+
 ## Contributions welcome!
 
 External contributions are very much welcome. Please follow the [PEP8 style guidelines](https://www.python.org/dev/peps/pep-0008/) using a linter like flake8. This is a non-exhaustive list of features that might be valuable additions:
 
+- [x] support pip install command
 - [x] add [CPU CI](.github/workflows/ci.yml)
 - [x] add webcam support
 - [x] add [line feature matching](https://github.com/Vincentqyw/LineSegmentsDetection) algorithms
 - [x] example to add a new feature extractor / matcher
 - [x] ransac to filter outliers
 - [ ] add [rotation images](https://github.com/pidahbus/deep-image-orientation-angle-detection) options before matching
 - [ ] support export matches to colmap ([#issue 6](https://github.com/Vincentqyw/image-matching-webui/issues/6))
-- [ ] add config file to set default parameters
-- [ ] dynamically load models and reduce GPU overload
+- [x] add config file to set default parameters
+- [x] dynamically load models and reduce GPU overload
 
 Adding local features / matchers as submodules is very easy. For example, to add the [GlueStick](https://github.com/cvg/GlueStick):
 

config/config.yaml

@@ -16,6 +16,23 @@ defaults:
   setting_geometry: Homography
 
 matcher_zoo:
+  minima(loftr):
+    matcher: minima_loftr
+    dense: true
+    info:
+      name: MINIMA(LoFTR) #dispaly name
+      source: "ARXIV 2024"
+      paper: https://arxiv.org/abs/2412.19412
+      display: false
+  minima(RoMa):
+    matcher: minima_roma
+    skip_ci: true
+    dense: true
+    info:
+      name: MINIMA(RoMa) #dispaly name
+      source: "ARXIV 2024"
+      paper: https://arxiv.org/abs/2412.19412
+      display: false
   omniglue:
     enable: true
     matcher: omniglue

imcui/hloc/match_dense.py

@@ -44,11 +44,31 @@
         "max_error": 1,  # max error for assigned keypoints (in px)
         "cell_size": 1,  # size of quantization patch (max 1 kp/patch)
     },
+    "minima_loftr": {
+        "output": "matches-minima_loftr",
+        "model": {
+            "name": "loftr",
+            "weights": "outdoor",
+            "model_name": "minima_loftr.ckpt",
+            "max_keypoints": 2000,
+            "match_threshold": 0.2,
+        },
+        "preprocessing": {
+            "grayscale": True,
+            "resize_max": 1024,
+            "dfactor": 8,
+            "width": 640,
+            "height": 480,
+            "force_resize": False,
+        },
+        "max_error": 1,  # max error for assigned keypoints (in px)
+        "cell_size": 1,  # size of quantization patch (max 1 kp/patch)
+    },
     "eloftr": {
         "output": "matches-eloftr",
         "model": {
             "name": "eloftr",
-            "weights": "weights/eloftr_outdoor.ckpt",
+            "model_name": "eloftr_outdoor.ckpt",
             "max_keypoints": 2000,
             "match_threshold": 0.2,
         },
@@ -282,6 +302,24 @@
             "dfactor": 8,
         },
     },
+    "minima_roma": {
+        "output": "matches-minima_roma",
+        "model": {
+            "name": "roma",
+            "weights": "outdoor",
+            "model_name": "minima_roma.pth",
+            "max_keypoints": 2000,
+            "match_threshold": 0.2,
+        },
+        "preprocessing": {
+            "grayscale": False,
+            "force_resize": False,
+            "resize_max": 1024,
+            "width": 320,
+            "height": 240,
+            "dfactor": 8,
+        },
+    },
     "gim(dkm)": {
         "output": "matches-gim",
         "model": {

imcui/hloc/matchers/loftr.py

@@ -3,8 +3,8 @@
 import torch
 from kornia.feature import LoFTR as LoFTR_
 from kornia.feature.loftr.loftr import default_cfg
-
-from .. import logger
+from pathlib import Path
+from .. import logger, MODEL_REPO_ID
 
 from ..utils.base_model import BaseModel
 
@@ -22,8 +22,21 @@ def _init(self, conf):
         cfg = default_cfg
         cfg["match_coarse"]["thr"] = conf["match_threshold"]
         cfg["match_coarse"]["skh_iters"] = conf["sinkhorn_iterations"]
-        self.net = LoFTR_(pretrained=conf["weights"], config=cfg)
-        logger.info(f"Loaded LoFTR with weights {conf['weights']}")
+
+        model_name = conf.get("model_name", None)
+        if model_name is not None and "minima" in model_name:
+            cfg["coarse"]["temp_bug_fix"] = True
+            model_path = self._download_model(
+                repo_id=MODEL_REPO_ID,
+                filename="{}/{}".format(Path(__file__).stem, self.conf["model_name"]),
+            )
+            state_dict = torch.load(model_path, map_location="cpu")["state_dict"]
+            self.net = LoFTR_(pretrained=conf["weights"], config=cfg)
+            self.net.load_state_dict(state_dict)
+            logger.info(f"ReLoaded LoFTR(minima) with weights {conf['model_name']}")
+        else:
+            self.net = LoFTR_(pretrained=conf["weights"], config=cfg)
+            logger.info(f"Loaded LoFTR with weights {conf['weights']}")
 
     def _forward(self, data):
         # For consistency with hloc pairs, we refine kpts in image0!

imcui/ui/utils.py

@@ -388,23 +388,33 @@ def _filter_matches_opencv(
         Tuple[np.ndarray, np.ndarray]: Homography matrix and mask.
     """
     if geometry_type == "Homography":
-        M, mask = cv2.findHomography(
-            kp0,
-            kp1,
-            method=method,
-            ransacReprojThreshold=reproj_threshold,
-            confidence=confidence,
-            maxIters=max_iter,
-        )
+        try:
+            M, mask = cv2.findHomography(
+                kp0,
+                kp1,
+                method=method,
+                ransacReprojThreshold=reproj_threshold,
+                confidence=confidence,
+                maxIters=max_iter,
+            )
+        except cv2.error:
+            logger.error("compute findHomography error, len(kp0): {}".format(len(kp0)))
+            return None, None
     elif geometry_type == "Fundamental":
-        M, mask = cv2.findFundamentalMat(
-            kp0,
-            kp1,
-            method=method,
-            ransacReprojThreshold=reproj_threshold,
-            confidence=confidence,
-            maxIters=max_iter,
-        )
+        try:
+            M, mask = cv2.findFundamentalMat(
+                kp0,
+                kp1,
+                method=method,
+                ransacReprojThreshold=reproj_threshold,
+                confidence=confidence,
+                maxIters=max_iter,
+            )
+        except cv2.error:
+            logger.error(
+                "compute findFundamentalMat error, len(kp0): {}".format(len(kp0))
+            )
+            return None, None
     mask = np.array(mask.ravel().astype("bool"), dtype="bool")
     return M, mask