diff --git a/belv_sp+lg.ipynb b/belv_sp+lg.ipynb index afad3f7..a0956fe 100644 --- a/belv_sp+lg.ipynb +++ b/belv_sp+lg.ipynb @@ -2,85 +2,209 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using a custom configuration file: /home/francesco/phd/deep-image-matching/datasets/belv_20230725/superpoint+lightglue.yaml\n", + "\u001b[1;33m2024-04-09 18:33:58 | [WARNING ] Extractor name in configuration file /home/francesco/phd/deep-image-matching/datasets/belv_20230725/superpoint+lightglue.yaml does not match with the extractor chosen from CLI or GUI. The custom configuration is not set, but matching is run with the default options.\u001b[0m\n", + "\u001b[1;33m2024-04-09 18:33:58 | [WARNING ] Matcher name in configuration file /home/francesco/phd/deep-image-matching/datasets/belv_20230725/superpoint+lightglue.yaml does not match with the matcher chosen from CLI or GUI. The custom configuration is not set, but matching is run with the default options.\u001b[0m\n", + "Config general:\n", + "{'camera_options': './datasets/belv_20230725/cameras.yaml',\n", + " 'db_path': None,\n", + " 'geom_verification': ,\n", + " 'graph': True,\n", + " 'gv_confidence': 0.99999,\n", + " 'gv_threshold': 4,\n", + " 'image_dir': PosixPath('datasets/belv_20230725/images'),\n", + " 'matching_strategy': 'bruteforce',\n", + " 'min_inlier_ratio_per_pair': 0.2,\n", + " 'min_inliers_per_pair': 2,\n", + " 'min_matches_per_tile': 10,\n", + " 'openmvg_conf': None,\n", + " 'output_dir': PosixPath('datasets/belv_20230725/results_sift+kornia_matcher_bruteforce_quality_high'),\n", + " 'overlap': None,\n", + " 'pair_file': PosixPath('datasets/belv_20230725/results_sift+kornia_matcher_bruteforce_quality_high/pairs.txt'),\n", + " 'quality': ,\n", + " 'refine_intrinsics': False,\n", + " 'retrieval': None,\n", + " 'skip_reconstruction': False,\n", + " 'tile_overlap': 100,\n", + " 'tile_preselection_size': 1000,\n", + " 'tile_selection': ,\n", + " 'tile_size': (3000, 2000),\n", + " 'try_match_full_images': False,\n", + " 'upright': False,\n", + " 'verbose': True}\n", + "\n", + "\n", + "Config extractor:\n", + "{'fix_sampling': True,\n", + " 'keypoint_threshold': 0.0001,\n", + " 'max_keypoints': 10000,\n", + " 'name': 'superpoint',\n", + " 'nms_radius': 4,\n", + " 'remove_borders': 4}\n", + "\n", + "\n", + "Config matcher:\n", + "{'depth_confidence': -1,\n", + " 'filter_threshold': 0.6,\n", + " 'flash': True,\n", + " 'match_mode': 'smnn',\n", + " 'mp': False,\n", + " 'name': 'lightglue',\n", + " 'th': 0.85,\n", + " 'width_confidence': -1}\n", + "\n", + "\n" + ] + } + ], "source": [ - "import logging\n", - "from importlib import import_module\n", - "\n", "import deep_image_matching as dim\n", "import yaml\n", "\n", "logger = dim.setup_logger(\"dim\")\n", "\n", - "cli_params = {\n", - " \"dir\": \"../assets/example_cyprus\",\n", + "params = {\n", + " \"dir\": \"./datasets/belv_20230725\",\n", " \"pipeline\": \"superpoint+lightglue\",\n", - " \"strategy\": \"matching_lowres\",\n", + " \"config_file\": \"./datasets/belv_20230725/superpoint+lightglue.yaml\",\n", + " \"strategy\": \"bruteforce\",\n", " \"quality\": \"high\",\n", - " \"tiling\": \"preselection\",\n", + " \"tiling\": \"none\",\n", " \"skip_reconstruction\": False,\n", " \"force\": True,\n", - " \"camera_options\": \"../config/cameras.yaml\",\n", + " \"camera_options\": \"./datasets/belv_20230725/cameras.yaml\",\n", " \"openmvg\": None,\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Build configuration\n", - "config = dim.Config(cli_params)\n", - "imgs_dir = config.general[\"image_dir\"]\n", - "output_dir = config.general[\"output_dir\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# - General configuration\n", - "config.general[\"min_inliers_per_pair\"] = 10\n", - "config.general[\"min_inlier_ratio_per_pair\"] = 0.2\n", - "\n", - "# - SuperPoint configuration\n", - "config.extractor[\"max_keypoints\"] = 8000\n", + " \"verbose\": True,\n", + "}\n", "\n", - "# - LightGue configuration\n", - "config.matcher[\"filter_threshold\"] = 0.1\n", - "\n", - "# Save configuration to a json file in the output directory\n", + "# Build configuration\n", + "config = dim.Config(params)\n", "config.save()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;30m2024-04-09 18:34:04 | [DEBUG ] Matching options: Quality: HIGH - Tiling: NONE\u001b[0m\n", + "\u001b[1;30m2024-04-09 18:34:04 | [DEBUG ] Saving directory: datasets/belv_20230725/results_sift+kornia_matcher_bruteforce_quality_high\u001b[0m\n", + "\u001b[1;30m2024-04-09 18:34:04 | [DEBUG ] Running inference on device cuda\u001b[0m\n", + "Loaded SuperPoint model\n", + "\u001b[1;30m2024-04-09 18:34:05 | [DEBUG ] Matching options: Tiling: NONE\u001b[0m\n", + "\u001b[1;30m2024-04-09 18:34:05 | [DEBUG ] Saving directory: datasets/belv_20230725/results_sift+kornia_matcher_bruteforce_quality_high\u001b[0m\n", + "\u001b[1;30m2024-04-09 18:34:05 | [DEBUG ] Running inference on device cuda\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Running image matching with the following configuration:\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Image folder: datasets/belv_20230725/images\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Output folder: datasets/belv_20230725/results_sift+kornia_matcher_bruteforce_quality_high\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Number of images: 2\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Matching strategy: bruteforce\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Image quality: HIGH\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Tile selection: NONE\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Feature extraction method: superpoint\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Matching method: lightglue\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Geometric verification: PYDEGENSAC\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] CUDA available: True\u001b[0m\n", + "\u001b[1;30m2024-04-09 18:34:06 | [DEBUG ] Bruteforce matching, generating pairs ..\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Number of pairs: 1\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Found 1 pairs.\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] Extracting features with superpoint...\u001b[0m\n", + "\u001b[0;37m2024-04-09 18:34:06 | [INFO ] superpoint configuration: \u001b[0m\n", + "{'fix_sampling': True,\n", + " 'keypoint_threshold': 0.0001,\n", + " 'max_keypoints': 10000,\n", + " 'name': 'superpoint',\n", + " 'nms_radius': 4,\n", + " 'remove_borders': 4}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/2 [00:03 5\u001b[0m feature_path, match_path \u001b[38;5;241m=\u001b[39m \u001b[43mmatcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# Read camera options\u001b[39;00m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(config\u001b[38;5;241m.\u001b[39mgeneral[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcamera_options\u001b[39m\u001b[38;5;124m\"\u001b[39m], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m file:\n", + "File \u001b[0;32m~/phd/deep-image-matching/src/deep_image_matching/image_matching.py:194\u001b[0m, in \u001b[0;36mImageMatcher.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 191\u001b[0m timer\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrotate_upright_images\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 193\u001b[0m \u001b[38;5;66;03m# Extract features\u001b[39;00m\n\u001b[0;32m--> 194\u001b[0m feature_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mextract_features\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 195\u001b[0m timer\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mextract_features\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 197\u001b[0m \u001b[38;5;66;03m# Matching\u001b[39;00m\n", + "File \u001b[0;32m~/phd/deep-image-matching/src/deep_image_matching/image_matching.py:361\u001b[0m, in \u001b[0;36mImageMatcher.extract_features\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[38;5;66;03m# Extract features\u001b[39;00m\n\u001b[1;32m 360\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m img \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mimage_list):\n\u001b[0;32m--> 361\u001b[0m feature_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_extractor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mextract\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 363\u001b[0m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mempty_cache()\n\u001b[1;32m 364\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFeatures extracted!\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/phd/deep-image-matching/src/deep_image_matching/extractors/extractor_base.py:186\u001b[0m, in \u001b[0;36mExtractorBase.extract\u001b[0;34m(self, img)\u001b[0m\n\u001b[1;32m 182\u001b[0m image_ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_resize_image(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_quality, image, interp\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minterp)\n\u001b[1;32m 184\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgeneral\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtile_selection\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m TileSelection\u001b[38;5;241m.\u001b[39mNONE:\n\u001b[1;32m 185\u001b[0m \u001b[38;5;66;03m# Extract features from the whole image\u001b[39;00m\n\u001b[0;32m--> 186\u001b[0m features \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_extract\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimage_\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 187\u001b[0m \u001b[38;5;66;03m# features[\"feature_path\"] = str(feature_path)\u001b[39;00m\n\u001b[1;32m 188\u001b[0m \u001b[38;5;66;03m# features[\"im_path\"] = str(im_path)\u001b[39;00m\n\u001b[1;32m 189\u001b[0m features[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtile_idx\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros(features[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mkeypoints\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m], dtype\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39mfloat32)\n", + "File \u001b[0;32m~/miniforge3/envs/deep-image-matching/lib/python3.10/site-packages/torch/utils/_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator..decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/phd/deep-image-matching/src/deep_image_matching/extractors/superpoint.py:121\u001b[0m, in \u001b[0;36mSuperPointExtractor._extract\u001b[0;34m(self, image)\u001b[0m\n\u001b[1;32m 118\u001b[0m image_ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_frame2tensor(image, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_device)\n\u001b[1;32m 120\u001b[0m \u001b[38;5;66;03m# Extract features\u001b[39;00m\n\u001b[0;32m--> 121\u001b[0m feats \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_extractor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mimage\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mimage_\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 123\u001b[0m \u001b[38;5;66;03m# Remove elements from list/tuple\u001b[39;00m\n\u001b[1;32m 124\u001b[0m feats \u001b[38;5;241m=\u001b[39m {k: v[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(v, (\u001b[38;5;28mlist\u001b[39m, \u001b[38;5;28mtuple\u001b[39m)) \u001b[38;5;28;01melse\u001b[39;00m v \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m feats\u001b[38;5;241m.\u001b[39mitems()}\n", + "File \u001b[0;32m~/miniforge3/envs/deep-image-matching/lib/python3.10/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniforge3/envs/deep-image-matching/lib/python3.10/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/phd/deep-image-matching/src/deep_image_matching/extractors/superpoint.py:51\u001b[0m, in \u001b[0;36mSuperPoint.forward\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequired_inputs:\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m data, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing key \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m in data\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(key)\n\u001b[0;32m---> 51\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/phd/deep-image-matching/src/deep_image_matching/extractors/superpoint.py:59\u001b[0m, in \u001b[0;36mSuperPoint._forward\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_forward\u001b[39m(\u001b[38;5;28mself\u001b[39m, data):\n\u001b[0;32m---> 59\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnet\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniforge3/envs/deep-image-matching/lib/python3.10/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniforge3/envs/deep-image-matching/lib/python3.10/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/phd/deep-image-matching/src/deep_image_matching/thirdparty/SuperGluePretrainedNetwork/models/superpoint.py:162\u001b[0m, in \u001b[0;36mSuperPoint.forward\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[38;5;66;03m# Shared Encoder\u001b[39;00m\n\u001b[1;32m 161\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv1a(data[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimage\u001b[39m\u001b[38;5;124m\"\u001b[39m]))\n\u001b[0;32m--> 162\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv1b\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 163\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool(x)\n\u001b[1;32m 164\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv2a(x))\n", + "File \u001b[0;32m~/miniforge3/envs/deep-image-matching/lib/python3.10/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1509\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniforge3/envs/deep-image-matching/lib/python3.10/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1518\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1519\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1523\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/miniforge3/envs/deep-image-matching/lib/python3.10/site-packages/torch/nn/modules/conv.py:460\u001b[0m, in \u001b[0;36mConv2d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 459\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 460\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conv_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniforge3/envs/deep-image-matching/lib/python3.10/site-packages/torch/nn/modules/conv.py:456\u001b[0m, in \u001b[0;36mConv2d._conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m 452\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mzeros\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m 453\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv2d(F\u001b[38;5;241m.\u001b[39mpad(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode),\n\u001b[1;32m 454\u001b[0m weight, bias, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstride,\n\u001b[1;32m 455\u001b[0m _pair(\u001b[38;5;241m0\u001b[39m), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdilation, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups)\n\u001b[0;32m--> 456\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv2d\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdilation\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 5.72 GiB. GPU 0 has a total capacity of 11.75 GiB of which 809.00 MiB is free. Process 276581 has 141.49 MiB memory in use. Process 298292 has 2.66 GiB memory in use. Including non-PyTorch memory, this process has 7.20 GiB memory in use. Of the allocated memory 5.86 GiB is allocated by PyTorch, and 17.82 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)" + ] + } + ], "source": [ "# Initialize ImageMatcher class\n", "matcher = dim.ImageMatcher(config)\n", "\n", "# Run image matching\n", - "feature_path, match_path = matcher.run()" + "feature_path, match_path = matcher.run()\n", + "\n", + "# Read camera options\n", + "with open(config.general[\"camera_options\"], \"r\") as file:\n", + " camera_options = yaml.safe_load(file)\n", + "\n", + "# Export in colmap format\n", + "database_path = config.general[\"output_dir\"] / \"database.db\"\n", + "dim.io.export_to_colmap(\n", + " img_dir=config.general[\"image_dir\"],\n", + " feature_path=feature_path,\n", + " match_path=match_path,\n", + " database_path=database_path,\n", + " camera_options=camera_options,\n", + ")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'output_dir' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[1], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Export in colmap format\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m database_path \u001b[38;5;241m=\u001b[39m \u001b[43moutput_dir\u001b[49m \u001b[38;5;241m/\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdatabase.db\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 3\u001b[0m dim\u001b[38;5;241m.\u001b[39mio\u001b[38;5;241m.\u001b[39mexport_to_colmap(\n\u001b[1;32m 4\u001b[0m img_dir\u001b[38;5;241m=\u001b[39mimgs_dir,\n\u001b[1;32m 5\u001b[0m feature_path\u001b[38;5;241m=\u001b[39mfeature_path,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 8\u001b[0m camera_options\u001b[38;5;241m=\u001b[39mcamera_options,\n\u001b[1;32m 9\u001b[0m )\n", + "\u001b[0;31mNameError\u001b[0m: name 'output_dir' is not defined" + ] + } + ], "source": [ "# Export in colmap format\n", - "with open(config.general[\"camera_options\"], \"r\") as file:\n", - " camera_options = yaml.safe_load(file)\n", "database_path = output_dir / \"database.db\"\n", "dim.io.export_to_colmap(\n", " img_dir=imgs_dir,\n", @@ -93,44 +217,182 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ - "from deep_image_matching import graph\n", + "# Set camera options\n", + "\n", + "import numpy as np\n", "\n", - "graph.view_graph(database_path, output_dir, imgs_dir)" + "cameras = {\n", + " 1: {\n", + " \"model\": 4, # 4 is OPENCV model\n", + " \"width\": 6000,\n", + " \"height\": 4000,\n", + " \"params\": [\n", + " 9.26789262766209504e03,\n", + " 9.26789262766209504e03,\n", + " 3.05349107994520591e03,\n", + " 1.94835654532114540e03,\n", + " -8.07042713029020586e-02,\n", + " 9.46617629940955385e-02,\n", + " 3.31782983128223608e-04,\n", + " -4.32106111976037410e-04,\n", + " ],\n", + " },\n", + " 2: {\n", + " \"model\": 4,\n", + " \"width\": 6000,\n", + " \"height\": 4000,\n", + " \"params\": [\n", + " 6.62174345720628298e03,\n", + " 6.62174345720628298e03,\n", + " 3.01324420057086490e03,\n", + " 1.94347461466223308e03,\n", + " -9.41830394356213407e-02,\n", + " 8.55303528514532035e-02,\n", + " 1.68948638308769863e-04,\n", + " -8.74637609310216697e-04,\n", + " ],\n", + " },\n", + "}\n", + "with dim.utils.COLMAPDatabase(database_path) as db:\n", + " for id, cam in cameras.items():\n", + " db.update_camera(\n", + " camera_id=id,\n", + " model=cam[\"model\"],\n", + " width=cam[\"width\"],\n", + " height=cam[\"height\"],\n", + " params=cam[\"params\"],\n", + " )" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 70, "metadata": {}, - "source": [] + "outputs": [], + "source": [ + "# import pycolmap\n", + "\n", + "# print(pycolmap.IncrementalPipelineOptions().summary())" + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 64, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[0;37m2024-04-09 15:15:21 | [INFO ] Running 3D reconstruction...\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I20240409 15:15:21.422063 167579 misc.cc:198] \n", + "==============================================================================\n", + "Loading database\n", + "==============================================================================\n", + "I20240409 15:15:21.422781 167579 database_cache.cc:54] Loading cameras...\n", + "I20240409 15:15:21.422809 167579 database_cache.cc:64] 2 in 0.000s\n", + "I20240409 15:15:21.422822 167579 database_cache.cc:72] Loading matches...\n", + "I20240409 15:15:21.422835 167579 database_cache.cc:78] 1 in 0.000s\n", + "I20240409 15:15:21.422838 167579 database_cache.cc:94] Loading images...\n", + "I20240409 15:15:21.423250 167579 database_cache.cc:143] 2 in 0.000s (connected 2)\n", + "I20240409 15:15:21.423261 167579 database_cache.cc:154] Building correspondence graph...\n", + "I20240409 15:15:21.423429 167579 database_cache.cc:190] in 0.000s (ignored 0)\n", + "I20240409 15:15:21.423439 167579 timer.cc:91] Elapsed time: 0.000 [minutes]\n", + "I20240409 15:15:21.423715 167579 misc.cc:198] \n", + "==============================================================================\n", + "Finding good initial image pair\n", + "==============================================================================\n", + "I20240409 15:15:21.490808 167579 misc.cc:198] \n", + "==============================================================================\n", + "Initializing with image pair #1 and #2\n", + "==============================================================================\n", + "I20240409 15:15:21.491742 167579 misc.cc:198] \n", + "==============================================================================\n", + "Global bundle adjustment\n", + "==============================================================================\n", + "I20240409 15:15:21.502663 167579 misc.cc:205] \n", + "Bundle adjustment report\n", + "------------------------\n", + "I20240409 15:15:21.502694 167579 bundle_adjustment.cc:942] \n", + " Residuals : 1424\n", + " Parameters : 1073\n", + " Iterations : 11\n", + " Time : 0.0105071 [s]\n", + " Initial cost : 1.28933 [px]\n", + " Final cost : 1.26975 [px]\n", + " Termination : Convergence\n", + "\n", + "I20240409 15:15:21.502931 167579 incremental_mapper.cc:160] => Filtered observations: 28\n", + "I20240409 15:15:21.502938 167579 incremental_mapper.cc:167] => Filtered images: 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[0;37m2024-04-09 15:15:21 | [INFO ] Reconstructed 1 model(s).\u001b[0m\n", + "\u001b[0;37m2024-04-09 15:15:21 | [INFO ] Largest model is #0 with 2 images.\u001b[0m\n", + "\u001b[0;37m2024-04-09 15:15:21 | [INFO ] Exporting model...\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I20240409 15:15:21.692776 167579 timer.cc:91] Elapsed time: 0.005 [minutes]\n" + ] + } + ], "source": [ - "reconst_opts = {}\n", + "# Run reconstruction\n", + "opt = dict(\n", + " triangulation=dict(\n", + " ignore_two_view_tracks=False,\n", + " min_angle=0.5,\n", + " ),\n", + " mapper=dict(filter_min_tri_angle=0.5, filter_max_reproj_error=5.0),\n", + ")\n", "refine_intrinsics = False\n", + "verbose = False\n", "\n", - "# Run reconstruction\n", "model = dim.reconstruction.pycolmap_reconstruction(\n", " database_path=output_dir / \"database.db\",\n", " sfm_dir=output_dir,\n", " image_dir=imgs_dir,\n", - " options=reconst_opts,\n", - " verbose=config.general[\"verbose\"],\n", " refine_intrinsics=refine_intrinsics,\n", + " options=opt,\n", + " verbose=verbose,\n", ")" ] } ], "metadata": { + "kernelspec": { + "display_name": "deep-image-matching", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" } }, "nbformat": 4, diff --git a/sanbox_triangulation_dense_pycomap.ipynb b/sanbox_triangulation_dense_pycomap.ipynb index aefb710..df5cfc3 100644 --- a/sanbox_triangulation_dense_pycomap.ipynb +++ b/sanbox_triangulation_dense_pycomap.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -30,7 +30,8 @@ ")\n", "\n", "# Define project directory\n", - "root_path = Path(\"/home/francesco/phd/deep-image-matching/datasets/belv_20230725\")\n", + "# root_path = Path(\"datasets/belv_20230725\")\n", + "root_path = Path(\"datasets/belv_winter\")\n", "\n", "# Path to the images\n", "image_dir = root_path / \"images\"\n", @@ -41,7 +42,7 @@ "sfm_rec_path = sfm_path / \"reconstruction\"\n", "\n", "# Path to the dense matching results to be triangulated\n", - "dense_path = root_path / \"results_roma_bruteforce_quality_high.1\"\n", + "dense_path = root_path / \"results_roma_bruteforce_quality_medium\"\n", "features_h5 = dense_path / \"features.h5\"\n", "matches_h5 = dense_path / \"matches.h5\"\n", "pair_file = dense_path / \"pairs.txt\"\n", @@ -58,29 +59,22 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1;33m2024-04-08 15:25:32 | [WARNING ] The database already exists, deleting it.\u001b[0m\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - "Importing keypoints: 100%|██████████| 2/2 [00:00<00:00, 96.12it/s]\n", - "Importing matches: 100%|██████████| 1/1 [00:00<00:00, 352.17it/s]" + "Importing keypoints: 100%|██████████| 5/5 [00:00<00:00, 181.20it/s]\n", + "Importing matches: 100%|██████████| 10/10 [00:00<00:00, 1322.67it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[0;37m2024-04-08 15:25:33 | [INFO ] Performing geometric verification of the matches...\u001b[0m\n" + "\u001b[0;37m2024-04-10 10:10:31 | [INFO ] Performing geometric verification of the matches...\u001b[0m\n" ] }, { @@ -88,14 +82,14 @@ "output_type": "stream", "text": [ "\n", - "Importing verified matches: 100%|██████████| 1/1 [00:00<00:00, 1.17it/s]" + "Importing verified matches: 100%|██████████| 4/4 [00:02<00:00, 1.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[0;37m2024-04-08 15:25:33 | [INFO ] mean/med/min/max valid matches 70.36/70.36/70.36/70.36%.\u001b[0m\n" + "\u001b[0;37m2024-04-10 10:10:34 | [INFO ] mean/med/min/max valid matches 50.29/61.79/3.48/87.35%.\u001b[0m\n" ] }, { @@ -141,45 +135,63 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "I20240408 15:25:37.644026 2077708 misc.cc:198] \n", + "I20240410 10:10:37.078500 720999 misc.cc:198] \n", "==============================================================================\n", "Loading database\n", "==============================================================================\n", - "I20240408 15:25:37.645126 2077708 database_cache.cc:54] Loading cameras...\n", - "I20240408 15:25:37.645164 2077708 database_cache.cc:64] 2 in 0.000s\n", - "I20240408 15:25:37.645174 2077708 database_cache.cc:72] Loading matches...\n", - "I20240408 15:25:37.646092 2077708 database_cache.cc:78] 1 in 0.001s\n", - "I20240408 15:25:37.646107 2077708 database_cache.cc:94] Loading images...\n", - "I20240408 15:25:37.679128 2077708 database_cache.cc:143] 2 in 0.033s (connected 2)\n", - "I20240408 15:25:37.679203 2077708 database_cache.cc:154] Building correspondence graph...\n", - "I20240408 15:25:37.706593 2077708 database_cache.cc:190] in 0.027s (ignored 0)\n", - "I20240408 15:25:37.707015 2077708 timer.cc:91] Elapsed time: 0.001 [minutes]\n", - "I20240408 15:25:37.717986 2077708 misc.cc:198] \n", + "I20240410 10:10:37.083670 720999 database_cache.cc:54] Loading cameras...\n", + "I20240410 10:10:37.083731 720999 database_cache.cc:64] 3 in 0.000s\n", + "I20240410 10:10:37.083747 720999 database_cache.cc:72] Loading matches...\n", + "I20240410 10:10:37.085050 720999 database_cache.cc:78] 10 in 0.001s\n", + "I20240410 10:10:37.085073 720999 database_cache.cc:94] Loading images...\n", + "I20240410 10:10:37.129077 720999 database_cache.cc:143] 5 in 0.044s (connected 5)\n", + "I20240410 10:10:37.129146 720999 database_cache.cc:154] Building correspondence graph...\n", + "I20240410 10:10:37.163204 720999 database_cache.cc:190] in 0.034s (ignored 0)\n", + "I20240410 10:10:37.164541 720999 timer.cc:91] Elapsed time: 0.001 [minutes]\n", + "I20240410 10:10:37.184716 720999 misc.cc:198] \n", "==============================================================================\n", "Triangulating image #1 (0)\n", "==============================================================================\n", - "I20240408 15:25:37.718021 2077708 sfm.cc:473] => Image sees 0 / 131622 points\n", - "I20240408 15:25:38.029822 2077708 sfm.cc:478] => Triangulated 131620 points\n", - "I20240408 15:25:38.029877 2077708 misc.cc:198] \n", + "I20240410 10:10:37.184751 720999 sfm.cc:473] => Image sees 0 / 38735 points\n", + "I20240410 10:10:37.289250 720999 sfm.cc:478] => Triangulated 38735 points\n", + "I20240410 10:10:37.289290 720999 misc.cc:198] \n", "==============================================================================\n", "Triangulating image #2 (1)\n", "==============================================================================\n", - "I20240408 15:25:38.029884 2077708 sfm.cc:473] => Image sees 131620 / 131622 points\n", - "I20240408 15:25:38.041481 2077708 sfm.cc:478] => Triangulated 0 points\n", - "I20240408 15:25:38.041504 2077708 misc.cc:198] \n", + "I20240410 10:10:37.289295 720999 sfm.cc:473] => Image sees 2223 / 72307 points\n", + "I20240410 10:10:37.470970 720999 sfm.cc:478] => Triangulated 70083 points\n", + "I20240410 10:10:37.471035 720999 misc.cc:198] \n", + "==============================================================================\n", + "Triangulating image #3 (2)\n", + "==============================================================================\n", + "I20240410 10:10:37.471041 720999 sfm.cc:473] => Image sees 23597 / 52781 points\n", + "I20240410 10:10:37.545953 720999 sfm.cc:478] => Triangulated 29184 points\n", + "I20240410 10:10:37.546006 720999 misc.cc:198] \n", + "==============================================================================\n", + "Triangulating image #4 (3)\n", + "==============================================================================\n", + "I20240410 10:10:37.546015 720999 sfm.cc:473] => Image sees 46690 / 75099 points\n", + "I20240410 10:10:37.631104 720999 sfm.cc:478] => Triangulated 28408 points\n", + "I20240410 10:10:37.631163 720999 misc.cc:198] \n", + "==============================================================================\n", + "Triangulating image #5 (4)\n", + "==============================================================================\n", + "I20240410 10:10:37.631170 720999 sfm.cc:473] => Image sees 93900 / 93902 points\n", + "I20240410 10:10:37.638203 720999 sfm.cc:478] => Triangulated 1 points\n", + "I20240410 10:10:37.638222 720999 misc.cc:198] \n", "==============================================================================\n", "Retriangulation\n", "==============================================================================\n", - "I20240408 15:25:38.054045 2077708 incremental_mapper.cc:175] => Completed observations: 0\n", - "I20240408 15:25:38.064352 2077708 incremental_mapper.cc:178] => Merged observations: 0\n", - "I20240408 15:25:38.069828 2077708 misc.cc:198] \n", + "I20240410 10:10:37.652860 720999 incremental_mapper.cc:175] => Completed observations: 0\n", + "I20240410 10:10:37.665251 720999 incremental_mapper.cc:178] => Merged observations: 0\n", + "I20240410 10:10:37.679041 720999 misc.cc:198] \n", "==============================================================================\n", "Bundle adjustment\n", "==============================================================================\n" @@ -190,31 +202,31 @@ "output_type": "stream", "text": [ "iter cost cost_change |gradient| |step| tr_ratio tr_radius ls_iter iter_time total_time\n", - " 0 6.183404e+04 0.00e+00 1.44e+03 0.00e+00 0.00e+00 1.00e+04 0 8.18e-02 3.27e-01\n", - " 1 5.939892e+04 2.44e+03 1.43e-01 2.36e-01 1.00e+00 3.00e+04 0 1.15e-01 4.43e-01\n" + " 0 9.772162e+04 0.00e+00 1.70e+03 0.00e+00 0.00e+00 1.00e+04 0 1.16e-01 4.70e-01\n", + " 1 8.318726e+04 1.45e+04 3.53e-01 4.51e+00 1.00e+00 3.00e+04 0 1.44e-01 6.15e-01\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "I20240408 15:25:38.691607 2077708 misc.cc:205] \n", + "I20240410 10:10:38.528729 720999 misc.cc:205] \n", "Bundle adjustment report\n", "------------------------\n", - "I20240408 15:25:38.691670 2077708 bundle_adjustment.cc:942] \n", - " Residuals : 526480\n", - " Parameters : 394860\n", + "I20240410 10:10:38.528859 720999 bundle_adjustment.cc:942] \n", + " Residuals : 665644\n", + " Parameters : 499233\n", " Iterations : 2\n", - " Time : 0.46784 [s]\n", - " Initial cost : 0.342707 [px]\n", - " Final cost : 0.335891 [px]\n", + " Time : 0.651156 [s]\n", + " Initial cost : 0.383155 [px]\n", + " Final cost : 0.353515 [px]\n", " Termination : Convergence\n", "\n", - "I20240408 15:25:38.706387 2077708 incremental_mapper.cc:175] => Completed observations: 0\n", - "I20240408 15:25:38.717056 2077708 incremental_mapper.cc:178] => Merged observations: 0\n", - "I20240408 15:25:38.741540 2077708 incremental_mapper.cc:160] => Filtered observations: 0\n", - "I20240408 15:25:38.741565 2077708 sfm.cc:521] => Changed observations: 0.000000\n", - "I20240408 15:25:38.834072 2077708 misc.cc:198] \n", + "I20240410 10:10:38.546653 720999 incremental_mapper.cc:175] => Completed observations: 0\n", + "I20240410 10:10:38.558491 720999 incremental_mapper.cc:178] => Merged observations: 0\n", + "I20240410 10:10:38.590533 720999 incremental_mapper.cc:160] => Filtered observations: 5\n", + "I20240410 10:10:38.590562 720999 sfm.cc:521] => Changed observations: 0.000015\n", + "I20240410 10:10:38.738544 720999 misc.cc:198] \n", "==============================================================================\n", "Extracting colors\n", "==============================================================================\n" @@ -226,7 +238,12 @@ "\n", "# Define the options for the triangulation according to the IncrementalPipelineOptions available in pycolmap\n", "# print(pycolmap.IncrementalPipelineOptions().summary())\n", - "opt = dict(triangulation=dict(ignore_two_view_tracks=False))\n", + "opt = dict(\n", + " triangulation=dict(\n", + " ignore_two_view_tracks=False,\n", + " min_angle=0.5,\n", + " ),\n", + ")\n", "verbose = True\n", "\n", "with OutputCapture(verbose):\n", @@ -242,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ diff --git a/src/deep_image_matching/matchers/matcher_base.py b/src/deep_image_matching/matchers/matcher_base.py index 25e645d..28509f3 100644 --- a/src/deep_image_matching/matchers/matcher_base.py +++ b/src/deep_image_matching/matchers/matcher_base.py @@ -930,8 +930,9 @@ def tile_selection( tile_overlap: int, tile_preselection_size: int = 1024, min_matches_per_tile: int = 5, - do_geometric_verification: bool = True, - device: str = "cpu", + do_geometric_verification: bool = False, + device: str = "cuda", + debug_dir: Path = None, ): """ Selects tile pairs for matching based on the specified method. @@ -945,6 +946,8 @@ def tile_selection( List[Tuple[int, int]]: The selected tile pairs. """ + timer = Timer(log_level="debug") + # Compute tiles limits and origin tiler = Tiler(tiling_mode="size") i0 = cv2.imread(str(img0), cv2.IMREAD_GRAYSCALE).astype(np.float32) @@ -981,8 +984,8 @@ def tile_selection( # match downsampled images with roma from ..thirdparty.RoMa.roma import roma_outdoor - n_matches = 5000 - matcher = roma_outdoor(device, coarse_res=448) + n_matches = 2000 + matcher = roma_outdoor(device, coarse_res=280, upsample_res=420) H_A, W_A = i0_new_size H_B, W_B = i1_new_size warp, certainty = matcher.match(str(img0), str(img1), device=device) @@ -1026,8 +1029,8 @@ def tile_selection( _, inlMask = geometric_verification( kpts0=kp0, kpts1=kp1, - threshold=5, - confidence=0.9999, + threshold=10, + confidence=0.99999, quiet=True, ) kp0 = kp0[inlMask] @@ -1044,34 +1047,38 @@ def tile_selection( tile_pairs.add((tidx0, tidx1)) tile_pairs = sorted(tile_pairs) + timer.update("preselection") + # For Debugging... - # if False: - from matplotlib import pyplot as plt - - out_dir = Path("sandbox/preselection") - out_dir.mkdir(parents=True, exist_ok=True) - image0 = cv2.imread(str(img0), cv2.IMREAD_GRAYSCALE) - image1 = cv2.imread(str(img1), cv2.IMREAD_GRAYSCALE) - image0 = resize_image(image0, (i0_new_size[1], i0_new_size[0])) - image1 = resize_image(image1, (i1_new_size[1], i1_new_size[0])) - c = "r" - s = 5 - fig, axes = plt.subplots(1, 2) - for ax, img, kp in zip(axes, [image0, image1], [kp0, kp1]): - ax.imshow(cv2.cvtColor(img, cv2.COLOR_BAYER_BG2BGR)) - ax.scatter(kp[:, 0], kp[:, 1], s=s, c=c) - ax.axis("off") - ax.set_aspect("equal") - for lim0, lim1 in zip(t_orig0.values(), t_orig0.values()): - axes[0].axvline(lim0[0]) - axes[0].axhline(lim0[1]) - axes[1].axvline(lim1[0]) - axes[1].axhline(lim1[1]) - axes[1].get_yaxis().set_visible(False) - fig.tight_layout() - # plt.show() - fig.savefig(out_dir / f"{img0.name}-{img1.name}.jpg") - plt.close() + if debug_dir: + from matplotlib import pyplot as plt + + out_dir = Path(debug_dir) / "preselection" + out_dir.mkdir(parents=True, exist_ok=True) + image0 = cv2.imread(str(img0), cv2.IMREAD_GRAYSCALE) + image1 = cv2.imread(str(img1), cv2.IMREAD_GRAYSCALE) + image0 = resize_image(image0, (i0_new_size[1], i0_new_size[0])) + image1 = resize_image(image1, (i1_new_size[1], i1_new_size[0])) + c = "r" + s = 2 + fig, axes = plt.subplots(1, 2) + for ax, img, kp in zip(axes, [image0, image1], [kp0, kp1]): + ax.imshow(cv2.cvtColor(img, cv2.COLOR_BAYER_BG2BGR)) + ax.scatter(kp[:, 0], kp[:, 1], s=s, c=c) + ax.axis("off") + ax.set_aspect("equal") + for lim0, lim1 in zip(t_orig0.values(), t_orig0.values()): + axes[0].axvline(lim0[0]) + axes[0].axhline(lim0[1]) + axes[1].axvline(lim1[0]) + axes[1].axhline(lim1[1]) + axes[1].get_yaxis().set_visible(False) + fig.tight_layout() + fig.savefig(out_dir / f"{img0.name}-{img1.name}.jpg") + plt.close() + + timer.update("Tile selection") + timer.print("Tile selection") return tile_pairs diff --git a/src/deep_image_matching/matchers/roma.py b/src/deep_image_matching/matchers/roma.py index 8a6886b..0937706 100644 --- a/src/deep_image_matching/matchers/roma.py +++ b/src/deep_image_matching/matchers/roma.py @@ -54,26 +54,26 @@ def __init__(self, config={}) -> None: # Set up RoMa matcher if self.config["general"]["tile_selection"] == TileSelection.NONE: logger.info( - f"RoMa always use a coarse resolution of {self.config['matcher']['coarse_res']} pixels, regardless of the quality parameter resolution." + f"RoMa always use a coarse resolution of {self.config['matcher']['upsample_res']} pixels, regardless of the quality parameter resolution." ) else: logger.info("Running RoMa by tile..") logger.info( - f"RoMa uses a fixed tile size of {self.config['matcher']['coarse_res']} pixels. This can result in a large number of tiles for high-resolution images. If the number of tiles is too high, consider reducing the image resolution via the 'Quality' parameter." + f"RoMa uses a fixed tile size of {self.config['matcher']['upsample_res']} pixels. This can result in a large number of tiles for high-resolution images. If the number of tiles is too high, consider reducing the image resolution via the 'Quality' parameter." ) - if isinstance(self.config["matcher"]["coarse_res"], tuple): + if isinstance(self.config["matcher"]["upsample_res"], tuple): tile_size = ( - self.config["matcher"]["coarse_res"][1], - self.config["matcher"]["coarse_res"][0], + self.config["matcher"]["upsample_res"][1], + self.config["matcher"]["upsample_res"][0], ) - elif isinstance(self.config["matcher"]["coarse_res"], int): + elif isinstance(self.config["matcher"]["upsample_res"], int): tile_size = ( - self.config["matcher"]["coarse_res"], - self.config["matcher"]["coarse_res"], + self.config["matcher"]["upsample_res"], + self.config["matcher"]["upsample_res"], ) else: - raise ValueError("Invalid type for 'coarse_res'. It should be an integer or a tuple of two integers.") + raise ValueError("Invalid type for 'upsample_res'. It should be an integer or a tuple of two integers.") # Force the tile size to be the same as the RoMa coarse_res self.config["general"]["tile_size"] = tile_size @@ -265,10 +265,11 @@ def write_tiles_disk(output_dir: Path, tiles: dict) -> None: tile_preselection_size=self.tile_preselection_size, min_matches_per_tile=self.min_matches_per_tile, device=self._device, + debug_dir=self.config["general"]["output_dir"] / "debug", ) if len(tile_pairs) > self.max_tile_pairs: raise RuntimeError( - f"Too many tile pairs ({len(tile_pairs)}) to match, the matching process will be too slow and it may be inaccurate. Try to reduce the image resolution using a lower 'Quality' value (or change the 'max_tile_pairs' class attribute, if you know what you are doing)." + f"Too many tile pairs ({len(tile_pairs)}) to match, the matching process will be too slow and it may be inaccurate. Try to reduce the image resolution using a lower 'Quality' parameter." ) else: logger.info(f"Matching {len(tile_pairs)} tile pairs") diff --git a/src/deep_image_matching/utils/database.py b/src/deep_image_matching/utils/database.py index 74d4b9e..ccebea3 100644 --- a/src/deep_image_matching/utils/database.py +++ b/src/deep_image_matching/utils/database.py @@ -225,6 +225,7 @@ def update_camera( camera_id, ), ) + self.commit() return cursor.lastrowid def add_image(