Merge pull request #38 from ThomasWaldmann/ssh-config-support

sftp: add support for ~/.ssh/config, fixes #37

pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 license = {text="BSD"}
 requires-python = ">=3.9"
 dependencies = [
-    "paramiko",
+    "paramiko >= 1.9.1",  # 1.9.1+ supports multiple IdentityKey entries in .ssh/config
 ]
 
 [project.urls]

src/borgstore/backends/sftp.py

@@ -26,21 +26,60 @@ def get_sftp_backend(url):
     """
     m = re.match(sftp_regex, url, re.VERBOSE)
     if m:
-        return Sftp(username=m["username"], hostname=m["hostname"], port=int(m["port"] or "22"), path=m["path"])
+        return Sftp(username=m["username"], hostname=m["hostname"], port=int(m["port"] or "0"), path=m["path"])
 
 
 class Sftp(BackendBase):
-    def __init__(self, hostname: str, path: str, port: int = 22, username: Optional[str] = None):
+    def __init__(self, hostname: str, path: str, port: int = 0, username: Optional[str] = None):
         self.username = username
         self.hostname = hostname
         self.port = port
         self.base_path = path
         self.opened = False
 
+    def _get_host_config_from_file(self, path: str, hostname: str):
+        """lookup the configuration for hostname in path (ssh config file)"""
+        config_path = Path(path).expanduser()
+        try:
+            ssh_config = paramiko.SSHConfig.from_path(config_path)
+        except FileNotFoundError:
+            return paramiko.SSHConfigDict()  # empty dict
+        else:
+            return ssh_config.lookup(hostname)
+
+    def _get_host_config(self):
+        """assemble all given and configured host config values"""
+        host_config = paramiko.SSHConfigDict()
+        # self.hostname might be an alias/shortcut (with real hostname given in configuration),
+        # but there might be also nothing in the configs at all for self.hostname:
+        host_config["hostname"] = self.hostname
+        # first process system-wide ssh config, then override with user ssh config:
+        host_config.update(self._get_host_config_from_file("/etc/ssh/ssh_config", self.hostname))
+        # note: no support yet for /etc/ssh/ssh_config.d/*
+        host_config.update(self._get_host_config_from_file("~/.ssh/config", self.hostname))
+        # now override configured values with given values
+        if self.username is not None:
+            host_config.update({"user": self.username})
+        if self.port != 0:
+            host_config.update({"port": self.port})
+        # make sure port is present and is an int
+        host_config["port"] = int(host_config.get("port") or 22)
+        return host_config
+
     def _connect(self):
         ssh = paramiko.SSHClient()
-        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        ssh.connect(hostname=self.hostname, username=self.username, port=self.port, allow_agent=True)
+        # note: we do not deal with unknown hosts and ssh.set_missing_host_key_policy here,
+        # the user shall just make "first contact" to any new host using ssh or sftp cli command
+        # and interactively verify remote host fingerprints.
+        ssh.load_system_host_keys()  # this is documented to load the USER's known_hosts file
+        host_config = self._get_host_config()
+        ssh.connect(
+            hostname=host_config["hostname"],
+            username=host_config.get("user"),  # if None, paramiko will use current user
+            port=host_config["port"],
+            key_filename=host_config.get("identityfile"),  # list of keys, ~ is already expanded
+            allow_agent=True,
+        )
         self.client = ssh.open_sftp()
 
     def _disconnect(self):

tests/test_backends.py

@@ -2,6 +2,7 @@
 Generic testing for the misc. backend implementations.
 """
 
+import os
 from pathlib import Path
 
 import pytest
@@ -32,12 +33,11 @@ def posixfs_backend_created(tmp_path):
 
 
 def _get_sftp_backend():
-    # needs an authorized key loaded into the ssh agent. pytest works, tox doesn't:
-    # return Sftp(username="tw", hostname="localhost", path="/Users/tw/w/borgstore/temp-store")
-    # for tests with higher latency:
-    return Sftp(
-        username="twaldmann", hostname="shell.ipv4.thinkmo.de", port=2222, path="/home/twaldmann/borgstore/temp-store"
-    )
+    # export BORGSTORE_TEST_SFTP_URL="sftp://user@host:port/home/user/borgstore/temp-store"
+    # needs an authorized key loaded into the ssh agent. pytest works, tox doesn't.
+    url = os.environ.get("BORGSTORE_TEST_SFTP_URL")
+    if url:
+        return get_sftp_backend(url)
 
 
 def check_sftp_available():
@@ -46,7 +46,7 @@ def check_sftp_available():
         be = _get_sftp_backend()
         be.create()  # first sftp activity happens here
     except Exception:
-        return False
+        return False  # use "raise" here for debugging sftp store issues
     else:
         be.destroy()
         return True
@@ -96,16 +96,16 @@ def test_file_url(url, path):
     "url,username,hostname,port,path",
     [
         ("sftp://username@hostname:2222/some/path", "username", "hostname", 2222, "/some/path"),
-        ("sftp://username@hostname/some/path", "username", "hostname", 22, "/some/path"),
-        ("sftp://hostname/some/path", None, "hostname", 22, "/some/path"),
+        ("sftp://username@hostname/some/path", "username", "hostname", 0, "/some/path"),
+        ("sftp://hostname/some/path", None, "hostname", 0, "/some/path"),
     ],
 )
 def test_sftp_url(url, username, hostname, port, path):
     backend = get_sftp_backend(url)
     assert isinstance(backend, Sftp)
     assert backend.username == username
     assert backend.hostname == hostname
-    assert backend.port == port
+    assert backend.port == port  # note: 0 means "not given" (and will usually mean 22 in the end)
     assert backend.base_path == path