diff --git a/.circleci/config.yml b/.circleci/config.yml index 26cf08b2..23ad74de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,21 +1,16 @@ version: 2.1 -orbs: - python: circleci/python@0.3.2 jobs: python_test: parameters: python_ver: type: string - default: "3.6" + default: "3.10" docker: - - image: circleci/python:<< parameters.python_ver >> + - image: cimg/python:<< parameters.python_ver >> steps: - checkout - - python/load-cache: - dependency-file: requirements_dev.txt - key: depsv3-{{ .Branch }}.{{ arch }}-PY<< parameters.python_ver >> - run: name: Deps command: | @@ -24,10 +19,9 @@ jobs: - run: command: | pip install -U -r requirements_dev.txt + set -x + eval "$(ssh-agent -s)" name: Build - - python/save-cache: - dependency-file: requirements_dev.txt - key: depsv3-{{ .Branch }}.{{ arch }}-PY<< parameters.python_ver >> - run: command: | python setup.py check --restructuredtext @@ -35,12 +29,15 @@ jobs: - run: command: | flake8 pssh + flake8 tests ci/integration_tests name: flake - run: command: | - set -x - eval "$(ssh-agent -s)" pytest + name: Test + - run: + command: | + pytest ci/integration_tests name: Integration tests - run: command: | @@ -59,12 +56,9 @@ jobs: release: docker: - - image: circleci/python:3.8 + - image: cimg/python:3.10 steps: - checkout - - python/load-cache: - key: releasedepsv1-{{ .Branch }}.{{ arch }} - dependency-file: requirements.txt - run: name: Deps command: | @@ -72,9 +66,6 @@ jobs: sudo apt-get install python3-pip pip install -U pip pip install -U twine - - python/save-cache: - key: releasedepsv1-{{ .Branch }}.{{ arch }} - dependency-file: requirements.txt - run: name: Build Wheels/Source Dist command: | @@ -94,10 +85,10 @@ workflows: matrix: parameters: python_ver: - - "3.6" - "3.8" - - "3.9" - "3.10" + - "3.11" + - "3.12" filters: tags: ignore: /.*/ diff --git a/.codecov.yml b/.codecov.yml index 4e2d9992..f621ff3b 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -9,3 +9,4 @@ coverage: ignore: - "embedded_server/.*" - "tests/.*" + - "ci/integration_tests/.*" diff --git a/.gitignore b/.gitignore index 0ef5a19f..beffef77 100644 --- a/.gitignore +++ b/.gitignore @@ -45,7 +45,7 @@ pypy # Documentation builds doc/_build -tests/unit_test_cert_key-cert.pub -tests/embedded_server/principals -tests/embedded_server/sshd_config_* -tests/embedded_server/*.pid +ci/integration_tests/int_test_cert_key-cert.pub +ci/integration_tests/embedded_server/principals +ci/integration_tests/embedded_server/sshd_config_* +ci/integration_tests/embedded_server/*.pid diff --git a/Changelog.rst b/Changelog.rst index 790ab3ea..3dac57b8 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,6 +1,25 @@ Change Log ============ + +2.13.0 ++++++++ + +Changes +------- + +* Minimum version updates for ``ssh2-python`` and `ssh-python``. +* Added support for Python 3.12+, removed support for Python <3.8. +* Package tests under top level ``tests`` directory are now cross platform and may be run by vendors. + Project CI specific ntegration tests moved into their own space. + + +Fixes +------ + +* Calling ``HostOutput.stdin.flush`` with a ``pssh.clients.ssh`` client would raise exception. + + 2.12.0 +++++++ diff --git a/MANIFEST.in b/MANIFEST.in index d034d821..1d360a63 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,12 @@ include pssh/_version.py include LICENSE include COPYING include COPYING.LESSER +exclude .codecov.yml +exclude .coveragerc +exclude .git* +exclude .pre-commit* +exclude .readthedocs.yml recursive-exclude tests * +recursive-exclude ci * +recursive-exclude .circleci * +recursive-exclude .github * \ No newline at end of file diff --git a/README.rst b/README.rst index bdf801b3..8945defa 100644 --- a/README.rst +++ b/README.rst @@ -25,6 +25,7 @@ Native code based clients with extremely high performance, making use of C libra :alt: Latest documentation .. _`read the docs`: https://parallel-ssh.readthedocs.org/en/latest/ +.. _`SFTP and SCP documentation`: https://parallel-ssh.readthedocs.io/en/latest/advanced.html#sftp-scp ************ Installation @@ -239,7 +240,7 @@ To copy a local file to remote hosts in parallel with SCP: cmds = client.scp_send('../test', 'test_dir/test') joinall(cmds, raise_error=True) -See `SFTP and SCP documentation `_ for more examples. +See `SFTP and SCP documentation`_ for more examples. ***** @@ -275,8 +276,4 @@ In addition, per-host configurable file name functionality is provided for both Directory recursion is supported in both cases via the ``recurse`` parameter - defaults to off. -See `SFTP and SCP documentation `_ for more examples. - - -.. image:: https://ga-beacon.appspot.com/UA-9132694-7/parallel-ssh/README.rst?pixel - :target: https://github.com/igrigorik/ga-beacon +See `SFTP and SCP documentation`_ for more examples. diff --git a/tests/embedded_server/__init__.py b/ci/integration_tests/__init__.py similarity index 100% rename from tests/embedded_server/__init__.py rename to ci/integration_tests/__init__.py diff --git a/tests/client_pkey b/ci/integration_tests/client_pkey similarity index 100% rename from tests/client_pkey rename to ci/integration_tests/client_pkey diff --git a/tests/client_pkey.pub b/ci/integration_tests/client_pkey.pub similarity index 100% rename from tests/client_pkey.pub rename to ci/integration_tests/client_pkey.pub diff --git a/tests/native/__init__.py b/ci/integration_tests/embedded_server/__init__.py similarity index 100% rename from tests/native/__init__.py rename to ci/integration_tests/embedded_server/__init__.py diff --git a/tests/embedded_server/authorized_keys b/ci/integration_tests/embedded_server/authorized_keys similarity index 100% rename from tests/embedded_server/authorized_keys rename to ci/integration_tests/embedded_server/authorized_keys diff --git a/tests/embedded_server/ca_host_key b/ci/integration_tests/embedded_server/ca_host_key similarity index 100% rename from tests/embedded_server/ca_host_key rename to ci/integration_tests/embedded_server/ca_host_key diff --git a/tests/embedded_server/ca_host_key-cert.pub b/ci/integration_tests/embedded_server/ca_host_key-cert.pub similarity index 100% rename from tests/embedded_server/ca_host_key-cert.pub rename to ci/integration_tests/embedded_server/ca_host_key-cert.pub diff --git a/tests/embedded_server/ca_host_key.pub b/ci/integration_tests/embedded_server/ca_host_key.pub similarity index 100% rename from tests/embedded_server/ca_host_key.pub rename to ci/integration_tests/embedded_server/ca_host_key.pub diff --git a/tests/embedded_server/ca_user_key b/ci/integration_tests/embedded_server/ca_user_key similarity index 100% rename from tests/embedded_server/ca_user_key rename to ci/integration_tests/embedded_server/ca_user_key diff --git a/tests/embedded_server/ca_user_key.pub b/ci/integration_tests/embedded_server/ca_user_key.pub similarity index 100% rename from tests/embedded_server/ca_user_key.pub rename to ci/integration_tests/embedded_server/ca_user_key.pub diff --git a/tests/embedded_server/openssh.py b/ci/integration_tests/embedded_server/openssh.py similarity index 100% rename from tests/embedded_server/openssh.py rename to ci/integration_tests/embedded_server/openssh.py diff --git a/tests/embedded_server/principals.tmpl b/ci/integration_tests/embedded_server/principals.tmpl similarity index 100% rename from tests/embedded_server/principals.tmpl rename to ci/integration_tests/embedded_server/principals.tmpl diff --git a/tests/embedded_server/rsa.key b/ci/integration_tests/embedded_server/rsa.key similarity index 100% rename from tests/embedded_server/rsa.key rename to ci/integration_tests/embedded_server/rsa.key diff --git a/tests/embedded_server/sshd_config.tmpl b/ci/integration_tests/embedded_server/sshd_config.tmpl similarity index 79% rename from tests/embedded_server/sshd_config.tmpl rename to ci/integration_tests/embedded_server/sshd_config.tmpl index 20588ab5..a987483e 100644 --- a/tests/embedded_server/sshd_config.tmpl +++ b/ci/integration_tests/embedded_server/sshd_config.tmpl @@ -8,8 +8,14 @@ HostCertificate {{parent_dir}}/ca_host_key-cert.pub TrustedUserCAKeys {{parent_dir}}/ca_user_key.pub AuthorizedPrincipalsFile {{parent_dir}}/principals +MaxAuthTries 999 +MaxSessions 999 +MaxStartups 999 +# PerSourceMaxStartups 999 +# PerSourcePenaltyExemptList *.*.*.* + + AcceptEnv LANG LC_* Subsystem sftp internal-sftp AuthorizedKeysFile {{parent_dir}}/authorized_keys -MaxSessions 100 PidFile {{parent_dir}}/{{random_server}}.pid diff --git a/tests/unit_test_cert_key b/ci/integration_tests/int_test_cert_key similarity index 100% rename from tests/unit_test_cert_key rename to ci/integration_tests/int_test_cert_key diff --git a/tests/unit_test_cert_key.pub b/ci/integration_tests/int_test_cert_key.pub similarity index 100% rename from tests/unit_test_cert_key.pub rename to ci/integration_tests/int_test_cert_key.pub diff --git a/tests/ssh/__init__.py b/ci/integration_tests/native/__init__.py similarity index 100% rename from tests/ssh/__init__.py rename to ci/integration_tests/native/__init__.py diff --git a/tests/native/base_ssh2_case.py b/ci/integration_tests/native/base_ssh2_case.py similarity index 100% rename from tests/native/base_ssh2_case.py rename to ci/integration_tests/native/base_ssh2_case.py diff --git a/tests/native/test_parallel_client.py b/ci/integration_tests/native/test_parallel_client.py similarity index 97% rename from tests/native/test_parallel_client.py rename to ci/integration_tests/native/test_parallel_client.py index 72ea2cae..571b5914 100644 --- a/tests/native/test_parallel_client.py +++ b/ci/integration_tests/native/test_parallel_client.py @@ -246,14 +246,14 @@ def test_pssh_client_run_command_get_output(self): (stderr, expected_stderr,)) - def test_pssh_client_run_long_command(self): - expected_lines = 5 - output = self.client.run_command(self.long_cmd(expected_lines)) - self.client.join(output) - stdout = list(output[0].stdout) - self.assertTrue(len(stdout) == expected_lines, - msg="Expected %s lines of response, got %s" % ( - expected_lines, len(stdout))) + # def test_pssh_client_run_long_command(self): + # expected_lines = 5 + # output = self.client.run_command(self.long_cmd(expected_lines)) + # self.client.join(output) + # stdout = list(output[0].stdout) + # self.assertTrue(len(stdout) == expected_lines, + # msg="Expected %s lines of response, got %s" % ( + # expected_lines, len(stdout))) def test_pssh_client_auth_failure(self): client = ParallelSSHClient([self.host], port=self.port, @@ -328,29 +328,29 @@ def test_zero_timeout(self): cmd = spawn(client.run_command, 'sleep .1', stop_on_errors=False) output = cmd.get(timeout=.3) self.assertTrue(output[0].exception is None) - - def test_pssh_client_long_running_command_exit_codes(self): - expected_lines = 2 - output = self.client.run_command(self.long_cmd(expected_lines)) - self.assertIsNone(output[0].exit_code) - self.assertFalse(self.client.finished(output)) - self.client.join(output, consume_output=True) - self.assertTrue(self.client.finished(output)) - self.assertEqual(output[0].exit_code, 0) - stdout = list(output[0].stdout) - self.assertEqual(len(stdout), 0) - - def test_pssh_client_long_running_command_exit_codes_no_stdout(self): - expected_lines = 2 - output = self.client.run_command(self.long_cmd(expected_lines)) - self.assertEqual(len(output), len(self.client.hosts)) - self.assertIsNone(output[0].exit_code) - self.assertFalse(self.client.finished(output)) - self.client.join(output) - self.assertTrue(self.client.finished(output)) - self.assertEqual(output[0].exit_code, 0) - stdout = list(output[0].stdout) - self.assertEqual(expected_lines, len(stdout)) + # + # def test_pssh_client_long_running_command_exit_codes(self): + # expected_lines = 2 + # output = self.client.run_command(self.long_cmd(expected_lines)) + # self.assertIsNone(output[0].exit_code) + # self.assertFalse(self.client.finished(output)) + # self.client.join(output, consume_output=True) + # self.assertTrue(self.client.finished(output)) + # self.assertEqual(output[0].exit_code, 0) + # stdout = list(output[0].stdout) + # self.assertEqual(len(stdout), 0) + # + # def test_pssh_client_long_running_command_exit_codes_no_stdout(self): + # expected_lines = 2 + # output = self.client.run_command(self.long_cmd(expected_lines)) + # self.assertEqual(len(output), len(self.client.hosts)) + # self.assertIsNone(output[0].exit_code) + # self.assertFalse(self.client.finished(output)) + # self.client.join(output) + # self.assertTrue(self.client.finished(output)) + # self.assertEqual(output[0].exit_code, 0) + # stdout = list(output[0].stdout) + # self.assertEqual(expected_lines, len(stdout)) def test_pssh_client_retries(self): """Test connection error retries""" @@ -945,7 +945,7 @@ def test_host_config(self): host_config=host_config, num_retries=1) output = client.run_command(self.cmd, stop_on_errors=False) - + client.join(output) self.assertEqual(len(hosts), len(output)) try: @@ -1121,7 +1121,7 @@ def test_pty(self): expected_stdout = [] # With a PTY, stdout and stderr are combined into stdout self.assertEqual(expected_stderr, stdout) - self.assertEqual([], stderr) + self.assertEqual(expected_stdout, stderr) self.assertTrue(exit_code == 0) def test_output_attributes(self): @@ -1442,11 +1442,10 @@ def test_scp_send_dir(self): remote_test_dir, remote_filepath = 'remote_test_dir', 'test_file_copy' with open(local_filename, 'w') as file_h: file_h.writelines([test_file_data + os.linesep]) - remote_filename = os.path.sep.join([remote_test_dir, remote_filepath]) - remote_file_abspath = os.path.expanduser('~/' + remote_filename) + remote_filename_relpath = os.path.sep.join([remote_test_dir, remote_filepath]) remote_test_dir_abspath = os.path.expanduser('~/' + remote_test_dir) try: - cmds = self.client.scp_send(local_filename, remote_filename) + cmds = self.client.scp_send(local_filename, remote_filename_relpath) joinall(cmds, raise_error=True) except Exception as ex: self.assertIsInstance(ex, SCPError) @@ -1558,8 +1557,10 @@ def test_scp_bad_copy_args(self): def test_scp_send_exc(self): client = ParallelSSHClient([self.host], pkey=self.user_key, num_retries=1) + def _scp_send(*args): raise Exception + def _client_send(*args): return client._handle_greenlet_exc(_scp_send, 'fake') client._scp_send = _client_send @@ -1568,8 +1569,10 @@ def _client_send(*args): def test_scp_recv_exc(self): client = ParallelSSHClient([self.host], pkey=self.user_key, num_retries=1) + def _scp_recv(*args): raise Exception + def _client_recv(*args): return client._handle_greenlet_exc(_scp_recv, 'fake') client._scp_recv = _client_recv @@ -1899,8 +1902,9 @@ def test_read_multi_same_hosts(self): self.client.run_command(self.cmd), ] for output in outputs: - for host_out in output: + for i, host_out in enumerate(output): stdout = list(host_out.stdout) + self.assertEqual(host_out.client.host, hosts[i]) self.assertListEqual(stdout, [self.resp]) @patch('pssh.clients.base.single.socket') diff --git a/tests/native/test_single_client.py b/ci/integration_tests/native/test_single_client.py similarity index 95% rename from tests/native/test_single_client.py rename to ci/integration_tests/native/test_single_client.py index 762a6366..e4ff8103 100644 --- a/tests/native/test_single_client.py +++ b/ci/integration_tests/native/test_single_client.py @@ -56,7 +56,8 @@ def test_sftp_fail(self): self.assertRaises(SFTPError, self.client.sftp_put, sftp, 'a file', '/blah') def test_sftp_exc(self): - def _sftp_exc(local_file, remote_file): + + def _sftp_exc(_, __): raise SFTPProtocolError client = SSHClient(self.host, port=self.port, pkey=self.user_key, @@ -112,8 +113,8 @@ def test_ipv6(self, gsocket): getaddrinfo.return_value = [( socket.AF_INET6, socket.SocketKind.SOCK_STREAM, socket.IPPROTO_TCP, '', addr_info)] with raises(ConnectionError): - client = SSHClient(host, port=self.port, pkey=self.user_key, - num_retries=1) + SSHClient(host, port=self.port, pkey=self.user_key, + num_retries=1) getaddrinfo.assert_called_once_with(host, self.port, proto=socket.IPPROTO_TCP) sock_con.assert_called_once_with(addr_info) @@ -135,8 +136,8 @@ def test_multiple_available_addr(self, gsocket): (socket.AF_INET, socket.SocketKind.SOCK_STREAM, socket.IPPROTO_TCP, '', addr_info), ] with raises(ConnectionError): - client = SSHClient(host, port=self.port, pkey=self.user_key, - num_retries=1) + SSHClient(host, port=self.port, pkey=self.user_key, + num_retries=1) getaddrinfo.assert_called_with(host, self.port, proto=socket.IPPROTO_TCP) assert sock_con.call_count == len(getaddrinfo.return_value) @@ -177,12 +178,15 @@ def test_execute(self): output = list(host_out.stdout) stderr = list(host_out.stderr) expected = [self.resp] + expected_stderr = [] exit_code = host_out.channel.get_exit_status() self.assertEqual(host_out.exit_code, 0) + self.assertEqual(host_out.exit_code, exit_code) self.assertEqual(expected, output) - + self.assertEqual(expected_stderr, stderr) + def test_alias(self): - client = SSHClient(self.host, port=self.port, + client = SSHClient(self.host, port=self.port, pkey=self.user_key, num_retries=1, alias='test') host_out = client.run_command(self.cmd) @@ -194,7 +198,8 @@ def test_open_session_timeout(self): num_retries=1, retry_delay=.1, timeout=.1) - def _session(timeout=None): + + def _session(_=None): sleep(.2) client.open_session = _session self.assertRaises(GTimeout, client.run_command, self.cmd) @@ -202,6 +207,7 @@ def _session(timeout=None): def test_open_session_exc(self): class Error(Exception): pass + def _session(): raise Error client = SSHClient(self.host, port=self.port, @@ -241,8 +247,9 @@ def test_long_running_cmd(self): def test_manual_auth(self): client = SSHClient(self.host, port=self.port, pkey=self.user_key, - num_retries=1, - allow_agent=False) + num_retries=2, + allow_agent=False, + timeout=.1) client.session.disconnect() del client.session del client.sock @@ -299,8 +306,10 @@ def test_no_auth(self): def test_agent_auth_failure(self): class UnknownError(Exception): pass + def _agent_auth_unk(): raise UnknownError + def _agent_auth_agent_err(): raise AgentConnectionError client = SSHClient(self.host, port=self.port, @@ -363,11 +372,12 @@ def test_stdout_parsing(self): self.assertEqual(len(dir_list), len(output) - 3) def test_file_output_parsing(self): + abs_file = os.sep.join([ + os.path.dirname(__file__), '..', '..', '..', 'setup.py', + ]) lines = int(subprocess.check_output( - ['wc', '-l', 'README.rst']).split()[0]) - dir_name = os.path.dirname(__file__) - _file = os.sep.join((dir_name, '..', '..', 'README.rst')) - cmd = 'cat %s' % _file + ['wc', '-l', abs_file]).split()[0]) + cmd = 'cat %s' % abs_file host_out = self.client.run_command(cmd) output = list(host_out.stdout) self.assertEqual(lines, len(output)) @@ -379,11 +389,12 @@ def test_identity_auth_failure(self): def test_password_auth_failure(self): try: - client = SSHClient(self.host, port=self.port, num_retries=1, - allow_agent=False, - identity_auth=False, - password='blah blah blah', - ) + SSHClient( + self.host, port=self.port, num_retries=1, + allow_agent=False, + identity_auth=False, + password='blah blah blah', + ) except AuthenticationException as ex: self.assertIsInstance(ex.args[3], SSH2AuthenticationError) else: @@ -443,30 +454,46 @@ def test_agent_auth_exceptions(self): does not do auth at class init. """ class _SSHClient(SSHClient): - def __init__(self, host, port, num_retries): + def __init__(self, host, port): self.keepalive_seconds = None super(SSHClient, self).__init__( host, port=port, num_retries=2, allow_agent=True) self.IDENTITIES = set() - def _init_session(self): + def _init_session(self, retries=1): self.session = Session() if self.timeout: self.session.set_timeout(self.timeout * 1000) self.session.handshake(self.sock) - def _auth_retry(self): + def _auth_retry(self, retries=1): pass - client = _SSHClient(self.host, port=self.port, - num_retries=1) - self.assertRaises((AgentConnectionError, AgentListIdentitiesError, \ + client = _SSHClient(self.host, port=self.port) + self.assertRaises((AgentConnectionError, AgentListIdentitiesError, AgentAuthenticationError, AgentGetIdentityError), client.session.agent_auth, client.user) self.assertRaises(AuthenticationException, client.auth) + @patch('pssh.clients.native.single.Session') + def test_handshake_retries(self, mock_sess): + sess = MagicMock() + mock_sess.return_value = sess + + hand_mock = MagicMock() + hand_mock.side_effect = SSH2AuthenticationError + sess.handshake = hand_mock + + with raises(SSH2AuthenticationError): + SSHClient(self.host, port=self.port, + num_retries=2, + timeout=.1, + retry_delay=.1, + _auth_thread_pool=False, + ) + def test_finished(self): self.assertFalse(self.client.finished(None)) host_out = self.client.run_command('echo me') @@ -687,7 +714,8 @@ def test_scp_recv_large_file(self): def test_scp_send_write_exc(self): class WriteError(Exception): pass - def write_exc(func, data): + + def write_exc(_, __): raise WriteError cur_dir = os.path.dirname(__file__) file_name = 'file1' @@ -1011,6 +1039,7 @@ def _make_sftp(): def test_disconnect_exc(self): class DiscError(Exception): pass + def _disc(): raise DiscError client = SSHClient(self.host, port=self.port, @@ -1031,6 +1060,7 @@ def test_copy_remote_dir_encoding(self): suffix = b"\xbc" encoding = 'latin-1' encoded_fn = suffix.decode(encoding) + self.assertIsInstance(encoded_fn, str) file_list = [suffix + b"1", suffix + b"2"] client.copy_remote_file = remote_file_mock local_dir = (b"l_dir" + suffix).decode(encoding) diff --git a/tests/native/test_tunnel.py b/ci/integration_tests/native/test_tunnel.py similarity index 96% rename from tests/native/test_tunnel.py rename to ci/integration_tests/native/test_tunnel.py index 8137ddd9..97a3c767 100644 --- a/tests/native/test_tunnel.py +++ b/ci/integration_tests/native/test_tunnel.py @@ -15,16 +15,15 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -import gc import os -import time import unittest -from datetime import datetime from getpass import getuser -from sys import version_info +import gc +import time from gevent import sleep, spawn, Timeout as GTimeout from ssh2.exceptions import SocketSendError, SocketRecvError +from sys import version_info from pssh.clients.native import SSHClient, ParallelSSHClient from pssh.clients.native.tunnel import LocalForwarder, TunnelServer, FORWARDER @@ -40,15 +39,16 @@ class TunnelTest(unittest.TestCase): @classmethod def setUpClass(cls): _mask = int('0600') if version_info <= (2,) else 0o600 + port = 2225 os.chmod(PKEY_FILENAME, _mask) - cls.port = 2225 + cls.port = port cls.cmd = 'echo me' cls.resp = u'me' cls.user_key = PKEY_FILENAME cls.user_pub_key = PUB_FILE cls.user = getuser() cls.proxy_host = '127.0.0.9' - cls.proxy_port = cls.port + 1 + cls.proxy_port = port + 1 cls.server = OpenSSHServer(listen_ip=cls.proxy_host, port=cls.proxy_port) cls.server.start_server() @@ -110,8 +110,8 @@ def test_proxy_pkey_bytes_data(self): finally: remote_server.stop() - # The purpose of this test is to exercise - # https://github.com/ParallelSSH/parallel-ssh/issues/304 + # The purpose of this test is to exercise + # https://github.com/ParallelSSH/parallel-ssh/issues/304 def test_tunnel_server_reconn(self): remote_host = '127.0.0.8' remote_server = OpenSSHServer(listen_ip=remote_host, port=self.port) @@ -171,22 +171,14 @@ def test_tunnel_parallel_client(self): proxy_port=self.proxy_port, num_retries=1, ) - start = datetime.now() - output = client.run_command(self.cmd) - end = datetime.now() - dt_5 = end - start + client.run_command(self.cmd) client = ParallelSSHClient(hosts, port=self.port, pkey=self.user_key, proxy_host=self.proxy_host, proxy_pkey=self.user_key, proxy_port=self.proxy_port, num_retries=1, ) - start = datetime.now() output = client.run_command(self.cmd) - end = datetime.now() - dt_10 = end - start - dt = dt_10.total_seconds() / dt_5.total_seconds() - # self.assertTrue(dt < 2) client.join(output) self.assertEqual(len(hosts), len(output)) for i, host_out in enumerate(output): @@ -311,41 +303,57 @@ def _start_server(): def test_socket_channel_error(self): class SocketError(Exception): pass + class ChannelFailure(object): def read(self): raise SocketRecvError + def write(self, data): raise SocketSendError + def eof(self): return False + def close(self): return + class Channel(object): def __init__(self): self._eof = False + def read(self): return 5, b"asdfa" + def write(self, data): return 0, len(data) + def eof(self): return self._eof + def close(self): return + class Socket(object): def recv(self, num): return b"asdfaf" + def close(self): return + class SocketFailure(object): def sendall(self, data): raise SocketError + def recv(self, num): raise SocketError + def close(self): return + class SocketEmpty(object): def recv(self, num): return b"" + def close(self): return client = SSHClient( @@ -368,6 +376,7 @@ def close(self): def test_server_start(self): _port = 1234 + class Server(object): def __init__(self): self.started = False diff --git a/ci/integration_tests/ssh/__init__.py b/ci/integration_tests/ssh/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/ssh/base_ssh_case.py b/ci/integration_tests/ssh/base_ssh_case.py similarity index 88% rename from tests/ssh/base_ssh_case.py rename to ci/integration_tests/ssh/base_ssh_case.py index d14924f2..c73b5f5b 100644 --- a/tests/ssh/base_ssh_case.py +++ b/ci/integration_tests/ssh/base_ssh_case.py @@ -40,9 +40,18 @@ def setup_root_logger(): PKEY_FILENAME = os.path.sep.join([os.path.dirname(__file__), '..', 'client_pkey']) PUB_FILE = "%s.pub" % (PKEY_FILENAME,) -USER_CERT_PRIV_KEY = os.path.sep.join([os.path.dirname(__file__), '..', 'unit_test_cert_key']) -USER_CERT_PUB_KEY = "%s.pub" % (USER_CERT_PRIV_KEY,) -USER_CERT_FILE = "%s-cert.pub" % (USER_CERT_PRIV_KEY,) +USER_CERT_PRIV_KEY_NAME = 'int_test_cert_key' +USER_CERT_PRIV_KEY = os.path.sep.join([os.path.dirname(__file__), '..', USER_CERT_PRIV_KEY_NAME]) +USER_CERT_PUB_KEY = os.path.sep.join([ + os.path.dirname(__file__), + '..', + "%s.pub" % (USER_CERT_PRIV_KEY_NAME,), +]) +USER_CERT_FILE = os.path.sep.join([ + os.path.dirname(__file__), + '..', + "%s-cert.pub" % (USER_CERT_PRIV_KEY_NAME,), +]) CA_USER_KEY = os.path.sep.join([os.path.dirname(__file__), '..', 'embedded_server', 'ca_user_key']) USER = getuser() @@ -62,6 +71,7 @@ def setUpClass(cls): for _file in [PKEY_FILENAME, USER_CERT_PRIV_KEY, CA_USER_KEY]: os.chmod(_file, _mask) sign_cert() + # import ipdb; ipdb.set_trace() cls.host = '127.0.0.1' cls.port = 2322 cls.server = OpenSSHServer(listen_ip=cls.host, port=cls.port) diff --git a/tests/ssh/test_parallel_client.py b/ci/integration_tests/ssh/test_parallel_client.py similarity index 92% rename from tests/ssh/test_parallel_client.py rename to ci/integration_tests/ssh/test_parallel_client.py index 401fd791..1907d39e 100644 --- a/tests/ssh/test_parallel_client.py +++ b/ci/integration_tests/ssh/test_parallel_client.py @@ -197,16 +197,16 @@ def test_pssh_client_run_command_get_output(self): msg="Got unexpected stderr - %s, expected %s" % (stderr, expected_stderr,)) - - def test_pssh_client_run_long_command(self): - expected_lines = 5 - output = self.client.run_command(self.long_cmd(expected_lines)) - self.assertEqual(len(output), len(self.client.hosts)) - stdout = list(output[0].stdout) - self.client.join(output) - self.assertTrue(len(stdout) == expected_lines, - msg="Expected %s lines of response, got %s" % ( - expected_lines, len(stdout))) + # + # def test_pssh_client_run_long_command(self): + # expected_lines = 5 + # output = self.client.run_command(self.long_cmd(expected_lines)) + # self.assertEqual(len(output), len(self.client.hosts)) + # stdout = list(output[0].stdout) + # self.client.join(output) + # self.assertTrue(len(stdout) == expected_lines, + # msg="Expected %s lines of response, got %s" % ( + # expected_lines, len(stdout))) def test_pssh_client_auth_failure(self): client = ParallelSSHClient([self.host], port=self.port, @@ -280,28 +280,28 @@ def test_zero_timeout(self): cmd = spawn(client.run_command, 'sleep 1', stop_on_errors=False) output = cmd.get(timeout=3) self.assertTrue(output[0].exception is None) - - def test_pssh_client_long_running_command_exit_codes(self): - expected_lines = 2 - output = self.client.run_command(self.long_cmd(expected_lines)) - self.assertEqual(len(output), len(self.client.hosts)) - self.assertIsNone(output[0].exit_code) - self.assertFalse(self.client.finished(output)) - self.client.join(output, consume_output=True) - self.assertTrue(self.client.finished(output)) - self.assertEqual(output[0].exit_code, 0) - - def test_pssh_client_long_running_command_exit_codes_no_stdout(self): - expected_lines = 2 - output = self.client.run_command(self.long_cmd(expected_lines)) - self.assertEqual(len(output), len(self.client.hosts)) - self.assertIsNone(output[0].exit_code) - self.assertFalse(self.client.finished(output)) - self.client.join(output) - self.assertTrue(self.client.finished(output)) - self.assertEqual(output[0].exit_code, 0) - stdout = list(output[0].stdout) - self.assertEqual(expected_lines, len(stdout)) + # + # def test_pssh_client_long_running_command_exit_codes(self): + # expected_lines = 2 + # output = self.client.run_command(self.long_cmd(expected_lines)) + # self.assertEqual(len(output), len(self.client.hosts)) + # self.assertIsNone(output[0].exit_code) + # self.assertFalse(self.client.finished(output)) + # self.client.join(output, consume_output=True) + # self.assertTrue(self.client.finished(output)) + # self.assertEqual(output[0].exit_code, 0) + # + # def test_pssh_client_long_running_command_exit_codes_no_stdout(self): + # expected_lines = 2 + # output = self.client.run_command(self.long_cmd(expected_lines)) + # self.assertEqual(len(output), len(self.client.hosts)) + # self.assertIsNone(output[0].exit_code) + # self.assertFalse(self.client.finished(output)) + # self.client.join(output) + # self.assertTrue(self.client.finished(output)) + # self.assertEqual(output[0].exit_code, 0) + # stdout = list(output[0].stdout) + # self.assertEqual(expected_lines, len(stdout)) def test_connection_error_exception(self): """Test that we get connection error exception in output with correct arguments""" @@ -334,7 +334,7 @@ def test_multiple_single_quotes_in_cmd(self): output = self.client.run_command("echo 'me' 'and me'") stdout = list(output[0].stdout) expected = 'me and me' - self.assertTrue(len(stdout)==1, + self.assertTrue(len(stdout) == 1, msg="Got incorrect number of lines in output - %s" % (stdout,)) self.assertEqual(output[0].exit_code, 0) self.assertEqual(expected, stdout[0], @@ -484,8 +484,9 @@ def test_read_multi_same_hosts(self): self.client.run_command(self.cmd), ] for output in outputs: - for host_out in output: + for i, host_out in enumerate(output): stdout = list(host_out.stdout) + self.assertEqual(host_out.client.host, hosts[i]) self.assertListEqual(stdout, [self.resp]) def test_join_bad_host_out(self): diff --git a/tests/ssh/test_single_client.py b/ci/integration_tests/ssh/test_single_client.py similarity index 95% rename from tests/ssh/test_single_client.py rename to ci/integration_tests/ssh/test_single_client.py index b109ba1f..7857308c 100644 --- a/tests/ssh/test_single_client.py +++ b/ci/integration_tests/ssh/test_single_client.py @@ -50,7 +50,9 @@ def test_execute(self): output = list(host_out.stdout) stderr = list(host_out.stderr) expected = [self.resp] + expected_stderr = [] self.assertEqual(expected, output) + self.assertEqual(expected_stderr, stderr) exit_code = host_out.channel.get_exit_status() self.assertEqual(exit_code, 0) @@ -186,11 +188,10 @@ def test_identity_auth_failure(self): def test_password_auth_failure(self): try: - client = SSHClient(self.host, port=self.port, num_retries=1, - allow_agent=False, - identity_auth=False, - password='blah blah blah', - ) + SSHClient(self.host, port=self.port, num_retries=1, allow_agent=False, + identity_auth=False, + password='blah blah blah', + ) except AuthenticationException as ex: self.assertIsInstance(ex.args[3], AuthenticationDenied) else: @@ -244,6 +245,7 @@ def test_client_read_timeout(self): def test_open_session_exc(self): class Error(Exception): pass + def _session(): raise Error client = SSHClient(self.host, port=self.port, @@ -255,6 +257,7 @@ def _session(): def test_session_connect_exc(self): class Error(Exception): pass + def _con(): raise Error client = SSHClient(self.host, port=self.port, @@ -282,8 +285,10 @@ def test_no_auth(self): def test_agent_auth_failure(self): class UnknownError(Exception): pass + def _agent_auth_unk(): raise UnknownError + def _agent_auth_agent_err(): raise AuthenticationDenied client = SSHClient(self.host, port=self.port, @@ -316,6 +321,7 @@ def _agent_auth(): def test_disconnect_exc(self): class DiscError(Exception): pass + def _disc(): raise DiscError client = SSHClient(self.host, port=self.port, @@ -326,3 +332,11 @@ def _disc(): client._disconnect_eagain = _disc client._connect_init_session_retry(0) client.disconnect() + + def test_stdin(self): + host_out = self.client.run_command('read line; echo $line') + host_out.stdin.write('a line\n') + host_out.stdin.flush() + self.client.wait_finished(host_out) + stdout = list(host_out.stdout) + self.assertListEqual(stdout, ['a line']) diff --git a/tests/test_client_private_key b/ci/integration_tests/test_client_private_key similarity index 100% rename from tests/test_client_private_key rename to ci/integration_tests/test_client_private_key diff --git a/tests/test_client_private_key_dsa b/ci/integration_tests/test_client_private_key_dsa similarity index 100% rename from tests/test_client_private_key_dsa rename to ci/integration_tests/test_client_private_key_dsa diff --git a/tests/test_client_private_key_ecdsa b/ci/integration_tests/test_client_private_key_ecdsa similarity index 100% rename from tests/test_client_private_key_ecdsa rename to ci/integration_tests/test_client_private_key_ecdsa diff --git a/pssh/__init__.py b/pssh/__init__.py index bce2c66a..2a425bd0 100644 --- a/pssh/__init__.py +++ b/pssh/__init__.py @@ -29,9 +29,9 @@ from logging import getLogger, NullHandler -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions +from . import _version +__version__ = _version.get_versions()['version'] +del _version host_logger = getLogger('pssh.host_logger') logger = getLogger('pssh') diff --git a/pssh/_version.py b/pssh/_version.py index f5d6cce1..187f483a 100644 --- a/pssh/_version.py +++ b/pssh/_version.py @@ -5,8 +5,9 @@ # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) +# This file is released into the public domain. +# Generated by versioneer-0.29 +# https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" @@ -15,9 +16,11 @@ import re import subprocess import sys +from typing import Any, Callable, Dict, List, Optional, Tuple +import functools -def get_keywords(): +def get_keywords() -> Dict[str, str]: """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must @@ -33,8 +36,15 @@ def get_keywords(): class VersioneerConfig: """Container for Versioneer configuration parameters.""" + VCS: str + style: str + tag_prefix: str + parentdir_prefix: str + versionfile_source: str + verbose: bool -def get_config(): + +def get_config() -> VersioneerConfig: """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py @@ -52,13 +62,13 @@ class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): +def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} @@ -67,22 +77,35 @@ def decorate(f): return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands: List[str], + args: List[str], + cwd: Optional[str] = None, + verbose: bool = False, + hide_stderr: bool = False, + env: Optional[Dict[str, str]] = None, +) -> Tuple[Optional[str], Optional[int]]: """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + + popen_kwargs: Dict[str, Any] = {} + if sys.platform == "win32": + # This hides the console window if pythonw.exe is used + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + popen_kwargs["startupinfo"] = startupinfo + + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None), **popen_kwargs) break - except EnvironmentError: - e = sys.exc_info()[1] + except OSError as e: if e.errno == errno.ENOENT: continue if verbose: @@ -93,18 +116,20 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, if verbose: print("unable to find command, tried %s" % (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode -def versions_from_parentdir(parentdir_prefix, root, verbose): +def versions_from_parentdir( + parentdir_prefix: str, + root: str, + verbose: bool, +) -> Dict[str, Any]: """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both @@ -113,15 +138,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % @@ -130,41 +154,48 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): @register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): +def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. - keywords = {} + keywords: Dict[str, str] = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): +def git_versions_from_keywords( + keywords: Dict[str, str], + tag_prefix: str, + verbose: bool, +) -> Dict[str, Any]: """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -177,11 +208,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -190,7 +221,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -199,6 +230,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %s" % r) return {"version": r, @@ -214,7 +250,12 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs( + tag_prefix: str, + root: str, + verbose: bool, + runner: Callable = run_command +) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -225,8 +266,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + # GIT_DIR can interfere with correct operation of Versioneer. + # It may be intended to be passed to the Versioneer-versioned project, + # but that should not change where we get our version from. + env = os.environ.copy() + env.pop("GIT_DIR", None) + runner = functools.partial(runner, env=env) + + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -234,24 +282,57 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = runner(GITS, [ + "describe", "--tags", "--dirty", "--always", "--long", + "--match", f"{tag_prefix}[[:digit:]]*" + ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() - pieces = {} + pieces: Dict[str, Any] = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -268,7 +349,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces @@ -293,26 +374,27 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits + out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) + pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces -def plus_or_dot(pieces): +def plus_or_dot(pieces: Dict[str, Any]) -> str: """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" -def render_pep440(pieces): +def render_pep440(pieces: Dict[str, Any]) -> str: """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you @@ -337,23 +419,71 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces: Dict[str, Any]) -> str: + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces: Dict[str, Any]) -> str: + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] + rendered = "0.post0.dev%d" % pieces["distance"] return rendered -def render_pep440_post(pieces): +def render_pep440_post(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards @@ -380,12 +510,41 @@ def render_pep440_post(pieces): return rendered -def render_pep440_old(pieces): +def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_old(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -402,7 +561,7 @@ def render_pep440_old(pieces): return rendered -def render_git_describe(pieces): +def render_git_describe(pieces: Dict[str, Any]) -> str: """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. @@ -422,7 +581,7 @@ def render_git_describe(pieces): return rendered -def render_git_describe_long(pieces): +def render_git_describe_long(pieces: Dict[str, Any]) -> str: """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. @@ -442,7 +601,7 @@ def render_git_describe_long(pieces): return rendered -def render(pieces, style): +def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", @@ -456,10 +615,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -474,7 +637,7 @@ def render(pieces, style): "date": pieces.get("date")} -def get_versions(): +def get_versions() -> Dict[str, Any]: """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some @@ -495,7 +658,7 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, diff --git a/pssh/clients/__init__.py b/pssh/clients/__init__.py index eb9b9d1a..6ac14d3e 100644 --- a/pssh/clients/__init__.py +++ b/pssh/clients/__init__.py @@ -1,6 +1,6 @@ # This file is part of parallel-ssh. # -# Copyright (C) 2014-2022 Panos Kittenis and contributors. +# Copyright (C) 2014-2025 Panos Kittenis and contributors. # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,5 +16,4 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # flake8: noqa: F401 -from .native.parallel import ParallelSSHClient -from .native.single import SSHClient +from .native import ParallelSSHClient, SSHClient diff --git a/pssh/clients/base/single.py b/pssh/clients/base/single.py index 6229e0f4..447db6f1 100644 --- a/pssh/clients/base/single.py +++ b/pssh/clients/base/single.py @@ -68,6 +68,8 @@ def write(self, data): def flush(self): """Flush pending data written to stdin.""" + if not hasattr(self._channel, "flush"): + return return self._client._eagain(self._channel.flush) diff --git a/pssh/clients/native/single.py b/pssh/clients/native/single.py index 3a225b55..4a05355e 100644 --- a/pssh/clients/native/single.py +++ b/pssh/clients/native/single.py @@ -218,6 +218,7 @@ def configure_keepalive(self): def _init_session(self, retries=1): self.session = Session() + if self.timeout: # libssh2 timeout is in ms self.session.set_timeout(self.timeout * 1000) @@ -666,9 +667,9 @@ def scp_send(self, local_file, remote_file, recurse=False, sftp=None): elif remote_file.endswith('/'): local_filename = local_file.rsplit('/')[-1] remote_file += local_filename - self._scp_send(local_file, remote_file) logger.info("SCP local file %s to remote destination %s:%s", local_file, self.host, remote_file) + self._scp_send(local_file, remote_file) def _scp_send(self, local_file, remote_file): fileinfo = os.stat(local_file) diff --git a/pssh/clients/native/tunnel.py b/pssh/clients/native/tunnel.py index 30c9ca01..ed937d8a 100644 --- a/pssh/clients/native/tunnel.py +++ b/pssh/clients/native/tunnel.py @@ -16,9 +16,8 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging - -from threading import Thread, Event from queue import Queue +from threading import Thread, Event from gevent import spawn, joinall, get_hub, sleep from gevent.server import StreamServer @@ -26,7 +25,6 @@ from ...constants import DEFAULT_RETRIES - logger = logging.getLogger(__name__) diff --git a/pssh/clients/reader.py b/pssh/clients/reader.py index c6b69b2a..bcb8685d 100644 --- a/pssh/clients/reader.py +++ b/pssh/clients/reader.py @@ -34,11 +34,13 @@ def set(self): class ConcurrentRWBuffer(object): """Concurrent reader/writer of bytes for use from multiple greenlets. - Supports both concurrent reading and writing. + Supports both concurrent reading and writing and combinations there of. Iterate on buffer object to read data, yielding event loop if no data exists until self.eof has been set. + Check if end-of-file without blocking with ``ConcurrentRWBuffer.eof.is_set()``. + Writers should call ``ConcurrentRWBuffer.eof.set()`` when finished writing data via ``write``. Readers can use ``read()`` to get any available data or ``None``. diff --git a/requirements.txt b/requirements.txt index 6a7d0069..eaba1f61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ gevent>=1.3.0 -ssh2-python>=1.0.0 -ssh-python>=0.10.0 +ssh2-python>=1.1.0 +ssh-python>=1.1.0 diff --git a/setup.py b/setup.py index 3087e9d1..3882f687 100644 --- a/setup.py +++ b/setup.py @@ -43,11 +43,11 @@ 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: System :: Networking', 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 3b95c7ed..10fb983f 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -16,7 +16,6 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import unittest -from logging import NullHandler from pssh.exceptions import AuthenticationError, AuthenticationException, UnknownHostError, \ UnknownHostException, ConnectionError, ConnectionErrorException, SSHError, SSHException, \ @@ -68,4 +67,3 @@ def test_errors(self): raise HostArgumentError except HostArgumentException: pass - diff --git a/tests/test_output.py b/tests/test_output.py index 15ad3dd6..cd083c08 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -44,6 +44,7 @@ def test_bad_exit_status(self): def test_excepting_client_exit_code(self): class ChannelError(Exception): pass + class ExcSSHClient(object): def get_exit_status(self, channel): raise ChannelError diff --git a/tests/test_reader.py b/tests/test_reader.py index b353f80c..45ad8fef 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -16,12 +16,12 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import unittest - -from random import random, randint, randrange +from random import randrange from string import ascii_letters -from gevent.queue import Queue from gevent import spawn, sleep +from gevent.queue import Queue + from pssh.clients.reader import ConcurrentRWBuffer @@ -55,24 +55,25 @@ def test_multi_write_read(self): def test_concurrent_rw(self): written_data = Queue() + def _writer(_buffer): while True: - data = b"".join([ascii_letters[m].encode() for m in [randrange(0, 8) for _ in range(8)]]) - _buffer.write(data) - written_data.put(data) + temp_data = b"".join([ascii_letters[m].encode() for m in [randrange(0, 8) for _ in range(8)]]) + _buffer.write(temp_data) + written_data.put(temp_data) sleep(0.2) writer = spawn(_writer, self.buffer) writer.start() sleep(0.5) data = self.buffer.read() _data = b"" - while written_data.qsize() !=0 : + while written_data.qsize() != 0: _data += written_data.get() self.assertEqual(data, _data) sleep(0.5) data = self.buffer.read() _data = b"" - while written_data.qsize() !=0 : + while written_data.qsize() != 0: _data += written_data.get() self.assertEqual(data, _data) writer.kill() diff --git a/tests/test_utils.py b/tests/test_utils.py index 1346e28b..0fcd83a1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -20,6 +20,7 @@ from pssh import utils + class ParallelSSHUtilsTest(unittest.TestCase): def test_enabling_host_logger(self): diff --git a/versioneer.py b/versioneer.py index a2870603..1e3753e6 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,5 +1,5 @@ -# Version: 0.18-1 +# Version: 0.29 """The Versioneer - like a rocketeer, but for versions. @@ -7,18 +7,14 @@ ============== * like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer +* https://github.com/python-versioneer/python-versioneer * Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based +* License: Public Domain (Unlicense) +* Compatible with: Python 3.7, 3.8, 3.9, 3.10, 3.11 and pypy3 +* [![Latest Version][pypi-image]][pypi-url] +* [![Build Status][travis-image]][travis-url] + +This is a tool for managing a recorded version number in setuptools-based python projects. The goal is to remove the tedious and error-prone "update the embedded version string" step from your release process. Making a new release should be as easy as recording a new tag in your version-control @@ -27,9 +23,38 @@ ## Quick Install -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results +Versioneer provides two installation modes. The "classic" vendored mode installs +a copy of versioneer into your repository. The experimental build-time dependency mode +is intended to allow you to skip this step and simplify the process of upgrading. + +### Vendored mode + +* `pip install versioneer` to somewhere in your $PATH + * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is + available, so you can also use `conda install -c conda-forge versioneer` +* add a `[tool.versioneer]` section to your `pyproject.toml` or a + `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) + * Note that you will need to add `tomli; python_version < "3.11"` to your + build-time dependencies if you use `pyproject.toml` +* run `versioneer install --vendor` in your source tree, commit the results +* verify version information with `python setup.py version` + +### Build-time dependency mode + +* `pip install versioneer` to somewhere in your $PATH + * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is + available, so you can also use `conda install -c conda-forge versioneer` +* add a `[tool.versioneer]` section to your `pyproject.toml` or a + `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) +* add `versioneer` (with `[toml]` extra, if configuring in `pyproject.toml`) + to the `requires` key of the `build-system` table in `pyproject.toml`: + ```toml + [build-system] + requires = ["setuptools", "versioneer[toml]"] + build-backend = "setuptools.build_meta" + ``` +* run `versioneer install --no-vendor` in your source tree, commit the results +* verify version information with `python setup.py version` ## Version Identifiers @@ -61,7 +86,7 @@ for example `git describe --tags --dirty --always` reports things like "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. +uncommitted changes). The version identifier is used for multiple purposes: @@ -166,7 +191,7 @@ Some situations are known to cause problems for Versioneer. This details the most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). +[issues page](https://github.com/python-versioneer/python-versioneer/issues). ### Subprojects @@ -180,7 +205,7 @@ `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI distributions (and upload multiple independently-installable tarballs). * Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. + provide bindings to Python (and perhaps other languages) in subdirectories. Versioneer will look for `.git` in parent directories, and most operations should get the right version string. However `pip` and `setuptools` have bugs @@ -194,9 +219,9 @@ Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in some later version. -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking +[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the +[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the issue from the Versioneer side in more detail. [pip PR#3176](https://github.com/pypa/pip/pull/3176) and [pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve @@ -224,31 +249,20 @@ cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into a different virtualenv), so this can be surprising. -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes +[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes this one, but upgrading to a newer version of setuptools should probably resolve it. -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - ## Updating Versioneer To upgrade your project to a new release of Versioneer, do the following: * install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace +* edit `setup.cfg` and `pyproject.toml`, if necessary, + to include any new configuration settings indicated by the release notes. + See [UPGRADING](./UPGRADING.md) for details. +* re-run `versioneer install --[no-]vendor` in your source tree, to replace `SRC/_version.py` * commit any changed files @@ -265,35 +279,70 @@ direction and include code from all supported VCS systems, reducing the number of intermediate scripts. +## Similar projects + +* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time + dependency +* [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of + versioneer +* [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools + plugin ## License To make Versioneer easier to embed, all its code is dedicated to the public domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . +Specifically, both are released under the "Unlicense", as described in +https://unlicense.org/. + +[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg +[pypi-url]: https://pypi.python.org/pypi/versioneer/ +[travis-image]: +https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg +[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer """ +# pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring +# pylint:disable=missing-class-docstring,too-many-branches,too-many-statements +# pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error +# pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with +# pylint:disable=attribute-defined-outside-init,too-many-arguments -from __future__ import print_function -try: - import configparser -except ImportError: - import ConfigParser as configparser +import configparser import errno import json import os import re import subprocess import sys +from pathlib import Path +from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Union +from typing import NoReturn +import functools + +have_tomllib = True +if sys.version_info >= (3, 11): + import tomllib +else: + try: + import tomli as tomllib + except ImportError: + have_tomllib = False class VersioneerConfig: """Container for Versioneer configuration parameters.""" + VCS: str + style: str + tag_prefix: str + versionfile_source: str + versionfile_build: Optional[str] + parentdir_prefix: Optional[str] + verbose: Optional[bool] + -def get_root(): +def get_root() -> str: """Get the project root directory. We require that all commands are run from the project root, i.e. the @@ -301,13 +350,23 @@ def get_root(): """ root = os.path.realpath(os.path.abspath(os.getcwd())) setup_py = os.path.join(root, "setup.py") + pyproject_toml = os.path.join(root, "pyproject.toml") versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + if not ( + os.path.exists(setup_py) + or os.path.exists(pyproject_toml) + or os.path.exists(versioneer_py) + ): # allow 'python path/to/setup.py COMMAND' root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) setup_py = os.path.join(root, "setup.py") + pyproject_toml = os.path.join(root, "pyproject.toml") versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + if not ( + os.path.exists(setup_py) + or os.path.exists(pyproject_toml) + or os.path.exists(versioneer_py) + ): err = ("Versioneer was unable to run the project root directory. " "Versioneer requires setup.py to be executed from " "its immediate directory (like 'python setup.py COMMAND'), " @@ -321,43 +380,62 @@ def get_root(): # module-import table will cache the first one. So we can't use # os.path.dirname(__file__), as that will find whichever # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) + my_path = os.path.realpath(os.path.abspath(__file__)) + me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: + if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + % (os.path.dirname(my_path), versioneer_py)) except NameError: pass return root -def get_config_from_root(root): +def get_config_from_root(root: str) -> VersioneerConfig: """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or + # This might raise OSError (if setup.cfg is missing), or # configparser.NoSectionError (if it lacks a [versioneer] section), or # configparser.NoOptionError (if it lacks "VCS="). See the docstring at # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None + root_pth = Path(root) + pyproject_toml = root_pth / "pyproject.toml" + setup_cfg = root_pth / "setup.cfg" + section: Union[Dict[str, Any], configparser.SectionProxy, None] = None + if pyproject_toml.exists() and have_tomllib: + try: + with open(pyproject_toml, 'rb') as fobj: + pp = tomllib.load(fobj) + section = pp['tool']['versioneer'] + except (tomllib.TOMLDecodeError, KeyError) as e: + print(f"Failed to load config from {pyproject_toml}: {e}") + print("Try to load it from setup.cfg") + if not section: + parser = configparser.ConfigParser() + with open(setup_cfg) as cfg_file: + parser.read_file(cfg_file) + parser.get("versioneer", "VCS") # raise error if missing + + section = parser["versioneer"] + + # `cast`` really shouldn't be used, but its simplest for the + # common VersioneerConfig users at the moment. We verify against + # `None` values elsewhere where it matters + cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): + cfg.VCS = section['VCS'] + cfg.style = section.get("style", "") + cfg.versionfile_source = cast(str, section.get("versionfile_source")) + cfg.versionfile_build = section.get("versionfile_build") + cfg.tag_prefix = cast(str, section.get("tag_prefix")) + if cfg.tag_prefix in ("''", '""', None): cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") + cfg.parentdir_prefix = section.get("parentdir_prefix") + if isinstance(section, configparser.SectionProxy): + # Make sure configparser translates to bool + cfg.verbose = section.getboolean("verbose") + else: + cfg.verbose = section.get("verbose") + return cfg @@ -366,37 +444,48 @@ class NotThisMethod(Exception): # these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): +def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f + HANDLERS.setdefault(vcs, {})[method] = f return f return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands: List[str], + args: List[str], + cwd: Optional[str] = None, + verbose: bool = False, + hide_stderr: bool = False, + env: Optional[Dict[str, str]] = None, +) -> Tuple[Optional[str], Optional[int]]: """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + + popen_kwargs: Dict[str, Any] = {} + if sys.platform == "win32": + # This hides the console window if pythonw.exe is used + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + popen_kwargs["startupinfo"] = startupinfo + + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None), **popen_kwargs) break - except EnvironmentError: - e = sys.exc_info()[1] + except OSError as e: if e.errno == errno.ENOENT: continue if verbose: @@ -407,26 +496,25 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, if verbose: print("unable to find command, tried %s" % (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY['git'] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) +# This file is released into the public domain. +# Generated by versioneer-0.29 +# https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" @@ -435,9 +523,11 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, import re import subprocess import sys +from typing import Any, Callable, Dict, List, Optional, Tuple +import functools -def get_keywords(): +def get_keywords() -> Dict[str, str]: """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must @@ -453,8 +543,15 @@ def get_keywords(): class VersioneerConfig: """Container for Versioneer configuration parameters.""" + VCS: str + style: str + tag_prefix: str + parentdir_prefix: str + versionfile_source: str + verbose: bool + -def get_config(): +def get_config() -> VersioneerConfig: """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py @@ -472,13 +569,13 @@ class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): +def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} @@ -487,22 +584,35 @@ def decorate(f): return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands: List[str], + args: List[str], + cwd: Optional[str] = None, + verbose: bool = False, + hide_stderr: bool = False, + env: Optional[Dict[str, str]] = None, +) -> Tuple[Optional[str], Optional[int]]: """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + + popen_kwargs: Dict[str, Any] = {} + if sys.platform == "win32": + # This hides the console window if pythonw.exe is used + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + popen_kwargs["startupinfo"] = startupinfo + + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None), **popen_kwargs) break - except EnvironmentError: - e = sys.exc_info()[1] + except OSError as e: if e.errno == errno.ENOENT: continue if verbose: @@ -513,18 +623,20 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, if verbose: print("unable to find command, tried %%s" %% (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %%s (error)" %% dispcmd) print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode -def versions_from_parentdir(parentdir_prefix, root, verbose): +def versions_from_parentdir( + parentdir_prefix: str, + root: str, + verbose: bool, +) -> Dict[str, Any]: """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both @@ -533,15 +645,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print("Tried directories %%s but none started with prefix %%s" %% @@ -550,41 +661,48 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): @register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): +def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. - keywords = {} + keywords: Dict[str, str] = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): +def git_versions_from_keywords( + keywords: Dict[str, str], + tag_prefix: str, + verbose: bool, +) -> Dict[str, Any]: """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -597,11 +715,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %%d @@ -610,7 +728,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%%s', no digits" %% ",".join(refs - tags)) if verbose: @@ -619,6 +737,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %%s" %% r) return {"version": r, @@ -634,7 +757,12 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs( + tag_prefix: str, + root: str, + verbose: bool, + runner: Callable = run_command +) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -645,8 +773,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + # GIT_DIR can interfere with correct operation of Versioneer. + # It may be intended to be passed to the Versioneer-versioned project, + # but that should not change where we get our version from. + env = os.environ.copy() + env.pop("GIT_DIR", None) + runner = functools.partial(runner, env=env) + + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %%s not under git control" %% root) @@ -654,24 +789,57 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) + describe_out, rc = runner(GITS, [ + "describe", "--tags", "--dirty", "--always", "--long", + "--match", f"{tag_prefix}[[:digit:]]*" + ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() - pieces = {} + pieces: Dict[str, Any] = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -688,7 +856,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%%s'" %% describe_out) return pieces @@ -713,26 +881,27 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits + out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) + pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() + date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces -def plus_or_dot(pieces): +def plus_or_dot(pieces: Dict[str, Any]) -> str: """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" -def render_pep440(pieces): +def render_pep440(pieces: Dict[str, Any]) -> str: """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you @@ -757,23 +926,71 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces: Dict[str, Any]) -> str: + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces: Dict[str, Any]) -> str: + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%%d.dev%%d" %% (post_version + 1, pieces["distance"]) + else: + rendered += ".post0.dev%%d" %% (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] + rendered = "0.post0.dev%%d" %% pieces["distance"] return rendered -def render_pep440_post(pieces): +def render_pep440_post(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards @@ -800,12 +1017,41 @@ def render_pep440_post(pieces): return rendered -def render_pep440_old(pieces): +def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_old(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -822,7 +1068,7 @@ def render_pep440_old(pieces): return rendered -def render_git_describe(pieces): +def render_git_describe(pieces: Dict[str, Any]) -> str: """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. @@ -842,7 +1088,7 @@ def render_git_describe(pieces): return rendered -def render_git_describe_long(pieces): +def render_git_describe_long(pieces: Dict[str, Any]) -> str: """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. @@ -862,7 +1108,7 @@ def render_git_describe_long(pieces): return rendered -def render(pieces, style): +def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", @@ -876,10 +1122,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -894,7 +1144,7 @@ def render(pieces, style): "date": pieces.get("date")} -def get_versions(): +def get_versions() -> Dict[str, Any]: """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some @@ -915,7 +1165,7 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, @@ -942,41 +1192,48 @@ def get_versions(): @register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): +def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. - keywords = {} + keywords: Dict[str, str] = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): +def git_versions_from_keywords( + keywords: Dict[str, str], + tag_prefix: str, + verbose: bool, +) -> Dict[str, Any]: """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -989,11 +1246,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1002,7 +1259,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1011,6 +1268,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %s" % r) return {"version": r, @@ -1026,7 +1288,12 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs( + tag_prefix: str, + root: str, + verbose: bool, + runner: Callable = run_command +) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -1037,8 +1304,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + # GIT_DIR can interfere with correct operation of Versioneer. + # It may be intended to be passed to the Versioneer-versioned project, + # but that should not change where we get our version from. + env = os.environ.copy() + env.pop("GIT_DIR", None) + runner = functools.partial(runner, env=env) + + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1046,24 +1320,57 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = runner(GITS, [ + "describe", "--tags", "--dirty", "--always", "--long", + "--match", f"{tag_prefix}[[:digit:]]*" + ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() - pieces = {} + pieces: Dict[str, Any] = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -1080,7 +1387,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces @@ -1105,19 +1412,20 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits + out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) + pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces -def do_vcs_install(manifest_in, versionfile_source, ipy): +def do_vcs_install(versionfile_source: str, ipy: Optional[str]) -> None: """Git-specific installation logic for Versioneer. For Git, this means creating/changing .gitattributes to mark _version.py @@ -1126,36 +1434,40 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] + files = [versionfile_source] if ipy: files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) + if "VERSIONEER_PEP518" not in globals(): + try: + my_path = __file__ + if my_path.endswith((".pyc", ".pyo")): + my_path = os.path.splitext(my_path)[0] + ".py" + versioneer_file = os.path.relpath(my_path) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) present = False try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: + with open(".gitattributes", "r") as fobj: + for line in fobj: + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + break + except OSError: pass if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() + with open(".gitattributes", "a+") as fobj: + fobj.write(f"{versionfile_source} export-subst\n") files.append(".gitattributes") run_command(GITS, ["add", "--"] + files) -def versions_from_parentdir(parentdir_prefix, root, verbose): +def versions_from_parentdir( + parentdir_prefix: str, + root: str, + verbose: bool, +) -> Dict[str, Any]: """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both @@ -1164,15 +1476,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % @@ -1181,7 +1492,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from +# This file was generated by 'versioneer.py' (0.29) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. @@ -1198,12 +1509,12 @@ def get_versions(): """ -def versions_from_file(filename): +def versions_from_file(filename: str) -> Dict[str, Any]: """Try to determine the version from _version.py if present.""" try: with open(filename) as f: contents = f.read() - except EnvironmentError: + except OSError: raise NotThisMethod("unable to read _version.py") mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) @@ -1215,9 +1526,8 @@ def versions_from_file(filename): return json.loads(mo.group(1)) -def write_to_version_file(filename, versions): +def write_to_version_file(filename: str, versions: Dict[str, Any]) -> None: """Write the given version number to the given _version.py file.""" - os.unlink(filename) contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: @@ -1226,14 +1536,14 @@ def write_to_version_file(filename, versions): print("set %s to '%s'" % (filename, versions["version"])) -def plus_or_dot(pieces): +def plus_or_dot(pieces: Dict[str, Any]) -> str: """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" -def render_pep440(pieces): +def render_pep440(pieces: Dict[str, Any]) -> str: """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you @@ -1258,23 +1568,71 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces: Dict[str, Any]) -> str: + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces: Dict[str, Any]) -> str: + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] + rendered = "0.post0.dev%d" % pieces["distance"] return rendered -def render_pep440_post(pieces): +def render_pep440_post(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards @@ -1301,12 +1659,41 @@ def render_pep440_post(pieces): return rendered -def render_pep440_old(pieces): +def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_old(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -1323,7 +1710,7 @@ def render_pep440_old(pieces): return rendered -def render_git_describe(pieces): +def render_git_describe(pieces: Dict[str, Any]) -> str: """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. @@ -1343,7 +1730,7 @@ def render_git_describe(pieces): return rendered -def render_git_describe_long(pieces): +def render_git_describe_long(pieces: Dict[str, Any]) -> str: """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. @@ -1363,7 +1750,7 @@ def render_git_describe_long(pieces): return rendered -def render(pieces, style): +def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", @@ -1377,10 +1764,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -1399,7 +1790,7 @@ class VersioneerBadRootError(Exception): """The project root directory is unknown or missing key files.""" -def get_versions(verbose=False): +def get_versions(verbose: bool = False) -> Dict[str, Any]: """Get the project version from whatever source is available. Returns dict with two keys: 'version' and 'full'. @@ -1414,7 +1805,7 @@ def get_versions(verbose=False): assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose + verbose = verbose or bool(cfg.verbose) # `bool()` used to avoid `None` assert cfg.versionfile_source is not None, \ "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" @@ -1475,13 +1866,17 @@ def get_versions(verbose=False): "date": None} -def get_version(): +def get_version() -> str: """Get the short version string for this project.""" return get_versions()["version"] -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" +def get_cmdclass(cmdclass: Optional[Dict[str, Any]] = None): + """Get the custom setuptools subclasses used by Versioneer. + + If the package uses a different cmdclass (e.g. one from numpy), it + should be provide as an argument. + """ if "versioneer" in sys.modules: del sys.modules["versioneer"] # this fixes the "python setup.py develop" case (also 'install' and @@ -1495,25 +1890,25 @@ def get_cmdclass(): # parent is protected against the child's "import versioneer". By # removing ourselves from sys.modules here, before the child build # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 + # Also see https://github.com/python-versioneer/python-versioneer/issues/52 - cmds = {} + cmds = {} if cmdclass is None else cmdclass.copy() - # we add "version" to both distutils and setuptools - from distutils.core import Command + # we add "version" to setuptools + from setuptools import Command class cmd_version(Command): description = "report generated version string" - user_options = [] - boolean_options = [] + user_options: List[Tuple[str, str, str]] = [] + boolean_options: List[str] = [] - def initialize_options(self): + def initialize_options(self) -> None: pass - def finalize_options(self): + def finalize_options(self) -> None: pass - def run(self): + def run(self) -> None: vers = get_versions(verbose=True) print("Version: %s" % vers["version"]) print(" full-revisionid: %s" % vers.get("full-revisionid")) @@ -1523,7 +1918,7 @@ def run(self): print(" error: %s" % vers["error"]) cmds["version"] = cmd_version - # we override "build_py" in both distutils and setuptools + # we override "build_py" in setuptools # # most invocation pathways end up running build_py: # distutils/build -> build_py @@ -1538,18 +1933,25 @@ def run(self): # then does setup.py bdist_wheel, or sometimes setup.py install # setup.py egg_info -> ? + # pip install -e . and setuptool/editable_wheel will invoke build_py + # but the build_py command is not expected to copy any files. + # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py + if 'build_py' in cmds: + _build_py: Any = cmds['build_py'] else: - from distutils.command.build_py import build_py as _build_py + from setuptools.command.build_py import build_py as _build_py class cmd_build_py(_build_py): - def run(self): + def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() _build_py.run(self) + if getattr(self, "editable_mode", False): + # During editable installs `.py` and data files are + # not copied to build_lib + return # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: @@ -1559,33 +1961,40 @@ def run(self): write_to_version_file(target_versionfile, versions) cmds["build_py"] = cmd_build_py - if "setuptools" in sys.modules: - from setuptools.command.build_ext import build_ext as _build_ext + if 'build_ext' in cmds: + _build_ext: Any = cmds['build_ext'] else: - from distutils.command.build_ext import build_ext as _build_ext + from setuptools.command.build_ext import build_ext as _build_ext class cmd_build_ext(_build_ext): - def run(self): + def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() _build_ext.run(self) if self.inplace: - # build_ext --inplace will only build modules in + # build_ext --inplace will only build extensions in # build/lib<..> dir with no _version.py to write to. # As in place builds will already have a _version.py # in the module dir, we do not need to write one. return # now locate _version.py in the new build/ directory and replace # it with an updated value + if not cfg.versionfile_build: + return target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_source) + cfg.versionfile_build) + if not os.path.exists(target_versionfile): + print(f"Warning: {target_versionfile} does not exist, skipping " + "version update. This can happen if you are running build_ext " + "without first running build_py.") + return print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) cmds["build_ext"] = cmd_build_ext if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe + from cx_Freeze.dist import build_exe as _build_exe # type: ignore # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -1594,7 +2003,7 @@ def run(self): # ... class cmd_build_exe(_build_exe): - def run(self): + def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() @@ -1618,12 +2027,12 @@ def run(self): if 'py2exe' in sys.modules: # py2exe enabled? try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 + from py2exe.setuptools_buildexe import py2exe as _py2exe # type: ignore except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 + from py2exe.distutils_buildexe import py2exe as _py2exe # type: ignore class cmd_py2exe(_py2exe): - def run(self): + def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() @@ -1644,14 +2053,51 @@ def run(self): }) cmds["py2exe"] = cmd_py2exe + # sdist farms its file list building out to egg_info + if 'egg_info' in cmds: + _egg_info: Any = cmds['egg_info'] + else: + from setuptools.command.egg_info import egg_info as _egg_info + + class cmd_egg_info(_egg_info): + def find_sources(self) -> None: + # egg_info.find_sources builds the manifest list and writes it + # in one shot + super().find_sources() + + # Modify the filelist and normalize it + root = get_root() + cfg = get_config_from_root(root) + self.filelist.append('versioneer.py') + if cfg.versionfile_source: + # There are rare cases where versionfile_source might not be + # included by default, so we must be explicit + self.filelist.append(cfg.versionfile_source) + self.filelist.sort() + self.filelist.remove_duplicates() + + # The write method is hidden in the manifest_maker instance that + # generated the filelist and was thrown away + # We will instead replicate their final normalization (to unicode, + # and POSIX-style paths) + from setuptools import unicode_utils + normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') + for f in self.filelist.files] + + manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') + with open(manifest_filename, 'w') as fobj: + fobj.write('\n'.join(normalized)) + + cmds['egg_info'] = cmd_egg_info + # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist + if 'sdist' in cmds: + _sdist: Any = cmds['sdist'] else: - from distutils.command.sdist import sdist as _sdist + from setuptools.command.sdist import sdist as _sdist class cmd_sdist(_sdist): - def run(self): + def run(self) -> None: versions = get_versions() self._versioneer_generated_versions = versions # unless we update this, the command will keep using the old @@ -1659,7 +2105,7 @@ def run(self): self.distribution.metadata.version = versions["version"] return _sdist.run(self) - def make_release_tree(self, base_dir, files): + def make_release_tree(self, base_dir: str, files: List[str]) -> None: root = get_root() cfg = get_config_from_root(root) _sdist.make_release_tree(self, base_dir, files) @@ -1712,21 +2158,26 @@ def make_release_tree(self, base_dir, files): """ -INIT_PY_SNIPPET = """ +OLD_SNIPPET = """ from ._version import get_versions __version__ = get_versions()['version'] del get_versions """ +INIT_PY_SNIPPET = """ +from . import {0} +__version__ = {0}.get_versions()['version'] +""" -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" + +def do_setup() -> int: + """Do main VCS-independent setup function for installing Versioneer.""" root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + if isinstance(e, (OSError, configparser.NoSectionError)): print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: @@ -1746,62 +2197,37 @@ def do_setup(): ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") + maybe_ipy: Optional[str] = ipy if os.path.exists(ipy): try: with open(ipy, "r") as f: old = f.read() - except EnvironmentError: + except OSError: old = "" - if INIT_PY_SNIPPET not in old: + module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] + snippet = INIT_PY_SNIPPET.format(module) + if OLD_SNIPPET in old: + print(" replacing boilerplate in %s" % ipy) + with open(ipy, "w") as f: + f.write(old.replace(OLD_SNIPPET, snippet)) + elif snippet not in old: print(" appending to %s" % ipy) with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) + f.write(snippet) else: print(" %s unmodified" % ipy) else: print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") + maybe_ipy = None # Make VCS-specific changes. For git, this means creating/changing # .gitattributes to mark _version.py for export-subst keyword # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + do_vcs_install(cfg.versionfile_source, maybe_ipy) return 0 -def scan_setup_py(): +def scan_setup_py() -> int: """Validate the contents of setup.py against Versioneer's expectations.""" found = set() setters = False @@ -1838,10 +2264,14 @@ def scan_setup_py(): return errors +def setup_command() -> NoReturn: + """Set up Versioneer and exit with appropriate error code.""" + errors = do_setup() + errors += scan_setup_py() + sys.exit(1 if errors else 0) + + if __name__ == "__main__": cmd = sys.argv[1] if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) + setup_command()