diff --git a/mypy.ini b/mypy.ini index 8c8acc0b9..39bd93b50 100644 --- a/mypy.ini +++ b/mypy.ini @@ -8,6 +8,11 @@ warn_unused_configs = True warn_redundant_casts = True warn_incomplete_stub = True follow_imports = normal +show_error_codes = True + +# Ignore errors in the docs/conf.py file +[mypy-conf] +ignore_errors = True # TODO: burn these down [mypy-tests.*] diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index 02fb02b8d..b6b1ab8ec 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -426,7 +426,7 @@ def _make_api_call(self, operation_name, operation_kwargs): code = data.get('__type', '') if '#' in code: code = code.rsplit('#', 1)[1] - botocore_expected_format = {'Error': {'Message': data.get('message', ''), 'Code': code}} + botocore_expected_format = {'Error': {'Message': data.get('message', '') or data.get('Message', ''), 'Code': code}} verbose_properties = { 'request_id': headers.get('x-amzn-RequestId') } diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..58797eacf --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +markers = + ddblocal: requires an mock dynamodb server running on localhost:8000 +env = + AWS_ACCESS_KEY_ID=1 + AWS_SECRET_ACCESS_KEY=2 diff --git a/requirements-dev.txt b/requirements-dev.txt index b16ebf016..5e5c26963 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,13 +1,12 @@ mock -pytest +pytest>=6 +pytest-env pytest-mock -# Due to https://github.com/boto/botocore/issues/1872. Remove after botocore fixes. -python-dateutil==2.8.0 - -# only used in .travis.yml +# only used in CI coveralls -mypy==0.761;python_version>="3.7" +mypy==0.950;python_version>="3.7" pytest-cov -sphinx -sphinx-rtd-theme + +# used for type-checking +botocore-stubs diff --git a/setup.py b/setup.py index f8f5c17ec..875d0bac5 100644 --- a/setup.py +++ b/setup.py @@ -31,12 +31,16 @@ def find_stubs(package): setup( name='pynamodb', version=__import__('pynamodb').__version__, - packages=find_packages(exclude=('tests', 'tests.integration',)), + packages=find_packages(exclude=('examples', 'tests', 'typing_tests', 'tests.integration',)), url='http://jlafon.io/pynamodb.html', + project_urls={ + 'Source': 'https://github.com/pynamodb/PynamoDB', + }, author='Jharrod LaFon', author_email='jlafon@eyesopen.com', description='A Pythonic Interface to DynamoDB', long_description=open('README.rst').read(), + long_description_content_type='text/x-rst', zip_safe=False, license='MIT', keywords='python dynamodb amazon', diff --git a/tests/mypy_helpers.py b/tests/mypy_helpers.py deleted file mode 100644 index beed7a600..000000000 --- a/tests/mypy_helpers.py +++ /dev/null @@ -1,48 +0,0 @@ -import re -import sys -from collections import defaultdict -from tempfile import TemporaryDirectory -from textwrap import dedent -from typing import Dict -from typing import Iterable -from typing import List - -import mypy.api - - -def _run_mypy(program: str) -> Iterable[str]: - with TemporaryDirectory() as tempdirname: - with open(f'{tempdirname}/__main__.py', 'w') as f: - f.write(program) - error_pattern = re.compile(fr'^{re.escape(f.name)}:(\d+): (?:error|note): (.*)$') - stdout, stderr, exit_status = mypy.api.run([ - f.name, - '--show-traceback', - ]) - if stderr: - print(stderr, file=sys.stderr) # allow "printf debugging" of the plugin - - # Group errors by line - errors_by_line: Dict[int, List[str]] = defaultdict(list) - for line in stdout.split('\n'): - m = error_pattern.match(line) - if m: - errors_by_line[int(m.group(1))].append(m.group(2)) - elif line: - print('x', repr(line)) # allow "printf debugging" of the plugin - - # Reconstruct the "actual" program with "error" comments - error_comment_pattern = re.compile(r'(\s+# E: .*)?$') - for line_no, line in enumerate(program.split('\n'), start=1): - line = error_comment_pattern.sub('', line) - errors = errors_by_line.get(line_no) - if errors: - yield line + ''.join(f' # E: {error}' for error in errors) - else: - yield line - - -def assert_mypy_output(program: str) -> None: - program = dedent(program).strip() - actual = '\n'.join(_run_mypy(program)) - assert actual == program diff --git a/tests/test_mypy.py b/tests/test_mypy.py deleted file mode 100644 index 784b9392b..000000000 --- a/tests/test_mypy.py +++ /dev/null @@ -1,210 +0,0 @@ -""" -Note: The expected error strings may change in a future version of mypy. - Please update as needed. -""" -import pytest - -pytest.importorskip('mypy') # we only install mypy in python>=3.6 tests -pytest.register_assert_rewrite('tests.mypy_helpers') -from .mypy_helpers import assert_mypy_output # noqa - - -def test_model(): - assert_mypy_output(""" - from pynamodb.models import Model - from pynamodb.expressions.operand import Path - - class MyModel(Model): - pass - - reveal_type(MyModel.count('hash', Path('a').between(1, 3))) # E: Revealed type is 'builtins.int' - """) - - -def test_model_query(): - assert_mypy_output(""" - from pynamodb.attributes import NumberAttribute - from pynamodb.models import Model - - class MyModel(Model): - my_attr = NumberAttribute() - - # test conditions - MyModel.query(123, range_key_condition=(MyModel.my_attr == 5), filter_condition=(MyModel.my_attr == 5)) - - # test conditions are optional - MyModel.query(123, range_key_condition=None, filter_condition=None) - """) - - -def test_pagination(): - assert_mypy_output(""" - from pynamodb.attributes import NumberAttribute - from pynamodb.models import Model - - class MyModel(Model): - my_attr = NumberAttribute() - - result_iterator = MyModel.query(123) - for model in result_iterator: - reveal_type(model) # E: Revealed type is '__main__.MyModel*' - if result_iterator.last_evaluated_key: - reveal_type(result_iterator.last_evaluated_key['my_attr']) # E: Revealed type is 'builtins.dict*[builtins.str, Any]' - """) - - -def test_model_update(): - assert_mypy_output(""" - from pynamodb.attributes import NumberAttribute - from pynamodb.models import Model - - class MyModel(Model): - my_attr = NumberAttribute() - - my_model = MyModel() - my_model.update(actions=[ - # test update expressions - MyModel.my_attr.set(MyModel.my_attr + 123), - MyModel.my_attr.set(123 + MyModel.my_attr), - MyModel.my_attr.set(MyModel.my_attr - 123), - MyModel.my_attr.set(123 - MyModel.my_attr), - MyModel.my_attr.set(MyModel.my_attr | 123), - ]) - """) # noqa: E501 - - -def test_number_attribute(): - assert_mypy_output(""" - from pynamodb.attributes import NumberAttribute - from pynamodb.models import Model - - class MyModel(Model): - my_attr = NumberAttribute() - - reveal_type(MyModel.my_attr) # E: Revealed type is 'pynamodb.attributes.NumberAttribute*' - reveal_type(MyModel().my_attr) # E: Revealed type is 'builtins.float' - """) - - -def test_unicode_attribute(): - assert_mypy_output(""" - from pynamodb.attributes import UnicodeAttribute - from pynamodb.models import Model - - class MyModel(Model): - my_attr = UnicodeAttribute() - - reveal_type(MyModel.my_attr) # E: Revealed type is 'pynamodb.attributes.UnicodeAttribute*' - reveal_type(MyModel().my_attr) # E: Revealed type is 'builtins.str' - """) - - -def test_map_attribute(): - assert_mypy_output(""" - from pynamodb.attributes import MapAttribute, UnicodeAttribute - from pynamodb.models import Model - - class MySubMap(MapAttribute): - s = UnicodeAttribute() - - class MyMap(MapAttribute): - m2 = MySubMap() - - class MyModel(Model): - m1 = MyMap() - - reveal_type(MyModel.m1) # E: Revealed type is '__main__.MyMap' - reveal_type(MyModel().m1) # E: Revealed type is '__main__.MyMap' - reveal_type(MyModel.m1.m2) # E: Revealed type is '__main__.MySubMap' - reveal_type(MyModel().m1.m2) # E: Revealed type is '__main__.MySubMap' - reveal_type(MyModel.m1.m2.s) # E: Revealed type is 'builtins.str' - reveal_type(MyModel().m1.m2.s) # E: Revealed type is 'builtins.str' - - reveal_type(MyMap.m2) # E: Revealed type is '__main__.MySubMap' - reveal_type(MyMap().m2) # E: Revealed type is '__main__.MySubMap' - - reveal_type(MySubMap.s) # E: Revealed type is 'pynamodb.attributes.UnicodeAttribute*' - reveal_type(MySubMap().s) # E: Revealed type is 'builtins.str' - """) - - -def test_list_attribute(): - assert_mypy_output(""" - from pynamodb.attributes import ListAttribute, MapAttribute, UnicodeAttribute - from pynamodb.models import Model - - class MyMap(MapAttribute): - my_sub_attr = UnicodeAttribute() - - class MyModel(Model): - my_list = ListAttribute(of=MyMap) - my_untyped_list = ListAttribute() # E: Need type annotation for 'my_untyped_list' - - reveal_type(MyModel.my_list) # E: Revealed type is 'pynamodb.attributes.ListAttribute[__main__.MyMap]' - reveal_type(MyModel().my_list) # E: Revealed type is 'builtins.list[__main__.MyMap*]' - reveal_type(MyModel().my_list[0].my_sub_attr) # E: Revealed type is 'builtins.str' - - # Untyped lists are not well supported yet - reveal_type(MyModel().my_untyped_list[0].my_sub_attr) # E: Revealed type is 'Any' - """) - - -def test_paths(): - assert_mypy_output(""" - from pynamodb.attributes import ListAttribute, MapAttribute, UnicodeAttribute - from pynamodb.models import Model - - class MyMap(MapAttribute): - my_sub_attr = UnicodeAttribute() - - class MyModel(Model): - my_list = ListAttribute(of=MyMap) - my_map = MyMap() - - reveal_type(MyModel.my_list[0]) # E: Revealed type is 'pynamodb.expressions.operand.Path' - reveal_type(MyModel.my_list[0] == MyModel()) # E: Revealed type is 'pynamodb.expressions.condition.Comparison' - # the following string indexing is not type checked - not by mypy nor in runtime - reveal_type(MyModel.my_list[0]['my_sub_attr'] == 'foobar') # E: Revealed type is 'pynamodb.expressions.condition.Comparison' - """) - - -def test_index_query_scan(): - assert_mypy_output(""" - from pynamodb.attributes import NumberAttribute - from pynamodb.models import Model - from pynamodb.indexes import GlobalSecondaryIndex - from pynamodb.pagination import ResultIterator - - class UntypedIndex(GlobalSecondaryIndex): - bar = NumberAttribute(hash_key=True) - - class TypedIndex(GlobalSecondaryIndex[MyModel]): - bar = NumberAttribute(hash_key=True) - - class MyModel(Model): - foo = NumberAttribute(hash_key=True) - bar = NumberAttribute() - - untyped_index = UntypedIndex() - typed_index = TypedIndex() - - # Ensure old code keeps working - untyped_result: ResultIterator = MyModel.untyped_index.query(123) - model: MyModel = next(untyped_result) - not_model: int = next(untyped_result) # this is legacy behavior so it's "fine" - - # Allow users to specify which model their indices return - typed_result: ResultIterator[MyModel] = MyModel.typed_index.query(123) - my_model = next(typed_result) - not_model = next(typed_result) # E: Incompatible types in assignment (expression has type "MyModel", variable has type "int") - - # Ensure old code keeps working - untyped_result = MyModel.untyped_index.scan() - model = next(untyped_result) - not_model = next(untyped_result) # this is legacy behavior so it's "fine" - - # Allow users to specify which model their indices return - untyped_result = MyModel.typed_index.scan() - model = next(untyped_result) - not_model = next(untyped_result) # E: Incompatible types in assignment (expression has type "MyModel", variable has type "int") - """) diff --git a/tests/test_settings.py b/tests/test_settings.py index 15067c758..afc0c1b60 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -18,5 +18,5 @@ def test_override_old_attributes(settings_str, tmpdir): with patch.dict('os.environ', {'PYNAMODB_CONFIG': str(custom_settings)}): with pytest.warns(UserWarning) as warns: reload_module(pynamodb.settings) - assert len(warns) == 1 - assert 'options are no longer supported' in str(warns[0].message) + + assert any(('options are no longer supported' in str(warn.message)) for warn in warns) diff --git a/typing_tests/__init__.py b/typing_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/typing_tests/attributes.py b/typing_tests/attributes.py new file mode 100644 index 000000000..4765585ca --- /dev/null +++ b/typing_tests/attributes.py @@ -0,0 +1,72 @@ +from __future__ import annotations +from typing import Any + +from typing_extensions import assert_type + + +def test_number_attribute() -> None: + from pynamodb.attributes import NumberAttribute + from pynamodb.models import Model + + class MyModel(Model): + my_attr = NumberAttribute() + + assert_type(MyModel.my_attr, NumberAttribute) + assert_type(MyModel().my_attr, float) + + +def test_unicode_attribute() -> None: + from pynamodb.attributes import UnicodeAttribute + from pynamodb.models import Model + + class MyModel(Model): + my_attr = UnicodeAttribute() + + assert_type(MyModel.my_attr, UnicodeAttribute) + assert_type(MyModel().my_attr, str) + + +def test_map_attribute() -> None: + from pynamodb.attributes import MapAttribute, UnicodeAttribute + from pynamodb.models import Model + + class MySubMap(MapAttribute): + s = UnicodeAttribute() + + class MyMap(MapAttribute): + m2 = MySubMap() + + class MyModel(Model): + m1 = MyMap() + + assert_type(MyModel.m1, MyMap) + assert_type(MyModel().m1, MyMap) + assert_type(MyModel.m1.m2, MySubMap) + assert_type(MyModel().m1.m2, MySubMap) + assert_type(MyModel.m1.m2.s, str) + assert_type(MyModel().m1.m2.s, str) + + assert_type(MyMap.m2, MySubMap) + assert_type(MyMap().m2, MySubMap) + + assert_type(MySubMap.s, UnicodeAttribute) + assert_type(MySubMap().s, str) + + +def test_list_attribute() -> None: + from pynamodb.attributes import ListAttribute, MapAttribute, UnicodeAttribute + from pynamodb.models import Model + + class MyMap(MapAttribute): + my_sub_attr = UnicodeAttribute() + + class MyModel(Model): + my_list = ListAttribute(of=MyMap) + my_untyped_list = ListAttribute() # type: ignore[var-annotated] + + assert_type(MyModel.my_list, ListAttribute[MyMap]) + assert_type(MyModel().my_list, list[MyMap]) + assert_type(MyModel().my_list[0].my_sub_attr, str) + + # Untyped lists are not well-supported yet + assert_type(MyModel().my_untyped_list[0].my_sub_attr, Any) diff --git a/typing_tests/models.py b/typing_tests/models.py new file mode 100644 index 000000000..b6a9ccd88 --- /dev/null +++ b/typing_tests/models.py @@ -0,0 +1,124 @@ +from __future__ import annotations +from typing import Any + +from typing_extensions import assert_type + + +def test_model_count() -> None: + from pynamodb.models import Model + from pynamodb.expressions.operand import Path + + class MyModel(Model): + pass + + assert_type(MyModel.count('hash', Path('a').between(1, 3)), int) + + +def test_model_query() -> None: + from pynamodb.attributes import NumberAttribute + from pynamodb.models import Model + + class MyModel(Model): + my_attr = NumberAttribute() + + # test conditions + MyModel.query(123, range_key_condition=(MyModel.my_attr == 5), filter_condition=(MyModel.my_attr == 5)) + + # test conditions are optional + MyModel.query(123, range_key_condition=None, filter_condition=None) + + +def test_pagination() -> None: + from pynamodb.attributes import NumberAttribute + from pynamodb.models import Model + + class MyModel(Model): + my_attr = NumberAttribute() + + result_iterator = MyModel.query(123) + for model in result_iterator: + assert_type(model, MyModel) + if result_iterator.last_evaluated_key: + assert_type(result_iterator.last_evaluated_key['my_attr'], dict[str, Any]) + + +def test_model_update() -> None: + from pynamodb.attributes import NumberAttribute, UnicodeAttribute + from pynamodb.models import Model + + class MyModel(Model): + my_attr = NumberAttribute() + my_str_attr = UnicodeAttribute() + + my_model = MyModel() + my_model.update(actions=[ + # test update expressions + MyModel.my_attr.set(MyModel.my_attr + 123), + MyModel.my_attr.set(123 + MyModel.my_attr), + MyModel.my_attr.set(MyModel.my_attr - 123), + MyModel.my_attr.set(123 - MyModel.my_attr), + MyModel.my_attr.set(MyModel.my_attr | 123), + ]) + + +def test_paths() -> None: + import pynamodb.expressions.operand + import pynamodb.expressions.condition + from pynamodb.attributes import ListAttribute, MapAttribute, UnicodeAttribute + from pynamodb.models import Model + + class MyMap(MapAttribute): + my_sub_attr = UnicodeAttribute() + + class MyModel(Model): + my_list = ListAttribute(of=MyMap) + my_map = MyMap() + + assert_type(MyModel.my_list[0], pynamodb.expressions.operand.Path) + assert_type(MyModel.my_list[0] == MyModel(), pynamodb.expressions.condition.Comparison) + # the following string indexing is not type checked - not by mypy nor in runtime + assert_type(MyModel.my_list[0]['my_sub_attr'] == 'foobar', pynamodb.expressions.condition.Comparison) + assert_type(MyModel.my_map == 'foobar', pynamodb.expressions.condition.Comparison) + + +def test_index_query_scan() -> None: + from pynamodb.attributes import NumberAttribute + from pynamodb.models import Model + from pynamodb.indexes import GlobalSecondaryIndex + from pynamodb.pagination import ResultIterator + + class UntypedIndex(GlobalSecondaryIndex): + bar = NumberAttribute(hash_key=True) + + class TypedIndex(GlobalSecondaryIndex['MyModel']): + bar = NumberAttribute(hash_key=True) + + class MyModel(Model): + foo = NumberAttribute(hash_key=True) + bar = NumberAttribute() + + untyped_index = UntypedIndex() + typed_index = TypedIndex() + + # Ensure old code keeps working + untyped_query_result: ResultIterator = MyModel.untyped_index.query(123) + assert_type(next(untyped_query_result), Any) + + # Allow users to specify which model their indices return + typed_query_result: ResultIterator[MyModel] = MyModel.typed_index.query(123) + assert_type(next(typed_query_result), MyModel) + + # Ensure old code keeps working + untyped_scan_result = MyModel.untyped_index.scan() + assert_type(next(untyped_scan_result), Any) + + # Allow users to specify which model their indices return + typed_scan_result = MyModel.typed_index.scan() + assert_type(next(typed_scan_result), MyModel) + + +def test_map_attribute_derivation() -> None: + from pynamodb.attributes import MapAttribute + + class MyMap(MapAttribute, object): + pass