From c034516baed4b9a949b299439bfcc0851fe23053 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 26 Aug 2022 14:18:04 -0400 Subject: [PATCH 001/342] Update some GitHub Action versions --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 30d8c0de9..b43b72f84 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,8 +28,8 @@ jobs: steps: - run: git config --global core.autocrlf false - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - run: pip install . && rm -r hy From a5a2021309ba1742ad89d935c9898dfb4f3f3f13 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 26 Aug 2022 14:20:11 -0400 Subject: [PATCH 002/342] On GitHub, test a newer PyPy --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b43b72f84..5554cc5d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: matrix: name-prefix: [''] os: [ubuntu-latest] - python: [3.7, 3.8, 3.9, '3.10', pypy-3.8] + python: [3.7, 3.8, 3.9, '3.10', pypy-3.9] include: # To keep the overall number of runs low, we test Windows # only on the latest CPython. From e1d58c8f22c13fac8591c014c3087afcd2ea564b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 26 Aug 2022 14:42:18 -0400 Subject: [PATCH 003/342] Fix `__file__` handling for PyPy 3.9 --- hy/cmdline.py | 11 ++++++----- tests/test_bin.py | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/hy/cmdline.py b/hy/cmdline.py index 76f905c4b..e1e634986 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -19,7 +19,7 @@ from pathlib import Path import hy -from hy._compat import PY3_9 +from hy._compat import PY3_9, PYPY from hy.compiler import HyASTCompiler, hy_ast_compile_flags, hy_compile, hy_eval from hy.completer import Completer, completion from hy.errors import ( @@ -684,10 +684,11 @@ def proc_opt(opt, arg=None, item=None, i=None): set_path(filename) # Ensure __file__ is set correctly in the code we're about # to run. - if PY3_9 and not filename.is_absolute(): - filename = Path.cwd() / filename - if PY3_9 and platform.system() == "Windows": - filename = os.path.normpath(filename) + if PY3_9 and not PYPY: + if not filename.is_absolute(): + filename = Path.cwd() / filename + if platform.system() == "Windows": + filename = os.path.normpath(filename) try: sys.argv = argv diff --git a/tests/test_bin.py b/tests/test_bin.py index 45bae911a..06881b5f1 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -12,7 +12,7 @@ import pytest -from hy._compat import PY3_9 +from hy._compat import PY3_9, PYPY def pyr(s=""): @@ -663,7 +663,7 @@ def test_uufileuu(tmp_path, monkeypatch): (tmp_path / "realdir" / "pyex.py").write_text('print(__file__)') def file_is(arg, expected_py3_9): - expected = expected_py3_9 if PY3_9 else Path(arg) + expected = expected_py3_9 if PY3_9 and not PYPY else Path(arg) output, _ = run_cmd("python3 " + shlex.quote(arg + "pyex.py")) assert output.rstrip() == str(expected / "pyex.py") output, _ = run_cmd("hy " + shlex.quote(arg + "hyex.hy")) From bdfdc1a0e69ac7f689706fcaaca66c7e5c1fe145 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 26 Aug 2022 15:15:14 -0400 Subject: [PATCH 004/342] Skip a test that fails from a PyPy regression --- tests/test_bin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_bin.py b/tests/test_bin.py index 06881b5f1..517db4434 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -522,6 +522,7 @@ def test_macro_require(): assert output.strip() == "abc" +@pytest.mark.skipif(PYPY, reason = 'https://foss.heptapod.net/pypy/pypy/-/issues/3800') def test_tracebacks(): """Make sure the printed tracebacks are correct.""" From 71c5152b993bb47fae19b5fa3a33115e84056b63 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 26 Aug 2022 16:27:03 -0400 Subject: [PATCH 005/342] Declare Python 3.11 compatibility --- .github/workflows/tests.yml | 4 ++-- NEWS.rst | 4 ++++ hy/_compat.py | 1 + setup.py | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5554cc5d2..842a97ac5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,13 +10,13 @@ jobs: matrix: name-prefix: [''] os: [ubuntu-latest] - python: [3.7, 3.8, 3.9, '3.10', pypy-3.9] + python: [3.7, 3.8, 3.9, '3.10', '3.11-dev', pypy-3.9] include: # To keep the overall number of runs low, we test Windows # only on the latest CPython. - name-prefix: 'win-' os: windows-latest - python: '3.10' + python: '3.11-dev' name: ${{ format('{0}{1}', matrix.name-prefix, matrix.python) }} runs-on: ${{ matrix.os }} diff --git a/NEWS.rst b/NEWS.rst index 0df14fc63..0d2a2597d 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,10 @@ Other Breaking Changes `(dfor x (range 5) [x (* 2 x)])` is now `(dfor x (range 5) x (* 2 x))`. +New Features +------------------------------ +* Python 3.11 is now supported. + Bug Fixes ------------------------------ * Fixed `hy.repr` of `slice` objects with non-integer arguments. diff --git a/hy/_compat.py b/hy/_compat.py index 815f3a5e5..ba709e092 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -5,6 +5,7 @@ PY3_8 = sys.version_info >= (3, 8) PY3_9 = sys.version_info >= (3, 9) PY3_10 = sys.version_info >= (3, 10) +PY3_11 = sys.version_info >= (3, 11) PYPY = platform.python_implementation() == "PyPy" diff --git a/setup.py b/setup.py index cacaa0968..8f427082e 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def run(self): version=__version__, setup_requires=["wheel"] + requires, install_requires=requires, - python_requires=">= 3.7, < 3.11", + python_requires=">= 3.7, < 3.12", entry_points={ "console_scripts": [ "hy = hy.cmdline:hy_main", @@ -81,6 +81,7 @@ def run(self): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Code Generators", "Topic :: Software Development :: Compilers", "Topic :: Software Development :: Libraries", From 7f79dd49c3b23889a5922faa9dd3aa0fd014d7bd Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 26 Aug 2022 16:02:23 -0400 Subject: [PATCH 006/342] Make an exception test a little more sensitive --- tests/macros/test_macro_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py index 4b4f9c848..5fb317911 100644 --- a/tests/macros/test_macro_processor.py +++ b/tests/macros/test_macro_processor.py @@ -40,7 +40,7 @@ def test_preprocessor_exceptions(): """Test that macro expansion raises appropriate exceptions""" with pytest.raises(HyMacroExpansionError) as excinfo: macroexpand(read("(when)"), __name__, HyASTCompiler(__name__)) - assert "_hy_anon_" not in excinfo.value.msg + assert "TypeError: when()" in excinfo.value.msg def test_macroexpand_nan(): From 11ceffccb065ac28b81b3cc717a64336b97916db Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 26 Aug 2022 16:01:58 -0400 Subject: [PATCH 007/342] Update `rename_function` for Python 3.11 --- hy/macros.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hy/macros.py b/hy/macros.py index 91f726593..a0682733a 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -3,6 +3,7 @@ import inspect import os import pkgutil +import re import sys import traceback import warnings @@ -11,7 +12,7 @@ from funcparserlib.parser import NoParseError import hy.compiler -from hy._compat import code_replace +from hy._compat import code_replace, PY3_11 from hy.errors import ( HyLanguageError, HyMacroExpansionError, @@ -439,7 +440,13 @@ def macroexpand_1(tree, module, compiler=None): def rename_function(f, new_name): """Create a copy of a function, but with a new name.""" f = type(f)( - code_replace(f.__code__, co_name=new_name), + code_replace(f.__code__, + co_name = new_name, + **({'co_qualname': + re.sub(r'\.[^.+]\Z', '.' + new_name, f.__code__.co_qualname) + if "." in f.__code__.co_qualname + else new_name} + if PY3_11 else {})), f.__globals__, str(new_name), f.__defaults__, From b0435b7d6cefc59b5821c8040ce68eadc1238e6e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 26 Aug 2022 16:15:40 -0400 Subject: [PATCH 008/342] Remove a leftover debugging print --- tests/test_bin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_bin.py b/tests/test_bin.py index 517db4434..f4646e887 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -552,7 +552,6 @@ def req_err(x): output, error = run_cmd('hy -i "(require not-a-real-module)"') assert output.startswith("=> ") - print(error.splitlines()) req_err(error.splitlines()[2]) # Modeled after From bdc1fadc751d9630b60a77a5fe56a47a909ea603 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 26 Aug 2022 16:24:47 -0400 Subject: [PATCH 009/342] Update some traceback tests for Python 3.11 --- tests/test_bin.py | 3 ++- tests/test_reader.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_bin.py b/tests/test_bin.py index f4646e887..1d5b5b32e 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -587,7 +587,8 @@ def req_err(x): # File "", line 1, in # NameError: name 'a' is not defined output, error = run_cmd('hy -c "(print a)"', expect=1) - error_lines = error.splitlines() + # Filter out the underline added by Python 3.11. + error_lines = [x for x in error.splitlines() if set(x) != {" ", "^"}] assert error_lines[3] == ' File "", line 1, in ' # PyPy will add "global" to this error message, so we work around that. assert error_lines[-1].strip().replace(" global", "") == ( diff --git a/tests/test_reader.py b/tests/test_reader.py index c2e76efe0..586cd0e81 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -48,7 +48,9 @@ def check_trace_output(capsys, execinfo, expected): hy_exc_handler(execinfo.type, execinfo.value, execinfo.tb) captured_w_filtering = capsys.readouterr()[-1].strip("\n") - output = [x.rstrip() for x in captured_w_filtering.split("\n")] + output = [x.rstrip() + for x in captured_w_filtering.split("\n") + if "^^^" not in x] # Make sure the filtered frames aren't the same as the unfiltered ones. assert output != captured_wo_filtering.split("\n") From 9bce14a9669caebd2549a07ef744aaf909ec6ef5 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 30 Aug 2022 13:26:11 -0400 Subject: [PATCH 010/342] Autoformat --- hy/macros.py | 22 +++++++++++++++------- tests/test_reader.py | 4 +--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/hy/macros.py b/hy/macros.py index a0682733a..294310564 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -12,7 +12,7 @@ from funcparserlib.parser import NoParseError import hy.compiler -from hy._compat import code_replace, PY3_11 +from hy._compat import PY3_11, code_replace from hy.errors import ( HyLanguageError, HyMacroExpansionError, @@ -440,13 +440,21 @@ def macroexpand_1(tree, module, compiler=None): def rename_function(f, new_name): """Create a copy of a function, but with a new name.""" f = type(f)( - code_replace(f.__code__, - co_name = new_name, - **({'co_qualname': - re.sub(r'\.[^.+]\Z', '.' + new_name, f.__code__.co_qualname) + code_replace( + f.__code__, + co_name=new_name, + **( + { + "co_qualname": re.sub( + r"\.[^.+]\Z", "." + new_name, f.__code__.co_qualname + ) if "." in f.__code__.co_qualname - else new_name} - if PY3_11 else {})), + else new_name + } + if PY3_11 + else {} + ), + ), f.__globals__, str(new_name), f.__defaults__, diff --git a/tests/test_reader.py b/tests/test_reader.py index 586cd0e81..f8815a550 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -48,9 +48,7 @@ def check_trace_output(capsys, execinfo, expected): hy_exc_handler(execinfo.type, execinfo.value, execinfo.tb) captured_w_filtering = capsys.readouterr()[-1].strip("\n") - output = [x.rstrip() - for x in captured_w_filtering.split("\n") - if "^^^" not in x] + output = [x.rstrip() for x in captured_w_filtering.split("\n") if "^^^" not in x] # Make sure the filtered frames aren't the same as the unfiltered ones. assert output != captured_wo_filtering.split("\n") From b50d5d7a04fe10e3e5a41de763c8009f0a244bb2 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 5 Sep 2022 15:30:59 -0400 Subject: [PATCH 011/342] Inline `compile_catch_expression` --- hy/core/result_macros.py | 83 +++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 44 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index cb043e0a8..cbd0244f0 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1287,8 +1287,45 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo handler_results = Result() handlers = [] for catcher in catchers: - handler_results += compile_catch_expression( - compiler, catcher, return_var, *catcher + # exceptions catch should be either: + # [[list of exceptions]] + # or + # [variable [list of exceptions]] + # or + # [variable exception] + # or + # [exception] + # or + # [] + exceptions, ebody = catcher + + name = None + if len(exceptions) == 2: + name = mangle(compiler._nonconst(exceptions[0])) + + exceptions_list = exceptions[-1] if exceptions else List() + if isinstance(exceptions_list, List): + if len(exceptions_list): + # [FooBar BarFoo] → catch Foobar and BarFoo exceptions + elts, types, _ = compiler._compile_collect(exceptions_list) + types += asty.Tuple(exceptions_list, elts=elts, ctx=ast.Load()) + else: + # [] → all exceptions caught + types = Result() + else: + types = compiler.compile(exceptions_list) + + # Create a "fake" scope for the exception variable. + # See: https://docs.python.org/3/reference/compound_stmts.html#the-try-statement + with compiler.scope.create(ScopeLet) as scope: + if name: + scope.add(name, name) + ebody = compiler._compile_branch(ebody) + ebody += asty.Assign(catcher, targets=[return_var], value=ebody.force_expr) + ebody += ebody.expr_as_stmt() + + handler_results += types + asty.ExceptHandler( + catcher, type=types.expr, name=name, body=ebody.stmts or [asty.Pass(catcher)] ) handlers.append(handler_results.stmts.pop()) @@ -1331,48 +1368,6 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo return handler_results + x + returnable -def compile_catch_expression(compiler, expr, var, exceptions, body): - # exceptions catch should be either: - # [[list of exceptions]] - # or - # [variable [list of exceptions]] - # or - # [variable exception] - # or - # [exception] - # or - # [] - - name = None - if len(exceptions) == 2: - name = mangle(compiler._nonconst(exceptions[0])) - - exceptions_list = exceptions[-1] if exceptions else List() - if isinstance(exceptions_list, List): - if len(exceptions_list): - # [FooBar BarFoo] → catch Foobar and BarFoo exceptions - elts, types, _ = compiler._compile_collect(exceptions_list) - types += asty.Tuple(exceptions_list, elts=elts, ctx=ast.Load()) - else: - # [] → all exceptions caught - types = Result() - else: - types = compiler.compile(exceptions_list) - - # Create a "fake" scope for the exception variable. - # See: https://docs.python.org/3/reference/compound_stmts.html#the-try-statement - with compiler.scope.create(ScopeLet) as scope: - if name: - scope.add(name, name) - body = compiler._compile_branch(body) - body += asty.Assign(expr, targets=[var], value=body.force_expr) - body += body.expr_as_stmt() - - return types + asty.ExceptHandler( - expr, type=types.expr, name=name, body=body.stmts or [asty.Pass(expr)] - ) - - # ------------------------------------------------ # * Functions and macros # ------------------------------------------------ From c2235c090721b1535d43a9a3bc3a3b0454705d1f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 5 Sep 2022 15:59:57 -0400 Subject: [PATCH 012/342] Support `except*` in `try` --- hy/core/result_macros.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index cbd0244f0..d0dc3a9e7 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -15,6 +15,7 @@ from funcparserlib.parser import finished, forward_decl, many, maybe, oneplus, some +from hy._compat import PY3_11 from hy.compiler import Result, asty, hy_eval, mkexpr from hy.errors import HyEvalError, HyInternalError, HyTypeError from hy.macros import pattern_macro, require, require_reader @@ -1267,10 +1268,10 @@ def compile_raise_expression(compiler, expr, root, exc, cause): @pattern_macro( "try", [ - many(notpexpr("except", "else", "finally")), + many(notpexpr("except", "except*", "else", "finally")), many( pexpr( - sym("except"), + keepsym("except") | keepsym("except*"), brackets() | brackets(FORM) | brackets(SYM, FORM), many(FORM), ) @@ -1286,6 +1287,7 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo handler_results = Result() handlers = [] + except_syms_seen = set() for catcher in catchers: # exceptions catch should be either: # [[list of exceptions]] @@ -1297,7 +1299,13 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo # [exception] # or # [] - exceptions, ebody = catcher + except_sym, exceptions, ebody = catcher + if not PY3_11 and except_sym == Symbol("except*"): + hy_compiler._syntax_error(except_sym, + "`{}` requires Python 3.11 or later") + except_syms_seen.add(str(except_sym)) + if len(except_syms_seen) > 1: + raise compiler._syntax_error(except_sym, "cannot have both `except` and `except*` on the same `try`") name = None if len(exceptions) == 2: @@ -1364,7 +1372,9 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo ) body = body.stmts or [asty.Pass(expr)] - x = asty.Try(expr, body=body, handlers=handlers, orelse=orelse, finalbody=finalbody) + x = (asty.TryStar if "except*" in except_syms_seen else asty.Try)( + expr, body=body, handlers=handlers, orelse=orelse, finalbody=finalbody + ) return handler_results + x + returnable From 20464681e18a6b6775152038cf9ee4920378384b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 5 Sep 2022 16:01:02 -0400 Subject: [PATCH 013/342] Test `except*` --- conftest.py | 3 ++- tests/compilers/test_ast.py | 11 +++++++++-- tests/native_tests/py3_11_only_tests.hy | 23 +++++++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 tests/native_tests/py3_11_only_tests.hy diff --git a/conftest.py b/conftest.py index 6b28427d3..442c3993e 100644 --- a/conftest.py +++ b/conftest.py @@ -8,7 +8,7 @@ import pytest import hy -from hy._compat import PY3_8, PY3_10 +from hy._compat import PY3_8, PY3_10, PY3_11 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "") @@ -21,6 +21,7 @@ def pytest_ignore_collect(path, config): (sys.version_info < (3, 8), "sub_py3_7_only"), (PY3_8, "py3_8_only"), (PY3_10, "py3_10_only"), + (PY3_11, "py3_11_only"), ] return ( diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index c38e55b73..7083be6b7 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -4,6 +4,7 @@ import pytest +from hy._compat import PY3_11 from hy.compiler import hy_compile, hy_eval from hy.errors import HyError, HyLanguageError from hy.reader import read_many @@ -17,8 +18,10 @@ def _ast_spotcheck(arg, root, secondary): assert getattr(root, arg) == getattr(secondary, arg) -def can_compile(expr, import_stdlib=False): - return hy_compile(read_many(expr), __name__, import_stdlib=import_stdlib) +def can_compile(expr, import_stdlib=False, iff=True): + return (hy_compile(read_many(expr), __name__, import_stdlib=import_stdlib) + if iff + else cant_compile(expr)) def can_eval(expr): @@ -128,6 +131,8 @@ def test_ast_good_try(): can_compile("(try 1 (except [x]) (except [y]) (finally 1))") can_compile("(try 1 (except []) (else 1) (finally 1))") can_compile("(try 1 (except [x]) (except [y]) (else 1) (finally 1))") + can_compile(iff = PY3_11, expr = "(try 1 (except* [x]))") + can_compile(iff = PY3_11, expr = "(try 1 (except* [x]) (else 1) (finally 1))") def test_ast_bad_try(): @@ -142,6 +147,8 @@ def test_ast_bad_try(): cant_compile("(try 1 (else 1) (except []))") cant_compile("(try 1 (finally 1) (except []))") cant_compile("(try 1 (except []) (finally 1) (else 1))") + cant_compile("(try 1 (except* [x]) (except [x]))") + cant_compile("(try 1 (except [x]) (except* [x]))") def test_ast_good_except(): diff --git a/tests/native_tests/py3_11_only_tests.hy b/tests/native_tests/py3_11_only_tests.hy new file mode 100644 index 000000000..f847b5574 --- /dev/null +++ b/tests/native_tests/py3_11_only_tests.hy @@ -0,0 +1,23 @@ +(defn test-except* [] + (setv got "") + + (setv return-value (try + (raise (ExceptionGroup "meep" [(KeyError) (ValueError)])) + (except* [KeyError] + (+= got "k") + "r1") + (except* [IndexError] + (+= got "i") + "r2") + (except* [ValueError] + (+= got "v") + "r3") + (else + (+= got "e") + "r4") + (finally + (+= got "f") + "r5"))) + + (assert (= got "kvf")) + (assert (= return-value "r3"))) From a3c5df4e40a6852623e111b9bd45da855d745f23 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 5 Sep 2022 16:32:47 -0400 Subject: [PATCH 014/342] Rewrite documentation of `try` --- docs/api.rst | 78 +++++++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index ecfabd77f..aa02a8601 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1167,40 +1167,50 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (try [#* body]) - The ``try`` form is used to catch exceptions (``except``) and run cleanup - actions (``finally``). - - :strong:`Examples` - - :: - - (try - (error-prone-function) - (another-error-prone-function) - (except [ZeroDivisionError] - (print "Division by zero")) - (except [[IndexError KeyboardInterrupt]] - (print "Index error or Ctrl-C")) - (except [e ValueError] - (print "ValueError:" (repr e))) - (except [e [TabError PermissionError ReferenceError]] - (print "Some sort of error:" (repr e))) - (else - (print "No errors")) - (finally - (print "All done"))) - - The first argument of ``try`` is its body, which can contain one or more forms. - Then comes any number of ``except`` clauses, then optionally an ``else`` - clause, then optionally a ``finally`` clause. If an exception is raised with a - matching ``except`` clause during the execution of the body, that ``except`` - clause will be executed. If no exceptions are raised, the ``else`` clause is - executed. The ``finally`` clause will be executed last regardless of whether an - exception was raised. - - The return value of ``try`` is the last form of the ``except`` clause that was - run, or the last form of ``else`` if no exception was raised, or the ``try`` - body if there is no ``else`` clause. + ``try`` compiles to a :py:keyword:`try` statement, which can catch + exceptions and run cleanup actions. It begins with any number of body forms. + Then follows any number of ``except`` or ``except*`` (:pep:`572`) forms, + which are expressions that begin with the symbol in question, followed by a + list of exception types, followed by more body forms. Finally there are an + optional ``else`` form and an optional ``finally`` form, which again are + expressions that begin with the symbol in question and then comprise body + forms. As in Python, at least one of ``except``, ``except*``, or ``finally`` + is required; ``else`` is only allowed if at least one ``except`` or + ``except*`` is provided; ``except*`` requires Python 3.11; and ``except`` + and ``except*`` may not both be used in the same ``try``. + + Here's an example of several of the allowed kinds of child forms:: + + (try + (error-prone-function) + (another-error-prone-function) + (except [ZeroDivisionError] + (print "Division by zero")) + (except [[IndexError KeyboardInterrupt]] + (print "Index error or Ctrl-C")) + (except [e ValueError] + (print "ValueError:" (repr e))) + (except [e [TabError PermissionError ReferenceError]] + (print "Some sort of error:" (repr e))) + (else + (print "No errors")) + (finally + (print "All done"))) + + Exception lists can be in any of several formats: + + - ``[]`` to catch any subtype of ``Exception``, like Python's ``except:`` + - ``[ETYPE]`` to catch only the single type ``ETYPE``, like Python's + ```except ETYPE:`` + - ``[[ETYPE1 ETYPE2 …]]`` to catch any of the named types, like Python's + ``except ETYPE1, ETYPE2, …:`` + - ``[VAR ETYPE]`` to catch ``ETYPE`` and bind it to ``VAR``, like Python's + ``except ETYPE as VAR:`` + - ``[VAR [ETYPE1 ETYPE2 …]]`` to catch any of the named types and bind it to + ``VAR``, like Python's ``except ETYPE1, ETYPE2, … as VAR:`` + + The return value of ``try`` is the last form evaluated among the main body, + ``except`` forms, ``except*`` forms, and ``else``. .. hy:function:: (unpack-iterable) .. hy:function:: (unpack-mapping) From 145ed1ce2c99ae70be4f3beaa84579185fa57a17 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 5 Sep 2022 16:31:16 -0400 Subject: [PATCH 015/342] Update NEWS --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index 0d2a2597d..73db09390 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -12,6 +12,7 @@ Other Breaking Changes New Features ------------------------------ * Python 3.11 is now supported. +* `except*` (PEP 654) is now recognized in `try`. Bug Fixes ------------------------------ From a9992b4fbc1cd152e2187a540bc1977b6f4502c1 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 9 Sep 2022 10:17:16 -0400 Subject: [PATCH 016/342] Autoformat --- hy/core/result_macros.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index d0dc3a9e7..c3871139a 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1301,11 +1301,12 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo # [] except_sym, exceptions, ebody = catcher if not PY3_11 and except_sym == Symbol("except*"): - hy_compiler._syntax_error(except_sym, - "`{}` requires Python 3.11 or later") + hy_compiler._syntax_error(except_sym, "`{}` requires Python 3.11 or later") except_syms_seen.add(str(except_sym)) if len(except_syms_seen) > 1: - raise compiler._syntax_error(except_sym, "cannot have both `except` and `except*` on the same `try`") + raise compiler._syntax_error( + except_sym, "cannot have both `except` and `except*` on the same `try`" + ) name = None if len(exceptions) == 2: @@ -1333,7 +1334,10 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo ebody += ebody.expr_as_stmt() handler_results += types + asty.ExceptHandler( - catcher, type=types.expr, name=name, body=ebody.stmts or [asty.Pass(catcher)] + catcher, + type=types.expr, + name=name, + body=ebody.stmts or [asty.Pass(catcher)], ) handlers.append(handler_results.stmts.pop()) From 849865c363aea78e964e71d3792b76ce1bfe9f23 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 18 Sep 2022 10:25:01 -0400 Subject: [PATCH 017/342] Document strategies for Python libraries --- docs/interop.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/interop.rst b/docs/interop.rst index 4df99f4bd..3ef2ab114 100644 --- a/docs/interop.rst +++ b/docs/interop.rst @@ -127,3 +127,19 @@ Then, use the ``hy.eval`` function to evaluate it: >>> hy.eval(expr) 38.0 + + +Libraries that expect Python +============================ + +There are various means by which Hy may interact poorly with a Python library +because the library doesn't account for the possibility of Hy. For example, +when you run the command-line program ``hy``, ``sys.executable`` will be set to +this program rather than the original Python binary. This is helpful more often +than not, but will lead to trouble if e.g. the library tries to call +:py:data:`sys.executable` with the ``-c`` option. In this case, you can try +setting :py:data:`sys.executable` back to ``hy.sys-executable``, which is a +saved copy of the original value. More generally, you can use ``hy2py``, or you +can put a simple Python wrapper script like ``import hy, my_hy_program`` in +front of your code; importing ``hy`` first is necessary here to install the +hooks that allow Python to load your Hy module. From c19f17a50fe8a6d0584cf63446b3a6475a2d2ee2 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 24 Sep 2022 10:05:45 -0400 Subject: [PATCH 018/342] Fix a PEP reference in the manual --- docs/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index aa02a8601..a76dcb966 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1169,7 +1169,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. ``try`` compiles to a :py:keyword:`try` statement, which can catch exceptions and run cleanup actions. It begins with any number of body forms. - Then follows any number of ``except`` or ``except*`` (:pep:`572`) forms, + Then follows any number of ``except`` or ``except*`` (:pep:`654`) forms, which are expressions that begin with the symbol in question, followed by a list of exception types, followed by more body forms. Finally there are an optional ``else`` form and an optional ``finally`` form, which again are From 296dd67edcbc9877723a6d16a1661e172c08c5a1 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 30 Oct 2022 11:24:37 -0400 Subject: [PATCH 019/342] On GitHub, test the full release of Python 3.11 --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 842a97ac5..0277c392b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,13 +10,13 @@ jobs: matrix: name-prefix: [''] os: [ubuntu-latest] - python: [3.7, 3.8, 3.9, '3.10', '3.11-dev', pypy-3.9] + python: [3.7, 3.8, 3.9, '3.10', 3.11, pypy-3.9] include: # To keep the overall number of runs low, we test Windows # only on the latest CPython. - name-prefix: 'win-' os: windows-latest - python: '3.11-dev' + python: 3.11 name: ${{ format('{0}{1}', matrix.name-prefix, matrix.python) }} runs-on: ${{ matrix.os }} From f5afc519051e86876dbf1688196a8f12ba6837db Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 30 Oct 2022 10:58:59 -0400 Subject: [PATCH 020/342] Simplify the syntax of `hy_repr_str_tests.txt` --- tests/native_tests/hy_repr.hy | 6 +- tests/resources/hy_repr_str_tests.txt | 132 +++++++++++++------------- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/tests/native_tests/hy_repr.hy b/tests/native_tests/hy_repr.hy index 0595d27ae..8f6dc98e8 100644 --- a/tests/native_tests/hy_repr.hy +++ b/tests/native_tests/hy_repr.hy @@ -13,9 +13,9 @@ (list o)) :setv x (.rstrip x) :if (and x (not (.startswith x ";"))) - x (if (.startswith x "!") - [(cut x 1 None) (+ "'" (cut x 1 None))] - [x]) + x (if (in (get x 0) "':") + [x] + [x (+ "'" x)]) x)] (setv rep (hy.repr (hy.eval (hy.read original-str)))) diff --git a/tests/resources/hy_repr_str_tests.txt b/tests/resources/hy_repr_str_tests.txt index 730849526..92105993d 100644 --- a/tests/resources/hy_repr_str_tests.txt +++ b/tests/resources/hy_repr_str_tests.txt @@ -1,29 +1,29 @@ ;; One test case per line. +;; If the case doesn't begin with a single quote or a colon, it also +;; gets a variant case that does. ;; Lines starting with `;` are comments. -;; Lines starting with `!` specify two tests: one with a leading -;; single quote and one without. ;; * Numeric -!None -!False -!True +None +False +True -!5 -!5.1 +5 +5.1 -!Inf -!-Inf -!NaN +Inf +-Inf +NaN -!5j -!5.1j -!2+1j -!1.2+3.4j -!Inf-Infj -!NaN+NaNj +5j +5.1j +2+1j +1.2+3.4j +Inf-Infj +NaN+NaNj -!(Fraction 1 3) +(Fraction 1 3) ;; * Symbols and keywords @@ -35,19 +35,19 @@ ;; * Stringy thingies -!"" -!b"" -!(bytearray b"") +"" +b"" +(bytearray b"") -!"apple bloom" -!b"apple bloom" -!(bytearray b"apple bloom") -!"⚘" +"apple bloom" +b"apple bloom" +(bytearray b"apple bloom") +"⚘" -!"single ' quotes" -!b"single ' quotes" -!"\"double \" quotes\"" -!b"\"double \" quotes\"" +"single ' quotes" +b"single ' quotes" +"\"double \" quotes\"" +b"\"double \" quotes\"" '#[[bracketed string]] '#[delim[bracketed string]delim] @@ -55,38 +55,38 @@ ;; * Collections -![] -!#() -!#{} -!(frozenset #{}) -!{} +[] +#() +#{} +(frozenset #{}) +{} -!['[]] -![1 2 3] -!#(1 2 3) -!#{1 2 3} +['[]] +[1 2 3] +#(1 2 3) +#{1 2 3} '#{3 2 1 2} -!(frozenset #{1 2 3}) -!{"a" 1 "b" 2} -!{"b" 2 "a" 1} +(frozenset #{1 2 3}) +{"a" 1 "b" 2} +{"b" 2 "a" 1} '[1 a 3] -![1 'a 3] -![1 '[2 3] 4] +[1 'a 3] +[1 '[2 3] 4] -!(deque []) -!(deque [1 2.5 None "hello"]) -!(ChainMap {}) -!(ChainMap {1 2} {3 4}) -!(OrderedDict []) -!(OrderedDict [#(1 2) #(3 4)]) +(deque []) +(deque [1 2.5 None "hello"]) +(ChainMap {}) +(ChainMap {1 2} {3 4}) +(OrderedDict []) +(OrderedDict [#(1 2) #(3 4)]) ;; * Expressions '(+ 1 2) '(f a b) '(f #* args #** kwargs) -![1 [2 3] #(4 #('mysymbol :mykeyword)) {"a" b"hello"} '(f #* a #** b)] +[1 [2 3] #(4 #('mysymbol :mykeyword)) {"a" b"hello"} '(f #* a #** b)] ;; * Quasiquoting @@ -107,23 +107,23 @@ ;; * Ranges and slices -!(range 5) -!(slice 5) -!(range 2 5) -!(slice 2 5) -!(range 5 2) -!(slice 5 2) -!(range 0 5 2) -!(slice 0 5 2) -!(range 0 5 -2) -!(slice 0 5 -2) -!(slice [1 2 3] #(4 6) "hello") +(range 5) +(slice 5) +(range 2 5) +(slice 2 5) +(range 5 2) +(slice 5 2) +(range 0 5 2) +(slice 0 5 2) +(range 0 5 -2) +(slice 0 5 -2) +(slice [1 2 3] #(4 6) "hello") ;; * Regexen -!(re.compile "foo") -!(re.compile "\\Ax\\Z") -!(re.compile "'") -!(re.compile "\"") -!(re.compile "foo" re.IGNORECASE) -!(re.compile "foo" (| re.IGNORECASE re.MULTILINE)) +(re.compile "foo") +(re.compile "\\Ax\\Z") +(re.compile "'") +(re.compile "\"") +(re.compile "foo" re.IGNORECASE) +(re.compile "foo" (| re.IGNORECASE re.MULTILINE)) From 8fbe0664af8c4e644366726f75e299b1728df295 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 30 Oct 2022 11:02:51 -0400 Subject: [PATCH 021/342] Test `hy.repr` on quoted keywords --- tests/native_tests/hy_repr.hy | 1 + tests/resources/hy_repr_str_tests.txt | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tests/native_tests/hy_repr.hy b/tests/native_tests/hy_repr.hy index 8f6dc98e8..276a056aa 100644 --- a/tests/native_tests/hy_repr.hy +++ b/tests/native_tests/hy_repr.hy @@ -28,6 +28,7 @@ ; hy-reprs from the input syntax. (setv values [ + ':mykeyword {"a" 1 "b" 2 "a" 3} '{"a" 1 "b" 2 "a" 3} 'f"the answer is {(+ 2 2) = }" diff --git a/tests/resources/hy_repr_str_tests.txt b/tests/resources/hy_repr_str_tests.txt index 92105993d..aa6c09abf 100644 --- a/tests/resources/hy_repr_str_tests.txt +++ b/tests/resources/hy_repr_str_tests.txt @@ -32,6 +32,10 @@ NaN+NaNj :mykeyword :my♥keyword? : +'':mykeyword +; A keyword with only one single quote gets represented without the +; quote (which is equivalent), so it's tested in +; `test-hy-repr-roundtrip-from-value`. ;; * Stringy thingies From b31e10628d4af8b26a670c8d57f2b2d9ef93a2c5 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 30 Oct 2022 11:04:18 -0400 Subject: [PATCH 022/342] Reword a test comment --- tests/native_tests/hy_repr.hy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/native_tests/hy_repr.hy b/tests/native_tests/hy_repr.hy index 276a056aa..f113bcf98 100644 --- a/tests/native_tests/hy_repr.hy +++ b/tests/native_tests/hy_repr.hy @@ -41,8 +41,8 @@ (defn test-hy-repr-no-roundtrip [] ; Test one of the corner cases in which hy-repr doesn't - ; round-trip: when a Hy Object contains a non-Hy Object, we - ; promote the constituent to a Hy Object. + ; round-trip: when a Hy model contains a non-model, we + ; promote the constituent to a model. (setv orig `[a ~5.0]) (setv reprd (hy.repr orig)) From 6145d92e19663495c69df6f8011dbd04adcde12a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 30 Oct 2022 11:22:58 -0400 Subject: [PATCH 023/342] Improve the docstring of `hy.repr` --- hy/core/hy_repr.hy | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/hy/core/hy_repr.hy b/hy/core/hy_repr.hy index 1ac05e92d..51fd88d1d 100644 --- a/hy/core/hy_repr.hy +++ b/hy/core/hy_repr.hy @@ -47,22 +47,29 @@ (setv _quoting False) (setv _seen (set)) (defn hy-repr [obj] - "This function is Hy's equivalent of Python's built-in ``repr``. - It returns a string representing the input object in Hy syntax. + #[[This function is Hy's equivalent of Python's :func:`repr`. + It returns a string representing the input object in Hy syntax. :: + + => (hy.repr [1 2 3]) + "[1 2 3]" + => (repr [1 2 3]) + "[1, 2, 3]" Like ``repr`` in Python, ``hy.repr`` can round-trip many kinds of values. Round-tripping implies that given an object ``x``, - ``(hy.eval (hy.read-str (hy.repr x)))`` returns ``x``, or at least a value - that's equal to ``x``. + ``(hy.eval (hy.read (hy.repr x)))`` returns ``x``, or at least a + value that's equal to ``x``. A notable exception to round-tripping + is that if a :class:`hy.models.Object` contains a non-model, the + latter will be promoted to a model in the output:: - Examples: - :: + (setv + x (hy.models.List [5]) + output (hy.repr x) + y (hy.eval (hy.read output))) + (print output) ; '[5] + (print (type (get x 0))) ; + (print (type (get y 0))) ; ]] - => (hy.repr [1 2 3]) - \"[1 2 3]\" - => (repr [1 2 3]) - \"[1, 2, 3]\" - " (setv [f placeholder] (.get _registry (type obj) [_base-repr None])) (global _quoting) From fa57b11b00a21047dc06ce02564db80904c82a58 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 30 Oct 2022 11:30:24 -0400 Subject: [PATCH 024/342] Remove `versionadded` lines These give the impression that parts of the interface without a `versionadded` line have been around longer, which is far from true. Documentation of how Hy has changed over time should be limited to NEWS, at least until 1.0. --- docs/api.rst | 8 -------- docs/cli.rst | 5 ----- hy/core/util.hy | 8 -------- 3 files changed, 21 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index a76dcb966..e02c5934b 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -69,8 +69,6 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:data:: . - .. versionadded:: 0.10.0 - ``.`` is used to perform attribute access on objects. It uses a small DSL to allow quick access to attributes and items in a nested data structure. @@ -822,8 +820,6 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (del [object]) - .. versionadded:: 0.9.12 - ``del`` removes an object from the current namespace. :strong:`Examples` @@ -856,8 +852,6 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (nonlocal [object]) - .. versionadded:: 0.11.1 - ``nonlocal`` can be used to mark a symbol as not local to the current scope. The parameters are the names of symbols to mark as nonlocal. This is necessary to modify variables through nested ``fn`` scopes: @@ -1427,8 +1421,6 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (yield-from [object]) - .. versionadded:: 0.9.13 - ``yield-from`` is used to call a subgenerator. This is useful if you want your coroutine to be able to delegate its processes to another coroutine, say, if using something fancy like diff --git a/docs/cli.rst b/docs/cli.rst index e1e34cfb3..e34393968 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -31,8 +31,6 @@ Command Line Options all arguments after the *module* name are passed to the module in ``sys.argv``. - .. versionadded:: 0.11.0 - .. cmdoption:: --spy Print equivalent Python code before executing in REPL. For example:: @@ -46,14 +44,11 @@ Command Line Options => `--spy` only works on REPL mode. - .. versionadded:: 0.9.11 .. cmdoption:: --repl-output-fn Format REPL output using specific function (e.g., ``repr``) - .. versionadded:: 0.13.0 - .. cmdoption:: -v Print the Hy version number and exit. diff --git a/hy/core/util.hy b/hy/core/util.hy index 55d90061a..27aba123d 100644 --- a/hy/core/util.hy +++ b/hy/core/util.hy @@ -9,8 +9,6 @@ If the second argument `codegen` is true, generate python code instead. - .. versionadded:: 0.10.0 - Dump the Python AST for given Hy *tree* to standard output. If *codegen* is ``True``, the function prints Python code instead. @@ -46,8 +44,6 @@ generated symbol, as an aid to debugging. Typically one calls ``hy.gensym`` without an argument. - .. versionadded:: 0.9.12 - .. seealso:: Section :ref:`using-gensym` @@ -89,8 +85,6 @@ (defn macroexpand [form [result-ok False]] "Return the full macro expansion of `form`. - .. versionadded:: 0.10.0 - Examples: :: @@ -107,8 +101,6 @@ (defn macroexpand-1 [form] "Return the single step macro expansion of `form`. - .. versionadded:: 0.10.0 - Examples: :: From dcf6f15a59fc04c439749e79a2fb8281d9370261 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 30 Oct 2022 10:39:50 -0400 Subject: [PATCH 025/342] Update examples in `model_patterns.rst` --- docs/model_patterns.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/model_patterns.rst b/docs/model_patterns.rst index 44272bfa5..7759f7328 100644 --- a/docs/model_patterns.rst +++ b/docs/model_patterns.rst @@ -35,8 +35,9 @@ a pattern for a ``try`` form of the above kind: .. code-block:: clj - (import funcparserlib.parser [maybe many]) - (import hy.model-patterns [*]) + (import + funcparserlib.parser [maybe many] + hy.model-patterns *) (setv parser (whole [ (sym "try") (many (notpexpr "except" "else" "finally")) @@ -98,15 +99,16 @@ Here's how you could write a simple macro using model patterns: .. code-block:: clj (defmacro pairs [#* args] - (import funcparserlib.parser [many]) - (import hy.model-patterns [whole SYM FORM]) + (import + funcparserlib.parser [many] + hy.model-patterns [whole SYM FORM]) (setv [args] (.parse (whole [(many (+ SYM FORM))]) args)) `[~@(gfor [a1 a2] args #((str a1) a2))]) - (print (pairs a 1 b 2 c 3)) - ; => [["a" 1] ["b" 2] ["c" 3]] + (print (hy.repr (pairs a 1 b 2 c 3))) + ; => [#("a" 1) #("b" 2) #("c" 3)] A failed parse will raise ``funcparserlib.parser.NoParseError``. From 8260676a90bb01713a869e503d7fbef437099e3d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 30 Oct 2022 11:52:23 -0400 Subject: [PATCH 026/342] Improve the documentation of annotation --- docs/api.rst | 65 ++++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 38 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index e02c5934b..c19cc5da0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,62 +8,51 @@ Core Macros The following macros are auto imported into all Hy modules as their base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. -.. hy:data:: #^ +.. hy:function:: (annotate [value type]) - The ``#^`` symbol is used to denote annotations in three different contexts: + ``annotate`` and its shorthand form ``#^`` are used to denote annotations, + including type hints, in three different contexts: - - Standalone variable annotations. - - Variable annotations in a setv call. - - Function argument annotations. + - Standalone variable annotations (:pep:`526`) + - Variable annotations in a :hy:func:`setv` call + - Function-parameter annotations (:pep:`3107`) - They implement `PEP 526 `_ and - `PEP 3107 `_. + The difference between ``annotate`` and ``#^`` is that ``annotate`` requires + parentheses and takes the name to be annotated first (like Python), whereas + ``#^`` doesn't require parentheses (it only applies to the next two forms) + and takes the type second:: - Syntax sugar for :hy:func:`annotate` where the type comes first. + (setv (annotate x int) 1) + (setv #^int x 1) - Here is some example syntax of all three usages: + The order difference is not merely visual: ``#^`` actually evaluates the + type first. - :strong:`Examples` + Here are examples with ``#^`` for all the places you can use annotations:: - :: - - ; Annotate the variable x as an int (equivalent to `x: int`). + ; Annotate the variable `x` as an `int` (equivalent to `x: int`). #^int x - ; Can annotate with expressions if needed (equivalent to `y: f(x)`). + ; You can annotate with expressions (equivalent to `y: f(x)`). #^(f x) y - ; Annotations with an assignment: each annotation (int, str) covers the term that - ; immediately follows. - ; Equivalent to: x: int = 1; y = 2; z: str = 3 - (setv #^int x 1 y 2 #^str z 3) + ; Annotations with an assignment: each annotation `(int, str)` + ; covers the term that immediately follows. + ; Equivalent to `x: int = 1; y = 2; z: str = 3` + (setv #^int x 1 y 2 #^str z 3) - ; Annotate a as an int, c as an int, and b as a str. - ; Equivalent to: def func(a: int, b: str = None, c: int = 1): ... - (defn func [#^int a #^str [b None] #^int [c 1]] ...) + ; Annotate `a` as an `int`, `c` as an `int`, and `b` as a `str`. + ; Equivalent to `def func(a: int, b: str = None, c: int = 1): ...` + (defn func [#^int a #^str [b None] #^int [c 1]] ...) - ; Function return annotations come before the function name (if it exists) + ; Function return annotations come before the function name (if + ; it exists). (defn #^int add1 [#^int x] (+ x 1)) (fn #^int [#^int y] (+ y 2)) - The rules are: - - - The type to annotate with is the form that immediately follows the caret. - - The annotation always comes (and is evaluated) *before* the value being annotated. This is - unlike Python, where it comes and is evaluated *after* the value being annotated. - For annotating items with generic types, the :hy:func:`of ` macro will likely be of use. -.. hy:function:: (annotate [value type]) - - Expanded form of :hy:data:`#^`. Syntactically equal to ``#^`` and usable wherever - you might use ``#^``:: - - (setv (annotate x int) 1) - (setv #^int x 1) ; the type comes first when using #^int - - (defn (annotate add1 int) [(annotate x int)] (+ x 1)) - + An issue with type annotations is that, as of this writing, we know of no Python type-checker that can work with :py:mod:`ast` objects or bytecode files. They all need Python source text. So you'll have to translate your Hy with ``hy2py`` in order to actually check the types. .. _dot: From d1a125239140ac856a35e7c0efd5efc5bc234fbe Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 27 Oct 2022 15:59:31 -0400 Subject: [PATCH 027/342] Remove exception to the merge policy for releases --- CONTRIBUTING.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b26b6d5ec..2a1c10d78 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -120,9 +120,7 @@ There are two situations in which a PR is allowed to be merged: author. Changes to the documentation, or trivial changes to code, need only **one** approving member. 2. When the PR is at least **three days** old and **no** member of the Hy core - team has expressed disapproval of the PR in its current state. (Exception: a - PR to create a new release is not eligible to be merged under this criterion, - only the first one.) + team has expressed disapproval of the PR in its current state. Anybody on the Hy core team may perform the merge. Merging should create a merge commit (don't squash unnecessarily, because that would remove separation between From 075c3eefa0eea52bf251c5b67f3ce3784299b6a8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 5 Nov 2022 14:41:17 -0400 Subject: [PATCH 028/342] Add a placeholder macro for `except*` --- docs/api.rst | 1 + hy/core/result_macros.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index c19cc5da0..942770ad9 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1428,6 +1428,7 @@ expanded, is crash, regardless of their arguments: - ``else`` - ``except`` +- ``except*`` - ``finally`` - ``unpack-mapping`` - ``unquote`` diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index c3871139a..8d5390e45 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1895,7 +1895,8 @@ def compile_let(compiler, expr, root, bindings, body): @pattern_macro( - "unquote unquote-splice unpack-mapping except finally else".split(), [many(FORM)] + "unquote unquote-splice unpack-mapping except except* finally else".split(), + [many(FORM)], ) def compile_placeholder(compiler, expr, root, body): raise ValueError(f"`{root}` is not allowed here") From 9a09916df0900dceb87bc2d61a1163bfb0283c09 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 5 Nov 2022 14:42:47 -0400 Subject: [PATCH 029/342] Extend tests of placeholder macros --- tests/compilers/test_ast.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 7083be6b7..800c830c3 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -579,14 +579,22 @@ def test_setv_builtins(): ) -def test_top_level_unquote(): +def placeholder_macro(x, ename=None): with pytest.raises(HyLanguageError) as e: - can_compile("(unquote)") - assert "`unquote` is not allowed here" in e.value.msg + can_compile(f"({x})") + assert f"`{ename or x}` is not allowed here" in e.value.msg - with pytest.raises(HyLanguageError) as e: - can_compile("(unquote-splice)") - assert "`unquote-splice` is not allowed here" in e.value.msg + +def test_top_level_unquote(): + placeholder_macro("unquote") + placeholder_macro("unquote-splice") + placeholder_macro("unquote_splice", "unquote-splice") + + +def test_bad_exception(): + placeholder_macro("except") + placeholder_macro("except*") + placeholder_macro(hy.mangle("except*"), "except*") def test_lots_of_comment_lines(): From 4404c11a79446b79d2a8453e57a767ec251baec1 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 8 Nov 2022 12:55:32 -0500 Subject: [PATCH 030/342] Clean up NEWS for release --- NEWS.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 73db09390..5f83fd543 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,26 +1,27 @@ .. default-role:: code -Unreleased +0.25.0 (released 2022-11-08) ============================== -Other Breaking Changes +Breaking Changes ------------------------------ * `dfor` no longer requires brackets around its final arguments, so `(dfor x (range 5) [x (* 2 x)])` is now `(dfor x (range 5) x (* 2 x))`. - -New Features ------------------------------- -* Python 3.11 is now supported. -* `except*` (PEP 654) is now recognized in `try`. +* `except*` (PEP 654) is now recognized in `try`, and a placeholder + macro for `except*` has been added. Bug Fixes ------------------------------ -* Fixed `hy.repr` of `slice` objects with non-integer arguments. * `__file__` should now be set the same way as in Python. +* `\N{…}` escape sequences are now recognized in f-strings. * Fixed a bug with `python -O` where assertions were still partly evaluated. -* `\N{…}` escape sequences are now recognized in f-strings. +* Fixed `hy.repr` of `slice` objects with non-integer arguments. + +New Features +------------------------------ +* Python 3.11 is now supported. Misc. Improvements ------------------------------ From 96446069c0488a2e42411caff22ff6d4e14708b9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 6 Nov 2022 13:45:33 -0500 Subject: [PATCH 031/342] Simplify `_could_be_hy_src` --- hy/importer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hy/importer.py b/hy/importer.py index 9579f168e..cf3bf1a25 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -107,10 +107,8 @@ def _get_code_from_file(run_name, fname=None, hy_src_check=lambda x: x.endswith( def _could_be_hy_src(filename): return os.path.isfile(filename) and ( - filename.endswith(".hy") - or not any( - filename.endswith(ext) for ext in importlib.machinery.SOURCE_SUFFIXES[1:] - ) + os.path.splitext(filename)[1] not in + set(importlib.machinery.SOURCE_SUFFIXES) - {".hy"} ) From 9cbbd5fad77ae5aa58e576ad25bac516c773d835 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 6 Nov 2022 14:03:15 -0500 Subject: [PATCH 032/342] Implement import from ZIP archives --- hy/importer.py | 17 +++++++++++++++++ tests/importer/test_importer.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/hy/importer.py b/hy/importer.py index cf3bf1a25..393d2d5d4 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -5,12 +5,14 @@ import pkgutil import sys import types +import zipimport from contextlib import contextmanager from functools import partial import hy from hy.compiler import hy_compile from hy.reader import read_many +from hy._compat import PY3_8 @contextmanager @@ -126,6 +128,21 @@ def _hy_source_to_code(self, data, path, _optimize=-1): importlib.machinery.SourceFileLoader.source_to_code = _hy_source_to_code + +if PY3_8 and ('.hy', False, False) not in zipimport._zip_searchorder: + zipimport._zip_searchorder += (('.hy', False, False),) + _py_compile_source = zipimport._compile_source + def _hy_compile_source(pathname, source): + if not pathname.endswith('.hy'): + return _py_compile_source(pathname, source) + return compile( + hy_compile( + read_many(source.decode('UTF-8'), filename=pathname, skip_shebang=True), + f''), + pathname, 'exec', dont_inherit=True) + zipimport._compile_source = _hy_compile_source + + # This is actually needed; otherwise, pre-created finders assigned to the # current dir (i.e. `''`) in `sys.path` will not catch absolute imports of # directory-local modules! diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index 3dcd1e08e..37bab90d6 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -13,6 +13,7 @@ from hy.importer import HyLoader from hy.reader import read_many from hy.reader.exceptions import PrematureEndOfInput +from hy._compat import PY3_8 def test_basics(): @@ -273,3 +274,20 @@ def test_filtered_importlib_frames(capsys): captured_w_filtering = capsys.readouterr()[-1].strip() assert "importlib._" not in captured_w_filtering + + +@pytest.mark.skipif(not PY3_8, reason = "Python 3.7's `zipimport` is written in C, it can't be monkey-patched") +def test_zipimport(tmp_path): + from zipfile import ZipFile + + zpath = tmp_path / 'archive.zip' + with ZipFile(zpath, 'w') as o: + o.writestr('example.hy', '(setv x "Hy from ZIP")') + + try: + sys.path.insert(0, str(zpath)) + import example + finally: + sys.path = [p for p in sys.path if p != str(zpath)] + assert example.x == "Hy from ZIP" + assert example.__file__ == str(zpath / 'example.hy') From 234f8f5a1f2b278b6db21fe06a98007c72d9f722 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 10 Nov 2022 10:00:05 -0500 Subject: [PATCH 033/342] Update NEWS --- NEWS.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 5f83fd543..d91d57ab7 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,5 +1,15 @@ .. default-role:: code +Unreleased +============================= + +New Features +------------------------------ +* On Pythons ≥ 3.7, Hy modules can now be imported from ZIP + archives in the same way as Python modules, via `zipimport`_. + +.. _zipimport: https://docs.python.org/3.11/library/zipimport.html + 0.25.0 (released 2022-11-08) ============================== From d50bf77b29ae8301a85dd8ad6f87f032bc6ff507 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 10 Nov 2022 10:05:15 -0500 Subject: [PATCH 034/342] Autoformat --- hy/importer.py | 24 +++++++++++++++--------- tests/importer/test_importer.py | 15 +++++++++------ 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/hy/importer.py b/hy/importer.py index 393d2d5d4..7f9a4d7c6 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -10,9 +10,9 @@ from functools import partial import hy +from hy._compat import PY3_8 from hy.compiler import hy_compile from hy.reader import read_many -from hy._compat import PY3_8 @contextmanager @@ -109,8 +109,8 @@ def _get_code_from_file(run_name, fname=None, hy_src_check=lambda x: x.endswith( def _could_be_hy_src(filename): return os.path.isfile(filename) and ( - os.path.splitext(filename)[1] not in - set(importlib.machinery.SOURCE_SUFFIXES) - {".hy"} + os.path.splitext(filename)[1] + not in set(importlib.machinery.SOURCE_SUFFIXES) - {".hy"} ) @@ -129,17 +129,23 @@ def _hy_source_to_code(self, data, path, _optimize=-1): importlib.machinery.SourceFileLoader.source_to_code = _hy_source_to_code -if PY3_8 and ('.hy', False, False) not in zipimport._zip_searchorder: - zipimport._zip_searchorder += (('.hy', False, False),) +if PY3_8 and (".hy", False, False) not in zipimport._zip_searchorder: + zipimport._zip_searchorder += ((".hy", False, False),) _py_compile_source = zipimport._compile_source + def _hy_compile_source(pathname, source): - if not pathname.endswith('.hy'): + if not pathname.endswith(".hy"): return _py_compile_source(pathname, source) return compile( hy_compile( - read_many(source.decode('UTF-8'), filename=pathname, skip_shebang=True), - f''), - pathname, 'exec', dont_inherit=True) + read_many(source.decode("UTF-8"), filename=pathname, skip_shebang=True), + f"", + ), + pathname, + "exec", + dont_inherit=True, + ) + zipimport._compile_source = _hy_compile_source diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index 37bab90d6..382a8d5fa 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -8,12 +8,12 @@ import pytest import hy +from hy._compat import PY3_8 from hy.compiler import hy_compile, hy_eval from hy.errors import HyLanguageError, hy_exc_handler from hy.importer import HyLoader from hy.reader import read_many from hy.reader.exceptions import PrematureEndOfInput -from hy._compat import PY3_8 def test_basics(): @@ -276,13 +276,16 @@ def test_filtered_importlib_frames(capsys): assert "importlib._" not in captured_w_filtering -@pytest.mark.skipif(not PY3_8, reason = "Python 3.7's `zipimport` is written in C, it can't be monkey-patched") +@pytest.mark.skipif( + not PY3_8, + reason="Python 3.7's `zipimport` is written in C, it can't be monkey-patched", +) def test_zipimport(tmp_path): from zipfile import ZipFile - zpath = tmp_path / 'archive.zip' - with ZipFile(zpath, 'w') as o: - o.writestr('example.hy', '(setv x "Hy from ZIP")') + zpath = tmp_path / "archive.zip" + with ZipFile(zpath, "w") as o: + o.writestr("example.hy", '(setv x "Hy from ZIP")') try: sys.path.insert(0, str(zpath)) @@ -290,4 +293,4 @@ def test_zipimport(tmp_path): finally: sys.path = [p for p in sys.path if p != str(zpath)] assert example.x == "Hy from ZIP" - assert example.__file__ == str(zpath / 'example.hy') + assert example.__file__ == str(zpath / "example.hy") From 1a619815ac7ac5195d0c7b639e858d818851eeb8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 12 Nov 2022 09:33:26 -0500 Subject: [PATCH 035/342] Restore `sys.ps1` etc. after exiting the REPL --- NEWS.rst | 5 ++++ hy/cmdline.py | 57 +++++++++++++++++++++++--------------- tests/native_tests/repl.hy | 14 ++++++++++ 3 files changed, 54 insertions(+), 22 deletions(-) create mode 100644 tests/native_tests/repl.hy diff --git a/NEWS.rst b/NEWS.rst index d91d57ab7..f369b20ab 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,6 +3,11 @@ Unreleased ============================= +Bug Fixes +------------------------------ +* `HyREPL` now restores the global values it changes (such as + `sys.ps1`) after `HyREPL.run` terminates. + New Features ------------------------------ * On Pythons ≥ 3.7, Hy modules can now be imported from ZIP diff --git a/hy/cmdline.py b/hy/cmdline.py index e1e634986..3a7d84164 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -408,29 +408,42 @@ def run(self): import colorama - sys.ps1 = "=> " - sys.ps2 = "... " - - builtins.quit = HyQuitter("quit") - builtins.exit = HyQuitter("exit") - builtins.help = HyHelper() - - colorama.init() - - namespace = self.locals - with filtered_hy_exceptions(), extend_linecache(self.cmdline_cache), completion( - Completer(namespace) - ): - self.interact( - "Hy {version} using " - "{py}({build}) {pyversion} on {os}".format( - version=hy.__version__, - py=platform.python_implementation(), - build=platform.python_build()[0], - pyversion=platform.python_version(), - os=platform.system(), + sentinel = [] + saved_values = ( + getattr(sys, 'ps1', sentinel), + getattr(sys, 'ps2', sentinel), + builtins.quit, + builtins.exit, + builtins.help) + try: + sys.ps1 = "=> " + sys.ps2 = "... " + builtins.quit = HyQuitter("quit") + builtins.exit = HyQuitter("exit") + builtins.help = HyHelper() + + colorama.init() + + namespace = self.locals + with filtered_hy_exceptions(), extend_linecache(self.cmdline_cache), completion( + Completer(namespace) + ): + self.interact( + "Hy {version} using " + "{py}({build}) {pyversion} on {os}".format( + version=hy.__version__, + py=platform.python_implementation(), + build=platform.python_build()[0], + pyversion=platform.python_version(), + os=platform.system(), + ) ) - ) + + finally: + sys.ps1, sys.ps2, builtins.quit, builtins.exit, builtins.help = saved_values + for a in 'ps1', 'ps2': + if getattr(sys, a) is sentinel: + delattr(sys, a) return 0 diff --git a/tests/native_tests/repl.hy b/tests/native_tests/repl.hy new file mode 100644 index 000000000..70b899e1c --- /dev/null +++ b/tests/native_tests/repl.hy @@ -0,0 +1,14 @@ +; Many other tests of the REPL are in `test_bin.py`. + +(import + io + sys + hy.cmdline [HyREPL]) + +(defn test-preserve-ps1 [monkeypatch] + ; https://github.com/hylang/hy/issues/1323#issuecomment-1310837340 + (monkeypatch.setattr "sys.stdin" (io.StringIO "(+ 1 1)")) + (setv sys.ps1 "chippy") + (assert (= sys.ps1 "chippy")) + (.run (HyREPL)) + (assert (= sys.ps1 "chippy"))) From e7355c1a6f755640efe877d9a0ca7fa265809ca7 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 10 Nov 2022 15:36:56 -0500 Subject: [PATCH 036/342] Update a bug URL in a comment --- hy/completer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/completer.py b/hy/completer.py index ae8967652..8fa242079 100644 --- a/hy/completer.py +++ b/hy/completer.py @@ -8,7 +8,7 @@ from hy import mangle, unmangle # Lazily import `readline` to work around -# https://bugs.python.org/issue2675#msg265564 +# https://github.com/python/cpython/issues/46927#issuecomment-1093418916 readline = None From 3c30898392f82b1d41d9789c6499f45bae22a238 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 10 Nov 2022 16:10:11 -0500 Subject: [PATCH 037/342] Fix a Readline history bug I haven't written a test for this because I think it will require a lot of elaborate mocking or manipulation of temporary history files for a relatively small issue. --- NEWS.rst | 2 ++ hy/completer.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index f369b20ab..75df90652 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -7,6 +7,8 @@ Bug Fixes ------------------------------ * `HyREPL` now restores the global values it changes (such as `sys.ps1`) after `HyREPL.run` terminates. +* `HyREPL` no longer mixes up Hy's and Python's Readline histories + when run inside Python's REPL. New Features ------------------------------ diff --git a/hy/completer.py b/hy/completer.py index 8fa242079..bc1aa6b01 100644 --- a/hy/completer.py +++ b/hy/completer.py @@ -123,6 +123,14 @@ def completion(completer=None): history = os.environ.get("HY_HISTORY", os.path.expanduser("~/.hy-history")) readline.parse_and_bind("set blink-matching-paren on") + # Save and clear any existing history. + history_was = [] + for _ in range(readline.get_current_history_length()): + history_was.append(readline.get_history_item(1)) + readline.remove_history_item(0) + # Yes, the first item is numbered 1 by one method and 0 by the + # other. + try: readline.read_history_file(history) except OSError: @@ -137,3 +145,7 @@ def completion(completer=None): readline.write_history_file(history) except OSError: pass + # Restore the previously saved history. + readline.clear_history() + for item in history_was: + readline.add_history(item) From 43025519cf6a7860edf7a5be02b760dc5e53c00c Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 12 Nov 2022 08:21:03 -0500 Subject: [PATCH 038/342] Update some REPL documentation --- docs/interop.rst | 23 ++++------------------- hy/cmdline.py | 10 +++++++++- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/docs/interop.rst b/docs/interop.rst index 3ef2ab114..35f64aa04 100644 --- a/docs/interop.rst +++ b/docs/interop.rst @@ -83,30 +83,15 @@ causes the import of ``name`` from ``hy.core.language``. Launching a Hy REPL from Python ------------------------------- -You can use the function ``run_repl()`` to launch the Hy REPL from Python: +You can use :meth:`hy.cmdline.HyREPL.run` to launch the Hy REPL from Python: -.. code-block:: python - - >>> import hy.cmdline - >>> hy.cmdline.run_repl() - hy 0.12.1 using CPython(default) 3.6.0 on Linux - => (defn foo [] (print "bar")) - => (test) - bar - -If you want to print the Python code Hy generates for you, use the ``spy`` -argument: - -.. code-block:: python +.. code-block:: text >>> import hy.cmdline - >>> hy.cmdline.run_repl(spy=True) - hy 0.12.1 using CPython(default) 3.6.0 on Linux + >>> hy.cmdline.HyREPL(locals = locals()).run() + Hy x.y.z using CPython(default) x.y.z on Linux => (defn test [] (print "bar")) - def test(): - return print('bar') => (test) - test() bar Evaluating strings of Hy code from Python diff --git a/hy/cmdline.py b/hy/cmdline.py index 3a7d84164..8a832f373 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -234,7 +234,15 @@ def __call__(self, *args, **kwargs): class HyREPL(code.InteractiveConsole): - "A subclass of :class:`code.InteractiveConsole` for Hy." + """A subclass of :class:`code.InteractiveConsole` for Hy. + + A convenient way to use this class to interactively debug code is to insert the + following in the code you want to debug:: + + (.run (hy.cmdline.HyREPL :locals (locals))) + + Note that as with :func:`code.interact`, changes to ``(locals)`` inside the REPL are + not propagated back to the original scope.""" def __init__(self, spy=False, output_fn=None, locals=None, filename=""): From 4607d7b43174b025b567d3ee4dafe6c0ef93dbb6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 12 Nov 2022 08:38:08 -0500 Subject: [PATCH 039/342] Join some string literals in a test --- tests/test_bin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bin.py b/tests/test_bin.py index 1d5b5b32e..d046ce8e3 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -530,7 +530,7 @@ def test_tracebacks(): os.environ["HY_DEBUG"] = "" def req_err(x): - assert x == "hy.errors.HyRequireError: No module named " "'not_a_real_module'" + assert x == "hy.errors.HyRequireError: No module named 'not_a_real_module'" # Modeled after # > python -c 'import not_a_real_module' From f87358c7900c73906f72eb5153ac70ea98a82d5c Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 12 Nov 2022 08:51:33 -0500 Subject: [PATCH 040/342] Move the REPL code to its own file --- hy/cmdline.py | 420 --------------------------------------------- hy/repl.py | 458 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 458 insertions(+), 420 deletions(-) create mode 100644 hy/repl.py diff --git a/hy/cmdline.py b/hy/cmdline.py index 8a832f373..53f2f4739 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -35,426 +35,6 @@ from hy.reader.exceptions import PrematureEndOfInput from hy.reader.hy_reader import HyReader -sys.last_type = None -sys.last_value = None -sys.last_traceback = None - - -class HyQuitter: - def __init__(self, name): - self.name = name - - def __repr__(self): - return "Use (%s) or Ctrl-D (i.e. EOF) to exit" % (self.name) - - __str__ = __repr__ - - def __call__(self, code=None): - try: - sys.stdin.close() - except: - pass - raise SystemExit(code) - - -class HyHelper: - def __repr__(self): - return ( - "Use (help) for interactive help, or (help object) for help " - "about object." - ) - - def __call__(self, *args, **kwds): - import pydoc - - return pydoc.help(*args, **kwds) - - -@contextmanager -def extend_linecache(add_cmdline_cache): - _linecache_checkcache = linecache.checkcache - - def _cmdline_checkcache(*args): - _linecache_checkcache(*args) - linecache.cache.update(add_cmdline_cache) - - linecache.checkcache = _cmdline_checkcache - yield - linecache.checkcache = _linecache_checkcache - - -_codeop_maybe_compile = codeop._maybe_compile - - -def _hy_maybe_compile(compiler, source, filename, symbol): - """The `codeop` version of this will compile the same source multiple - times, and, since we have macros and things like `eval-and-compile`, we - can't allow that. - """ - if not isinstance(compiler, HyCompile): - return _codeop_maybe_compile(compiler, source, filename, symbol) - - for line in source.split("\n"): - line = line.strip() - if line and line[0] != ";": - # Leave it alone (could do more with Hy syntax) - break - else: - if symbol != "eval": - # Replace it with a 'pass' statement (i.e. tell the compiler to do - # nothing) - source = "pass" - - return compiler(source, filename, symbol) - - -codeop._maybe_compile = _hy_maybe_compile - - -class HyCompile(codeop.Compile): - """This compiler uses `linecache` like - `IPython.core.compilerop.CachingCompiler`. - """ - - def __init__( - self, module, locals, ast_callback=None, hy_compiler=None, cmdline_cache={} - ): - self.module = module - self.locals = locals - self.ast_callback = ast_callback - self.hy_compiler = hy_compiler - self.reader = HyReader() - - super().__init__() - - if hasattr(self.module, "__reader_macros__"): - enable_readers( - self.module, self.reader, self.module.__reader_macros__.keys() - ) - - self.flags |= hy_ast_compile_flags - - self.cmdline_cache = cmdline_cache - - def _cache(self, source, name): - entry = ( - len(source), - time.time(), - [line + "\n" for line in source.splitlines()], - name, - ) - - linecache.cache[name] = entry - self.cmdline_cache[name] = entry - - def _update_exc_info(self): - self.locals["_hy_last_type"] = sys.last_type - self.locals["_hy_last_value"] = sys.last_value - # Skip our frame. - sys.last_traceback = getattr(sys.last_traceback, "tb_next", sys.last_traceback) - self.locals["_hy_last_traceback"] = sys.last_traceback - - def __call__(self, source, filename="", symbol="single"): - - if source == "pass": - # We need to return a no-op to signal that no more input is needed. - return (compile(source, filename, symbol),) * 2 - - hash_digest = hashlib.sha1(source.encode("utf-8").strip()).hexdigest() - name = "{}-{}".format(filename.strip("<>"), hash_digest) - - self._cache(source, name) - - try: - root_ast = ast.Interactive if symbol == "single" else ast.Module - - # Our compiler doesn't correspond to a real, fixed source file, so - # we need to [re]set these. - self.hy_compiler.filename = name - self.hy_compiler.source = source - hy_ast = read_many( - source, filename=name, reader=self.reader, skip_shebang=True - ) - exec_ast, eval_ast = hy_compile( - hy_ast, - self.module, - root=root_ast, - get_expr=True, - compiler=self.hy_compiler, - filename=name, - source=source, - import_stdlib=False, - ) - - if self.ast_callback: - self.ast_callback(exec_ast, eval_ast) - - exec_code = super().__call__(exec_ast, name, symbol) - eval_code = super().__call__(eval_ast, name, "eval") - - except Exception as e: - # Capture and save the error before we handle further - - sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() - self._update_exc_info() - - if isinstance(e, (PrematureEndOfInput, SyntaxError)): - raise - else: - # Hy will raise exceptions during compile-time that Python would - # raise during run-time (e.g. import errors for `require`). In - # order to work gracefully with the Python world, we convert such - # Hy errors to code that purposefully reraises those exceptions in - # the places where Python code expects them. - # Capture a traceback without the compiler/REPL frames. - exec_code = super(HyCompile, self).__call__( - "raise _hy_last_value.with_traceback(_hy_last_traceback)", - name, - symbol, - ) - eval_code = super(HyCompile, self).__call__("None", name, "eval") - - return exec_code, eval_code - - -class HyCommandCompiler(codeop.CommandCompiler): - def __init__(self, *args, **kwargs): - self.compiler = HyCompile(*args, **kwargs) - - def __call__(self, *args, **kwargs): - try: - return super().__call__(*args, **kwargs) - except PrematureEndOfInput: - # We have to do this here, because `codeop._maybe_compile` won't - # take `None` for a return value (at least not in Python 2.7) and - # this exception type is also a `SyntaxError`, so it will be caught - # by `code.InteractiveConsole` base methods before it reaches our - # `runsource`. - return None - - -class HyREPL(code.InteractiveConsole): - """A subclass of :class:`code.InteractiveConsole` for Hy. - - A convenient way to use this class to interactively debug code is to insert the - following in the code you want to debug:: - - (.run (hy.cmdline.HyREPL :locals (locals))) - - Note that as with :func:`code.interact`, changes to ``(locals)`` inside the REPL are - not propagated back to the original scope.""" - - def __init__(self, spy=False, output_fn=None, locals=None, filename=""): - - # Create a proper module for this REPL so that we can obtain it easily - # (e.g. using `importlib.import_module`). - # We let `InteractiveConsole` initialize `self.locals` when it's - # `None`. - super().__init__(locals=locals, filename=filename) - - module_name = self.locals.get("__name__", "__console__") - # Make sure our newly created module is properly introduced to - # `sys.modules`, and consistently use its namespace as `self.locals` - # from here on. - self.module = sys.modules.setdefault(module_name, types.ModuleType(module_name)) - self.module.__dict__.update(self.locals) - self.locals = self.module.__dict__ - - if os.environ.get("HYSTARTUP"): - try: - loader = HyLoader("__hystartup__", os.environ.get("HYSTARTUP")) - spec = importlib.util.spec_from_loader(loader.name, loader) - mod = importlib.util.module_from_spec(spec) - sys.modules.setdefault(mod.__name__, mod) - loader.exec_module(mod) - imports = mod.__dict__.get( - "__all__", - [name for name in mod.__dict__ if not name.startswith("_")], - ) - imports = {name: mod.__dict__[name] for name in imports} - spy = spy or imports.get("repl_spy") - output_fn = output_fn or imports.get("repl_output_fn") - - # Load imports and defs - self.locals.update(imports) - - # load module macros - require(mod, self.module, assignments="ALL") - require_reader(mod, self.module, assignments="ALL") - except Exception as e: - print(e) - - self.hy_compiler = HyASTCompiler(self.module, module_name) - - self.cmdline_cache = {} - self.compile = HyCommandCompiler( - self.module, - self.locals, - ast_callback=self.ast_callback, - hy_compiler=self.hy_compiler, - cmdline_cache=self.cmdline_cache, - ) - - self.spy = spy - self.last_value = None - self.print_last_value = True - - if output_fn is None: - self.output_fn = hy.repr - elif callable(output_fn): - self.output_fn = output_fn - elif "." in output_fn: - parts = [mangle(x) for x in output_fn.split(".")] - module, f = ".".join(parts[:-1]), parts[-1] - self.output_fn = getattr(importlib.import_module(module), f) - else: - self.output_fn = getattr(builtins, mangle(output_fn)) - - # Pre-mangle symbols for repl recent results: *1, *2, *3 - self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)] - self.locals.update({sym: None for sym in self._repl_results_symbols}) - - # Allow access to the running REPL instance - self.locals["_hy_repl"] = self - - # Compile an empty statement to load the standard prelude - exec_ast = hy_compile( - read_many(""), self.module, compiler=self.hy_compiler, import_stdlib=True - ) - if self.ast_callback: - self.ast_callback(exec_ast, None) - - def ast_callback(self, exec_ast, eval_ast): - if self.spy: - try: - # Mush the two AST chunks into a single module for - # conversion into Python. - new_ast = ast.Module( - exec_ast.body - + ([] if eval_ast is None else [ast.Expr(eval_ast.body)]), - type_ignores=[], - ) - print(ast.unparse(new_ast)) - except Exception: - msg = "Exception in AST callback:\n{}\n".format(traceback.format_exc()) - self.write(msg) - - def _error_wrap(self, error_fn, exc_info_override=False, *args, **kwargs): - sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() - - if exc_info_override: - # Use a traceback that doesn't have the REPL frames. - sys.last_type = self.locals.get("_hy_last_type", sys.last_type) - sys.last_value = self.locals.get("_hy_last_value", sys.last_value) - sys.last_traceback = self.locals.get( - "_hy_last_traceback", sys.last_traceback - ) - - sys.excepthook(sys.last_type, sys.last_value, sys.last_traceback) - - self.locals[mangle("*e")] = sys.last_value - - def showsyntaxerror(self, filename=None): - if filename is None: - filename = self.filename - self.print_last_value = False - - self._error_wrap( - super().showsyntaxerror, exc_info_override=True, filename=filename - ) - - def showtraceback(self): - self._error_wrap(super().showtraceback) - - def runcode(self, code): - try: - eval(code[0], self.locals) - self.last_value = eval(code[1], self.locals) - # Don't print `None` values. - self.print_last_value = self.last_value is not None - except SystemExit: - raise - except Exception as e: - # Set this to avoid a print-out of the last value on errors. - self.print_last_value = False - self.showtraceback() - - def runsource(self, source, filename="", symbol="exec"): - try: - res = super().runsource(source, filename, symbol) - except (HyMacroExpansionError, HyRequireError): - # We need to handle these exceptions ourselves, because the base - # method only handles `OverflowError`, `SyntaxError` and - # `ValueError`. - self.showsyntaxerror(filename) - return False - except (HyLanguageError): - # Our compiler will also raise `TypeError`s - self.showtraceback() - return False - - # Shift exisitng REPL results - if not res: - next_result = self.last_value - for sym in self._repl_results_symbols: - self.locals[sym], next_result = next_result, self.locals[sym] - - # Print the value. - if self.print_last_value: - try: - output = self.output_fn(self.last_value) - except Exception: - self.showtraceback() - return False - - print(output) - - return res - - def run(self): - "Start running the REPL. Return 0 when done." - - import colorama - - sentinel = [] - saved_values = ( - getattr(sys, 'ps1', sentinel), - getattr(sys, 'ps2', sentinel), - builtins.quit, - builtins.exit, - builtins.help) - try: - sys.ps1 = "=> " - sys.ps2 = "... " - builtins.quit = HyQuitter("quit") - builtins.exit = HyQuitter("exit") - builtins.help = HyHelper() - - colorama.init() - - namespace = self.locals - with filtered_hy_exceptions(), extend_linecache(self.cmdline_cache), completion( - Completer(namespace) - ): - self.interact( - "Hy {version} using " - "{py}({build}) {pyversion} on {os}".format( - version=hy.__version__, - py=platform.python_implementation(), - build=platform.python_build()[0], - pyversion=platform.python_version(), - os=platform.system(), - ) - ) - - finally: - sys.ps1, sys.ps2, builtins.quit, builtins.exit, builtins.help = saved_values - for a in 'ps1', 'ps2': - if getattr(sys, a) is sentinel: - delattr(sys, a) - - return 0 - def set_path(filename): """Emulate Python cmdline behavior by setting `sys.path` relative diff --git a/hy/repl.py b/hy/repl.py new file mode 100644 index 000000000..3a466aa4d --- /dev/null +++ b/hy/repl.py @@ -0,0 +1,458 @@ +import argparse +import ast +import builtins +import code +import codeop +import hashlib +import importlib +import io +import linecache +import os +import platform +import py_compile +import runpy +import sys +import time +import traceback +import types +from contextlib import contextmanager +from pathlib import Path + +import hy +from hy._compat import PY3_9, PYPY +from hy.compiler import HyASTCompiler, hy_ast_compile_flags, hy_compile, hy_eval +from hy.completer import Completer, completion +from hy.errors import ( + HyLanguageError, + HyMacroExpansionError, + HyRequireError, + filtered_hy_exceptions, + hy_exc_handler, +) +from hy.importer import HyLoader, runhy +from hy.macros import enable_readers, require, require_reader +from hy.reader import mangle, read_many +from hy.reader.exceptions import PrematureEndOfInput +from hy.reader.hy_reader import HyReader + + +class HyQuitter: + def __init__(self, name): + self.name = name + + def __repr__(self): + return "Use (%s) or Ctrl-D (i.e. EOF) to exit" % (self.name) + + __str__ = __repr__ + + def __call__(self, code=None): + try: + sys.stdin.close() + except: + pass + raise SystemExit(code) + + +class HyHelper: + def __repr__(self): + return ( + "Use (help) for interactive help, or (help object) for help " + "about object." + ) + + def __call__(self, *args, **kwds): + import pydoc + + return pydoc.help(*args, **kwds) + +sys.last_type = None +sys.last_value = None +sys.last_traceback = None + + + + +@contextmanager +def extend_linecache(add_cmdline_cache): + _linecache_checkcache = linecache.checkcache + + def _cmdline_checkcache(*args): + _linecache_checkcache(*args) + linecache.cache.update(add_cmdline_cache) + + linecache.checkcache = _cmdline_checkcache + yield + linecache.checkcache = _linecache_checkcache + + +_codeop_maybe_compile = codeop._maybe_compile + + +def _hy_maybe_compile(compiler, source, filename, symbol): + """The `codeop` version of this will compile the same source multiple + times, and, since we have macros and things like `eval-and-compile`, we + can't allow that. + """ + if not isinstance(compiler, HyCompile): + return _codeop_maybe_compile(compiler, source, filename, symbol) + + for line in source.split("\n"): + line = line.strip() + if line and line[0] != ";": + # Leave it alone (could do more with Hy syntax) + break + else: + if symbol != "eval": + # Replace it with a 'pass' statement (i.e. tell the compiler to do + # nothing) + source = "pass" + + return compiler(source, filename, symbol) + + +codeop._maybe_compile = _hy_maybe_compile + + +class HyCompile(codeop.Compile): + """This compiler uses `linecache` like + `IPython.core.compilerop.CachingCompiler`. + """ + + def __init__( + self, module, locals, ast_callback=None, hy_compiler=None, cmdline_cache={} + ): + self.module = module + self.locals = locals + self.ast_callback = ast_callback + self.hy_compiler = hy_compiler + self.reader = HyReader() + + super().__init__() + + if hasattr(self.module, "__reader_macros__"): + enable_readers( + self.module, self.reader, self.module.__reader_macros__.keys() + ) + + self.flags |= hy_ast_compile_flags + + self.cmdline_cache = cmdline_cache + + def _cache(self, source, name): + entry = ( + len(source), + time.time(), + [line + "\n" for line in source.splitlines()], + name, + ) + + linecache.cache[name] = entry + self.cmdline_cache[name] = entry + + def _update_exc_info(self): + self.locals["_hy_last_type"] = sys.last_type + self.locals["_hy_last_value"] = sys.last_value + # Skip our frame. + sys.last_traceback = getattr(sys.last_traceback, "tb_next", sys.last_traceback) + self.locals["_hy_last_traceback"] = sys.last_traceback + + def __call__(self, source, filename="", symbol="single"): + + if source == "pass": + # We need to return a no-op to signal that no more input is needed. + return (compile(source, filename, symbol),) * 2 + + hash_digest = hashlib.sha1(source.encode("utf-8").strip()).hexdigest() + name = "{}-{}".format(filename.strip("<>"), hash_digest) + + self._cache(source, name) + + try: + root_ast = ast.Interactive if symbol == "single" else ast.Module + + # Our compiler doesn't correspond to a real, fixed source file, so + # we need to [re]set these. + self.hy_compiler.filename = name + self.hy_compiler.source = source + hy_ast = read_many( + source, filename=name, reader=self.reader, skip_shebang=True + ) + exec_ast, eval_ast = hy_compile( + hy_ast, + self.module, + root=root_ast, + get_expr=True, + compiler=self.hy_compiler, + filename=name, + source=source, + import_stdlib=False, + ) + + if self.ast_callback: + self.ast_callback(exec_ast, eval_ast) + + exec_code = super().__call__(exec_ast, name, symbol) + eval_code = super().__call__(eval_ast, name, "eval") + + except Exception as e: + # Capture and save the error before we handle further + + sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + self._update_exc_info() + + if isinstance(e, (PrematureEndOfInput, SyntaxError)): + raise + else: + # Hy will raise exceptions during compile-time that Python would + # raise during run-time (e.g. import errors for `require`). In + # order to work gracefully with the Python world, we convert such + # Hy errors to code that purposefully reraises those exceptions in + # the places where Python code expects them. + # Capture a traceback without the compiler/REPL frames. + exec_code = super(HyCompile, self).__call__( + "raise _hy_last_value.with_traceback(_hy_last_traceback)", + name, + symbol, + ) + eval_code = super(HyCompile, self).__call__("None", name, "eval") + + return exec_code, eval_code + + +class HyCommandCompiler(codeop.CommandCompiler): + def __init__(self, *args, **kwargs): + self.compiler = HyCompile(*args, **kwargs) + + def __call__(self, *args, **kwargs): + try: + return super().__call__(*args, **kwargs) + except PrematureEndOfInput: + # We have to do this here, because `codeop._maybe_compile` won't + # take `None` for a return value (at least not in Python 2.7) and + # this exception type is also a `SyntaxError`, so it will be caught + # by `code.InteractiveConsole` base methods before it reaches our + # `runsource`. + return None + + +class HyREPL(code.InteractiveConsole): + """A subclass of :class:`code.InteractiveConsole` for Hy. + + A convenient way to use this class to interactively debug code is to insert the + following in the code you want to debug:: + + (.run (hy.cmdline.HyREPL :locals (locals))) + + Note that as with :func:`code.interact`, changes to ``(locals)`` inside the REPL are + not propagated back to the original scope.""" + + def __init__(self, spy=False, output_fn=None, locals=None, filename=""): + + # Create a proper module for this REPL so that we can obtain it easily + # (e.g. using `importlib.import_module`). + # We let `InteractiveConsole` initialize `self.locals` when it's + # `None`. + super().__init__(locals=locals, filename=filename) + + module_name = self.locals.get("__name__", "__console__") + # Make sure our newly created module is properly introduced to + # `sys.modules`, and consistently use its namespace as `self.locals` + # from here on. + self.module = sys.modules.setdefault(module_name, types.ModuleType(module_name)) + self.module.__dict__.update(self.locals) + self.locals = self.module.__dict__ + + if os.environ.get("HYSTARTUP"): + try: + loader = HyLoader("__hystartup__", os.environ.get("HYSTARTUP")) + spec = importlib.util.spec_from_loader(loader.name, loader) + mod = importlib.util.module_from_spec(spec) + sys.modules.setdefault(mod.__name__, mod) + loader.exec_module(mod) + imports = mod.__dict__.get( + "__all__", + [name for name in mod.__dict__ if not name.startswith("_")], + ) + imports = {name: mod.__dict__[name] for name in imports} + spy = spy or imports.get("repl_spy") + output_fn = output_fn or imports.get("repl_output_fn") + + # Load imports and defs + self.locals.update(imports) + + # load module macros + require(mod, self.module, assignments="ALL") + require_reader(mod, self.module, assignments="ALL") + except Exception as e: + print(e) + + self.hy_compiler = HyASTCompiler(self.module, module_name) + + self.cmdline_cache = {} + self.compile = HyCommandCompiler( + self.module, + self.locals, + ast_callback=self.ast_callback, + hy_compiler=self.hy_compiler, + cmdline_cache=self.cmdline_cache, + ) + + self.spy = spy + self.last_value = None + self.print_last_value = True + + if output_fn is None: + self.output_fn = hy.repr + elif callable(output_fn): + self.output_fn = output_fn + elif "." in output_fn: + parts = [mangle(x) for x in output_fn.split(".")] + module, f = ".".join(parts[:-1]), parts[-1] + self.output_fn = getattr(importlib.import_module(module), f) + else: + self.output_fn = getattr(builtins, mangle(output_fn)) + + # Pre-mangle symbols for repl recent results: *1, *2, *3 + self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)] + self.locals.update({sym: None for sym in self._repl_results_symbols}) + + # Allow access to the running REPL instance + self.locals["_hy_repl"] = self + + # Compile an empty statement to load the standard prelude + exec_ast = hy_compile( + read_many(""), self.module, compiler=self.hy_compiler, import_stdlib=True + ) + if self.ast_callback: + self.ast_callback(exec_ast, None) + + def ast_callback(self, exec_ast, eval_ast): + if self.spy: + try: + # Mush the two AST chunks into a single module for + # conversion into Python. + new_ast = ast.Module( + exec_ast.body + + ([] if eval_ast is None else [ast.Expr(eval_ast.body)]), + type_ignores=[], + ) + print(ast.unparse(new_ast)) + except Exception: + msg = "Exception in AST callback:\n{}\n".format(traceback.format_exc()) + self.write(msg) + + def _error_wrap(self, error_fn, exc_info_override=False, *args, **kwargs): + sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + + if exc_info_override: + # Use a traceback that doesn't have the REPL frames. + sys.last_type = self.locals.get("_hy_last_type", sys.last_type) + sys.last_value = self.locals.get("_hy_last_value", sys.last_value) + sys.last_traceback = self.locals.get( + "_hy_last_traceback", sys.last_traceback + ) + + sys.excepthook(sys.last_type, sys.last_value, sys.last_traceback) + + self.locals[mangle("*e")] = sys.last_value + + def showsyntaxerror(self, filename=None): + if filename is None: + filename = self.filename + self.print_last_value = False + + self._error_wrap( + super().showsyntaxerror, exc_info_override=True, filename=filename + ) + + def showtraceback(self): + self._error_wrap(super().showtraceback) + + def runcode(self, code): + try: + eval(code[0], self.locals) + self.last_value = eval(code[1], self.locals) + # Don't print `None` values. + self.print_last_value = self.last_value is not None + except SystemExit: + raise + except Exception as e: + # Set this to avoid a print-out of the last value on errors. + self.print_last_value = False + self.showtraceback() + + def runsource(self, source, filename="", symbol="exec"): + try: + res = super().runsource(source, filename, symbol) + except (HyMacroExpansionError, HyRequireError): + # We need to handle these exceptions ourselves, because the base + # method only handles `OverflowError`, `SyntaxError` and + # `ValueError`. + self.showsyntaxerror(filename) + return False + except (HyLanguageError): + # Our compiler will also raise `TypeError`s + self.showtraceback() + return False + + # Shift exisitng REPL results + if not res: + next_result = self.last_value + for sym in self._repl_results_symbols: + self.locals[sym], next_result = next_result, self.locals[sym] + + # Print the value. + if self.print_last_value: + try: + output = self.output_fn(self.last_value) + except Exception: + self.showtraceback() + return False + + print(output) + + return res + + def run(self): + "Start running the REPL. Return 0 when done." + + import colorama + + sentinel = [] + saved_values = ( + getattr(sys, 'ps1', sentinel), + getattr(sys, 'ps2', sentinel), + builtins.quit, + builtins.exit, + builtins.help) + try: + sys.ps1 = "=> " + sys.ps2 = "... " + builtins.quit = HyQuitter("quit") + builtins.exit = HyQuitter("exit") + builtins.help = HyHelper() + + colorama.init() + + namespace = self.locals + with filtered_hy_exceptions(), extend_linecache(self.cmdline_cache), completion( + Completer(namespace) + ): + self.interact( + "Hy {version} using " + "{py}({build}) {pyversion} on {os}".format( + version=hy.__version__, + py=platform.python_implementation(), + build=platform.python_build()[0], + pyversion=platform.python_version(), + os=platform.system(), + ) + ) + + finally: + sys.ps1, sys.ps2, builtins.quit, builtins.exit, builtins.help = saved_values + for a in 'ps1', 'ps2': + if getattr(sys, a) is sentinel: + delattr(sys, a) + + return 0 From a7db96358330170241e35e4d954dcdaa56c2c7b4 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 12 Nov 2022 09:20:12 -0500 Subject: [PATCH 041/342] Rename `hy.cmdline.HyREPL` to `hy.REPL` --- NEWS.rst | 10 +++++++--- docs/interop.rst | 6 +++--- docs/repl.rst | 6 +++--- hy/__init__.py | 1 + hy/cmdline.py | 5 +++-- hy/errors.py | 1 + hy/repl.py | 7 +++++-- tests/native_tests/repl.hy | 5 ++--- 8 files changed, 25 insertions(+), 16 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 75df90652..2f9f57b9e 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,11 +3,15 @@ Unreleased ============================= +Breaking Changes +------------------------------ +* `hy.cmdline.HyREPL` is now `hy.REPL`. + Bug Fixes ------------------------------ -* `HyREPL` now restores the global values it changes (such as - `sys.ps1`) after `HyREPL.run` terminates. -* `HyREPL` no longer mixes up Hy's and Python's Readline histories +* `hy.REPL` now restores the global values it changes (such as + `sys.ps1`) after `hy.REPL.run` terminates. +* `hy.REPL` no longer mixes up Hy's and Python's Readline histories when run inside Python's REPL. New Features diff --git a/docs/interop.rst b/docs/interop.rst index 35f64aa04..34a68b26f 100644 --- a/docs/interop.rst +++ b/docs/interop.rst @@ -83,12 +83,12 @@ causes the import of ``name`` from ``hy.core.language``. Launching a Hy REPL from Python ------------------------------- -You can use :meth:`hy.cmdline.HyREPL.run` to launch the Hy REPL from Python: +You can use :meth:`hy.REPL.run` to launch the Hy REPL from Python: .. code-block:: text - >>> import hy.cmdline - >>> hy.cmdline.HyREPL(locals = locals()).run() + >>> import hy + >>> hy.REPL(locals = locals()).run() Hy x.y.z using CPython(default) x.y.z on Linux => (defn test [] (print "bar")) => (test) diff --git a/docs/repl.rst b/docs/repl.rst index 0f881b2cb..3df91088d 100644 --- a/docs/repl.rst +++ b/docs/repl.rst @@ -4,16 +4,16 @@ The Hy REPL Hy's `read-eval-print loop `_ (REPL) is implemented in -the class :class:`hy.cmdline.HyREPL`. The REPL can be started interactively +the class :class:`hy.REPL`. The REPL can be started interactively :doc:`from the command line ` or programmatically with the instance method -:meth:`hy.cmdline.HyREPL.run`. +:meth:`hy.REPL.run`. Two :doc:`environment variables ` useful for the REPL are ``HY_HISTORY``, which specifies where the REPL input history is saved, and ``HYSTARTUP``, which specifies :ref:`a file to run when the REPL starts `. -.. autoclass:: hy.cmdline.HyREPL +.. autoclass:: hy.REPL :members: run .. _repl-output-function: diff --git a/hy/__init__.py b/hy/__init__.py index fba3b9afc..aa60b7a48 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -31,6 +31,7 @@ def _initialize_env_var(env_var, default_val): macroexpand_1="hy.core.util", disassemble="hy.core.util", as_model="hy.models", + REPL="hy.repl" ) diff --git a/hy/cmdline.py b/hy/cmdline.py index 53f2f4739..d915301c1 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -34,6 +34,7 @@ from hy.reader import mangle, read_many from hy.reader.exceptions import PrematureEndOfInput from hy.reader.hy_reader import HyReader +from hy.repl import REPL def set_path(filename): @@ -72,7 +73,7 @@ def run_icommand(source, **kwargs): else: filename = "" - hr = HyREPL(**kwargs) + hr = REPL(**kwargs) with filtered_hy_exceptions(): res = hr.runsource(source, filename=filename) @@ -308,7 +309,7 @@ def proc_opt(opt, arg=None, item=None, i=None): hy_exc_handler(*sys.exc_info()) sys.exit(1) - return HyREPL(spy=options.get("spy"), output_fn=options.get("repl_output_fn")).run() + return REPL(spy=options.get("spy"), output_fn=options.get("repl_output_fn")).run() # entry point for cmd line script "hy" diff --git a/hy/errors.py b/hy/errors.py index 084effaf5..6361550e1 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -237,6 +237,7 @@ def _module_filter_name(module_name): "hy.compiler", "hy.reader", "hy.cmdline", + "hy.repl", "hy.reader.parser", "hy.importer", "hy._compat", diff --git a/hy/repl.py b/hy/repl.py index 3a466aa4d..2cffb6ebc 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -235,13 +235,13 @@ def __call__(self, *args, **kwargs): return None -class HyREPL(code.InteractiveConsole): +class REPL(code.InteractiveConsole): """A subclass of :class:`code.InteractiveConsole` for Hy. A convenient way to use this class to interactively debug code is to insert the following in the code you want to debug:: - (.run (hy.cmdline.HyREPL :locals (locals))) + (.run (hy.REPL :locals (locals))) Note that as with :func:`code.interact`, changes to ``(locals)`` inside the REPL are not propagated back to the original scope.""" @@ -456,3 +456,6 @@ def run(self): delattr(sys, a) return 0 + + +REPL.__module__ = "hy" # Print as `hy.REPL` instead of `hy.repl.REPL`. diff --git a/tests/native_tests/repl.hy b/tests/native_tests/repl.hy index 70b899e1c..2b37e0ba2 100644 --- a/tests/native_tests/repl.hy +++ b/tests/native_tests/repl.hy @@ -2,13 +2,12 @@ (import io - sys - hy.cmdline [HyREPL]) + sys) (defn test-preserve-ps1 [monkeypatch] ; https://github.com/hylang/hy/issues/1323#issuecomment-1310837340 (monkeypatch.setattr "sys.stdin" (io.StringIO "(+ 1 1)")) (setv sys.ps1 "chippy") (assert (= sys.ps1 "chippy")) - (.run (HyREPL)) + (.run (hy.REPL)) (assert (= sys.ps1 "chippy"))) From ca9db459b1a8d04a878d48d4e79ddd6e12a43233 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 12 Nov 2022 09:29:38 -0500 Subject: [PATCH 042/342] Weed imports --- hy/cmdline.py | 22 ++++------------------ hy/compiler.py | 1 - hy/completer.py | 1 - hy/reader/mangling.py | 1 - hy/reader/reader.py | 3 +-- hy/repl.py | 11 ++--------- hy/scoping.py | 1 - 7 files changed, 7 insertions(+), 33 deletions(-) diff --git a/hy/cmdline.py b/hy/cmdline.py index d915301c1..b4caf9648 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -1,39 +1,25 @@ import argparse import ast -import builtins -import code -import codeop -import hashlib import importlib import io -import linecache import os import platform import py_compile import runpy import sys -import time -import traceback -import types -from contextlib import contextmanager from pathlib import Path import hy from hy._compat import PY3_9, PYPY -from hy.compiler import HyASTCompiler, hy_ast_compile_flags, hy_compile, hy_eval -from hy.completer import Completer, completion +from hy.compiler import hy_compile, hy_eval from hy.errors import ( HyLanguageError, - HyMacroExpansionError, - HyRequireError, filtered_hy_exceptions, hy_exc_handler, ) -from hy.importer import HyLoader, runhy -from hy.macros import enable_readers, require, require_reader -from hy.reader import mangle, read_many -from hy.reader.exceptions import PrematureEndOfInput -from hy.reader.hy_reader import HyReader +from hy.importer import runhy +from hy.macros import require +from hy.reader import read_many from hy.repl import REPL diff --git a/hy/compiler.py b/hy/compiler.py index ae4f02eba..8db7db2b0 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -2,7 +2,6 @@ import copy import importlib import inspect -import keyword import traceback import types diff --git a/hy/completer.py b/hy/completer.py index bc1aa6b01..adeebfa48 100644 --- a/hy/completer.py +++ b/hy/completer.py @@ -4,7 +4,6 @@ import re import sys -import hy.macros from hy import mangle, unmangle # Lazily import `readline` to work around diff --git a/hy/reader/mangling.py b/hy/reader/mangling.py index 5329be4f8..c22ac5282 100644 --- a/hy/reader/mangling.py +++ b/hy/reader/mangling.py @@ -1,4 +1,3 @@ -import keyword import re import unicodedata diff --git a/hy/reader/reader.py b/hy/reader/reader.py index b2282f597..2651945c3 100644 --- a/hy/reader/reader.py +++ b/hy/reader/reader.py @@ -4,9 +4,8 @@ import re from collections import deque from contextlib import contextmanager -from io import StringIO -from .exceptions import LexException, PrematureEndOfInput +from .exceptions import PrematureEndOfInput _whitespace = re.compile(r"[ \t\n\r\f\v]+") diff --git a/hy/repl.py b/hy/repl.py index 2cffb6ebc..08749cce6 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -1,35 +1,28 @@ -import argparse import ast import builtins import code import codeop import hashlib import importlib -import io import linecache import os import platform -import py_compile -import runpy import sys import time import traceback import types from contextlib import contextmanager -from pathlib import Path import hy -from hy._compat import PY3_9, PYPY -from hy.compiler import HyASTCompiler, hy_ast_compile_flags, hy_compile, hy_eval +from hy.compiler import HyASTCompiler, hy_ast_compile_flags, hy_compile from hy.completer import Completer, completion from hy.errors import ( HyLanguageError, HyMacroExpansionError, HyRequireError, filtered_hy_exceptions, - hy_exc_handler, ) -from hy.importer import HyLoader, runhy +from hy.importer import HyLoader from hy.macros import enable_readers, require, require_reader from hy.reader import mangle, read_many from hy.reader.exceptions import PrematureEndOfInput diff --git a/hy/scoping.py b/hy/scoping.py index 3fd8f2daf..d7bb03ea7 100644 --- a/hy/scoping.py +++ b/hy/scoping.py @@ -5,7 +5,6 @@ from abc import ABC, abstractmethod import hy._compat -from hy.errors import HyInternalError from hy.models import Expression, List, Symbol, Tuple from hy.reader import mangle From b7cd5f70300984e12fa08cabeb958feb331a0934 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 14 Nov 2022 13:13:35 -0500 Subject: [PATCH 043/342] Autoformat --- hy/__init__.py | 2 +- hy/cmdline.py | 6 +----- hy/repl.py | 18 +++++++++--------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/hy/__init__.py b/hy/__init__.py index aa60b7a48..ca6adfa1c 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -31,7 +31,7 @@ def _initialize_env_var(env_var, default_val): macroexpand_1="hy.core.util", disassemble="hy.core.util", as_model="hy.models", - REPL="hy.repl" + REPL="hy.repl", ) diff --git a/hy/cmdline.py b/hy/cmdline.py index b4caf9648..965d7de6d 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -12,11 +12,7 @@ import hy from hy._compat import PY3_9, PYPY from hy.compiler import hy_compile, hy_eval -from hy.errors import ( - HyLanguageError, - filtered_hy_exceptions, - hy_exc_handler, -) +from hy.errors import HyLanguageError, filtered_hy_exceptions, hy_exc_handler from hy.importer import runhy from hy.macros import require from hy.reader import read_many diff --git a/hy/repl.py b/hy/repl.py index 08749cce6..b34f52938 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -58,13 +58,12 @@ def __call__(self, *args, **kwds): return pydoc.help(*args, **kwds) + sys.last_type = None sys.last_value = None sys.last_traceback = None - - @contextmanager def extend_linecache(add_cmdline_cache): _linecache_checkcache = linecache.checkcache @@ -413,11 +412,12 @@ def run(self): sentinel = [] saved_values = ( - getattr(sys, 'ps1', sentinel), - getattr(sys, 'ps2', sentinel), + getattr(sys, "ps1", sentinel), + getattr(sys, "ps2", sentinel), builtins.quit, builtins.exit, - builtins.help) + builtins.help, + ) try: sys.ps1 = "=> " sys.ps2 = "... " @@ -428,9 +428,9 @@ def run(self): colorama.init() namespace = self.locals - with filtered_hy_exceptions(), extend_linecache(self.cmdline_cache), completion( - Completer(namespace) - ): + with filtered_hy_exceptions(), extend_linecache( + self.cmdline_cache + ), completion(Completer(namespace)): self.interact( "Hy {version} using " "{py}({build}) {pyversion} on {os}".format( @@ -444,7 +444,7 @@ def run(self): finally: sys.ps1, sys.ps2, builtins.quit, builtins.exit, builtins.help = saved_values - for a in 'ps1', 'ps2': + for a in "ps1", "ps2": if getattr(sys, a) is sentinel: delattr(sys, a) From d4d466e1730c73d465544f047a20de846c419843 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 22 Nov 2022 09:32:50 -0500 Subject: [PATCH 044/342] Peg Sphinx at 5.0.2 On the latest version, 5.3.0, module docstrings don't appear. This is likely a bug in `sphinxcontrib-hydomain`, but I don't have the tuits to learn that codebase yet. --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e7802ba88..6aaaf00e6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,6 +7,6 @@ pre_commit==2.17.0 # documentation Pygments >= 2 -Sphinx >= 5 +Sphinx == 5.0.2 sphinx_rtd_theme >= 1 git+https://github.com/hylang/sphinxcontrib-hydomain.git From cc617d1e21f604b0a02c0c3607bc78fdbf3a3e7f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 22 Nov 2022 09:41:49 -0500 Subject: [PATCH 045/342] Add another example non-short-circuiting function --- hy/pyops.hy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/pyops.hy b/hy/pyops.hy index b09ca8c9d..992e9ad0b 100644 --- a/hy/pyops.hy +++ b/hy/pyops.hy @@ -13,7 +13,7 @@ macro instead of the function. The functions in ``hy.pyops`` have the same semantics as their macro equivalents, with one exception: functions can't short-circuit, so the -functions for the logical operators, such as ``and``, unconditionally +functions for operators such as ``and`` and ``!=`` unconditionally evaluate all arguments." ;;;; Hy shadow functions From ebb39b5b3397b3a15beb31cbc00050d6bf3d2229 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 22 Nov 2022 09:46:06 -0500 Subject: [PATCH 046/342] Fix the documentation of nullary `*` --- hy/pyops.hy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/pyops.hy b/hy/pyops.hy index 992e9ad0b..7f905e2e8 100644 --- a/hy/pyops.hy +++ b/hy/pyops.hy @@ -66,7 +66,7 @@ evaluate all arguments." (defop * [#* args] ["multiplication" - :nullary "0" + :nullary "1" :unary "x"] (if (= (len args) 0) 1 From c7fa84beb527df3ea79af0a60ef9b994665e2f8f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 29 Nov 2022 14:52:34 -0500 Subject: [PATCH 047/342] Fix `hy.repr` of bad uses of `quote` etc. --- NEWS.rst | 2 ++ hy/core/hy_repr.hy | 2 +- tests/resources/hy_repr_str_tests.txt | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 2f9f57b9e..d127c5d3e 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -13,6 +13,8 @@ Bug Fixes `sys.ps1`) after `hy.REPL.run` terminates. * `hy.REPL` no longer mixes up Hy's and Python's Readline histories when run inside Python's REPL. +* Fixed `hy.repr` of non-compilable uses of sugared macros, such as + `(quote)` and `(quote 1 2)`. New Features ------------------------------ diff --git a/hy/core/hy_repr.hy b/hy/core/hy_repr.hy index 51fd88d1d..e351fd4ef 100644 --- a/hy/core/hy_repr.hy +++ b/hy/core/hy_repr.hy @@ -115,7 +115,7 @@ 'unpack-iterable "#* " 'unpack-mapping "#** "}) (cond - (and x (in (get x 0) syntax)) + (and (= (len x) 2) (in (get x 0) syntax)) (+ (get syntax (get x 0)) (hy-repr (get x 1))) True (+ "(" (_cat x) ")")))) diff --git a/tests/resources/hy_repr_str_tests.txt b/tests/resources/hy_repr_str_tests.txt index aa6c09abf..3b901060c 100644 --- a/tests/resources/hy_repr_str_tests.txt +++ b/tests/resources/hy_repr_str_tests.txt @@ -91,6 +91,7 @@ b"\"double \" quotes\"" '(f a b) '(f #* args #** kwargs) [1 [2 3] #(4 #('mysymbol :mykeyword)) {"a" b"hello"} '(f #* a #** b)] +'(quote) ;; * Quasiquoting @@ -98,6 +99,8 @@ b"\"double \" quotes\"" '[1 `[~foo ~@bar] 4] '[1 `[~(+ 1 2) ~@(+ [1] [2])] 4] '[1 `[~(do (print x 'y) 1)] 4] +'(quasiquote 1 2 3) +'[1 `[2 (unquote foo bar)]] ;; * F-strings From 2de9a3e8d0b7e641eb6565b7a7732e5b1dbc27e9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 29 Nov 2022 14:04:47 -0500 Subject: [PATCH 048/342] Delete some AST tests redundant with the op tests --- tests/compilers/test_ast.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 800c830c3..efa2882f0 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -81,21 +81,6 @@ def test_ast_valid_if(): can_compile("(if foo bar baz)") -def test_ast_valid_unary_op(): - "Make sure AST can compile valid unary operator" - can_compile("(not 2)") - can_compile("(~ 1)") - - -def test_ast_invalid_unary_op(): - "Make sure AST can't compile invalid unary operator" - cant_compile("(not 2 3 4)") - cant_compile("(not)") - cant_compile("(not 2 3 4)") - cant_compile("(~ 2 2 3 4)") - cant_compile("(~)") - - def test_ast_bad_while(): "Make sure AST can't compile invalid while" cant_compile("(while)") From 72f59a7d363595e4b48515e961975f01d74c58e1 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 29 Nov 2022 14:08:13 -0500 Subject: [PATCH 049/342] Rename the bitwise NOT operator `~` to `bnot` --- hy/core/hy_repr.hy | 2 +- hy/core/result_macros.py | 4 ++-- hy/pyops.hy | 7 ++++--- tests/native_tests/operators.hy | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/hy/core/hy_repr.hy b/hy/core/hy_repr.hy index e351fd4ef..22c848cd4 100644 --- a/hy/core/hy_repr.hy +++ b/hy/core/hy_repr.hy @@ -201,7 +201,7 @@ (hy-repr (.span x)) (hy-repr (.group x 0))))) (hy-repr-register re.Pattern (fn [x] - (setv flags (& x.flags (~ re.UNICODE))) + (setv flags (& x.flags (bnot re.UNICODE))) ; We remove re.UNICODE since it's redundant with the type ; of the pattern, and Python's `repr` omits it, too. (.format "(re.compile {}{})" diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 8d5390e45..540c07fcb 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -223,9 +223,9 @@ def render_quoted_form(compiler, form, level): # ------------------------------------------------ -@pattern_macro(["not", "~"], [FORM], shadow=True) +@pattern_macro(["not", "bnot"], [FORM], shadow=True) def compile_unary_operator(compiler, expr, root, arg): - ops = {"not": ast.Not, "~": ast.Invert} + ops = {"not": ast.Not, "bnot": ast.Invert} operand = compiler.compile(arg) return operand + asty.UnaryOp(expr, op=ops[root](), operand=operand.force_expr) diff --git a/hy/pyops.hy b/hy/pyops.hy index 7f905e2e8..6aa5ce30b 100644 --- a/hy/pyops.hy +++ b/hy/pyops.hy @@ -132,12 +132,13 @@ evaluate all arguments." :n-ary None] (^ x y)) -(defop ~ [x] +(defop bnot [x] ["bitwise NOT" + :pyop "~" :unary "~x" :binary None :n-ary None] - (~ x)) + (bnot x)) (defn comp-op [op a1 a-rest] "Helper for shadow comparison operators" @@ -239,7 +240,7 @@ evaluate all arguments." (setv __all__ (list (map hy.mangle [ '+ '- '* '** '/ '// '% '@ - '<< '>> '& '| '^ '~ + '<< '>> '& '| '^ 'bnot '< '> '<= '>= '= '!= 'and 'or 'not 'is 'is-not 'in 'not-in diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy index 0d2618bc1..757e732bb 100644 --- a/tests/native_tests/operators.hy +++ b/tests/native_tests/operators.hy @@ -158,7 +158,7 @@ ; so we don't allow `^` with 3 arguments, either. -(op-and-shadow-test ~ +(op-and-shadow-test bnot (forbid (f)) (assert (= (& (f 0b00101111) 0xFF) 0b11010000)) From f19bef626d6713dc1ee866dc2aafbc8b6f2740ed Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 29 Nov 2022 14:10:37 -0500 Subject: [PATCH 050/342] Disallow "`" and "~" in identifiers --- docs/syntax.rst | 2 +- hy/reader/hy_reader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/syntax.rst b/docs/syntax.rst index 5f4ba7867..26fe094c5 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -130,7 +130,7 @@ Identifiers Identifiers are a broad class of syntax in Hy, comprising not only variable names, but any nonempty sequence of characters that aren't ASCII whitespace nor -one of the following: ``()[]{};"'``. The reader will attempt to read each +one of the following: ``()[]{};"'`~``. The reader will attempt to read each identifier as a :ref:`numeric literal `, then attempt to read it as a :ref:`keyword ` if that fails, then fall back on reading it as a :ref:`symbol ` if that fails. diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 0e80b2a3a..ca7f92714 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -97,7 +97,7 @@ class HyReader(Reader): # Components necessary for Reader implementation ### - NON_IDENT = set("()[]{};\"'") + NON_IDENT = set("()[]{};\"'`~") def fill_pos(self, model, start): """Attach line/col information to a model. From d56943d29cd293e37eeb00ab7070cbbb96d42773 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 29 Nov 2022 14:42:09 -0500 Subject: [PATCH 051/342] Simplify reading of "'", "`", and "~" --- hy/reader/hy_reader.py | 18 +----------------- tests/test_reader.py | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index ca7f92714..fd77c8470 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -283,26 +283,10 @@ def quote_closing(c): @reader_for("'", ("quote",)) @reader_for("`", ("quasiquote",)) def tag_as(root): - def _tag_as(self, _): - nc = self.peekc() - if ( - not nc - or isnormalizedspace(nc) - or self.reader_table.get(nc) == self.INVALID - ): - raise LexException.from_reader( - "Could not identify the next token.", self - ) - model = self.parse_one_form() - return mkexpr(root, model) - - return _tag_as + return lambda self, _: mkexpr(root, self.parse_one_form()) @reader_for("~") def unquote(self, key): - nc = self.peekc() - if not nc or isnormalizedspace(nc) or self.reader_table.get(nc) == self.INVALID: - return sym(key) return mkexpr( "unquote" + ("-splice" if self.peek_and_getc("@") else ""), self.parse_one_form(), diff --git a/tests/test_reader.py b/tests/test_reader.py index f8815a550..94c49cda2 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -86,7 +86,7 @@ def test_lex_single_quote_err(): ' File "", line 1', " '", " ^", - "hy.reader.exceptions.LexException: Could not identify the next token.", + "hy.reader.exceptions.PrematureEndOfInput: Premature end of input while attempting to parse one form" ], ) @@ -97,6 +97,27 @@ def test_lex_expression_symbols(): assert objs == [Expression([Symbol("foo"), Symbol("bar")])] +def test_symbol_and_sugar(): + # https://github.com/hylang/hy/issues/1798 + + s = Symbol + def e(*x): return Expression(x) + + for char, head in ( + ("'", 'quote'), + ('`', 'quasiquote'), + ('~', 'unquote'), + ('~@', 'unquote-splice')): + for string in ( + f'a{s1}{char}{s2}b' + for s1 in ('', ' ') + for s2 in ('', ' ')): + assert tokenize(string) == [s("a"), e(s(head), s("b"))] + + assert (tokenize("a~ @b") == tokenize("a ~ @b") == + [s("a"), e(s("unquote"), s("@b"))]) + + def test_lex_expression_strings(): """Test that expressions can produce strings""" objs = tokenize('(foo "bar")') From ee11d7ca6d5d3927bd59984ad04a41cc9f383762 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 29 Nov 2022 14:59:46 -0500 Subject: [PATCH 052/342] Fix `hy.repr` of unquoting a name with "@" --- hy/core/hy_repr.hy | 9 ++++++++- tests/resources/hy_repr_str_tests.txt | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hy/core/hy_repr.hy b/hy/core/hy_repr.hy index 22c848cd4..b98b36e98 100644 --- a/hy/core/hy_repr.hy +++ b/hy/core/hy_repr.hy @@ -116,7 +116,14 @@ 'unpack-mapping "#** "}) (cond (and (= (len x) 2) (in (get x 0) syntax)) - (+ (get syntax (get x 0)) (hy-repr (get x 1))) + (if (and + (= (get x 0) 'unquote) + (isinstance (get x 1) hy.models.Symbol) + (.startswith (get x 1) "@")) + ; This case is special because `~@b` would be wrongly + ; interpreted as `(unquote-splice b)` instead of `(unquote @b)`. + (+ "~ " (hy-repr (get x 1))) + (+ (get syntax (get x 0)) (hy-repr (get x 1)))) True (+ "(" (_cat x) ")")))) diff --git a/tests/resources/hy_repr_str_tests.txt b/tests/resources/hy_repr_str_tests.txt index 3b901060c..668eb4592 100644 --- a/tests/resources/hy_repr_str_tests.txt +++ b/tests/resources/hy_repr_str_tests.txt @@ -101,6 +101,8 @@ b"\"double \" quotes\"" '[1 `[~(do (print x 'y) 1)] 4] '(quasiquote 1 2 3) '[1 `[2 (unquote foo bar)]] +'[a ~@b] +'[a ~ @b] ;; * F-strings From f19c78b0b81977ec2bb121c2dc915d5fa7ba9668 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 29 Nov 2022 15:02:31 -0500 Subject: [PATCH 053/342] Reorder the sugar table in the manual --- docs/syntax.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/syntax.rst b/docs/syntax.rst index 26fe094c5..23b94b0c4 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -453,12 +453,12 @@ gets produced is the same whether you take your code with sugar or without. ========================== ================ Macro Syntax ========================== ================ -:hy:func:`quasiquote` ```FORM`` :hy:func:`quote` ``'FORM`` -:hy:func:`unpack-iterable` ``#* FORM`` -:hy:func:`unpack-mapping` ``#** FORM`` +:hy:func:`quasiquote` ```FORM`` :hy:func:`unquote` ``~FORM`` :hy:func:`unquote-splice` ``~@FORM`` +:hy:func:`unpack-iterable` ``#* FORM`` +:hy:func:`unpack-mapping` ``#** FORM`` ========================== ================ Reader macros From 5490ab36d62794f285ed98c80c2fc904463658d3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 29 Nov 2022 15:23:03 -0500 Subject: [PATCH 054/342] Update docs and NEWS --- NEWS.rst | 9 +++++++++ docs/api.rst | 4 ++++ docs/syntax.rst | 5 +++-- hy/pyops.hy | 7 +++++-- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index d127c5d3e..95375059a 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,15 @@ Unreleased Breaking Changes ------------------------------ +* Various warts have been smoothed over in the syntax of `'`, + \`, `~`, and `~@`: + + * Whitespace is now allowed after these syntactic elements. Thus one + can apply `~` to a symbol whose name begins with "@". + * \` and `~` are no longer allowed in identifiers. (This was already + the case for `'`.) + * The bitwise NOT operator `~` has been renamed to `bnot`. + * `hy.cmdline.HyREPL` is now `hy.REPL`. Bug Fixes diff --git a/docs/api.rst b/docs/api.rst index 942770ad9..843508ac7 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1275,6 +1275,10 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. leaving no effects on the list it is enclosed in, therefore resulting in ``('+' 1 2)``. + A symbol name can begin with ``@`` in Hy, but ``~@`` takes precedence in the + parser. So, if you want to unquote the symbol ``@foo`` with ``~``, you must + use whitespace to separate ``~`` and ``@``, as in ``~ @foo``. + .. hy:function:: (while [condition #* body]) ``while`` compiles to a :py:keyword:`while` statement. It is used to execute a diff --git a/docs/syntax.rst b/docs/syntax.rst index 23b94b0c4..511f2a2da 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -447,8 +447,9 @@ Syntactic sugar is available to construct two-item :ref:`expressions by the reader, a new expression is created with the corresponding macro as the first element and the next parsed form as the second. No parentheses are required. Thus, since ``'`` is short for ``quote``, ``'FORM`` is read as -``(quote FORM)``. This is all resolved at the reader level, so the model that -gets produced is the same whether you take your code with sugar or without. +``(quote FORM)``. Whitespace is allowed, as in ``' FORM``. This is all resolved +at the reader level, so the model that gets produced is the same whether you +take your code with sugar or without. ========================== ================ Macro Syntax diff --git a/hy/pyops.hy b/hy/pyops.hy index 6aa5ce30b..339ec6f4b 100644 --- a/hy/pyops.hy +++ b/hy/pyops.hy @@ -1,8 +1,11 @@ "Python provides various :ref:`binary and unary operators `. These are usually invoked in Hy using core macros of the same name: for example, ``(+ 1 2)`` calls the core macro named -``+``, which uses Python's addition operator. An exception to the names -being the same is that Python's ``==`` is called ``=`` in Hy. +``+``, which uses Python's addition operator. There are two exceptions +to the names being the same: + +- ``==`` in Python is ``=`` in Hy. +- ``~`` in Python is ``bnot`` in Hy. By importing from the module ``hy.pyops`` (typically with a star import, as in ``(import hy.pyops *)``), you can also use these operators as From 1537bd7686582b3fc352ff5510ed59701e5683e9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 2 Dec 2022 16:53:50 -0500 Subject: [PATCH 055/342] Autoformat --- tests/test_reader.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/test_reader.py b/tests/test_reader.py index 94c49cda2..68813438b 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -86,7 +86,7 @@ def test_lex_single_quote_err(): ' File "", line 1', " '", " ^", - "hy.reader.exceptions.PrematureEndOfInput: Premature end of input while attempting to parse one form" + "hy.reader.exceptions.PrematureEndOfInput: Premature end of input while attempting to parse one form", ], ) @@ -101,21 +101,20 @@ def test_symbol_and_sugar(): # https://github.com/hylang/hy/issues/1798 s = Symbol - def e(*x): return Expression(x) + + def e(*x): + return Expression(x) for char, head in ( - ("'", 'quote'), - ('`', 'quasiquote'), - ('~', 'unquote'), - ('~@', 'unquote-splice')): - for string in ( - f'a{s1}{char}{s2}b' - for s1 in ('', ' ') - for s2 in ('', ' ')): + ("'", "quote"), + ("`", "quasiquote"), + ("~", "unquote"), + ("~@", "unquote-splice"), + ): + for string in (f"a{s1}{char}{s2}b" for s1 in ("", " ") for s2 in ("", " ")): assert tokenize(string) == [s("a"), e(s(head), s("b"))] - assert (tokenize("a~ @b") == tokenize("a ~ @b") == - [s("a"), e(s("unquote"), s("@b"))]) + assert tokenize("a~ @b") == tokenize("a ~ @b") == [s("a"), e(s("unquote"), s("@b"))] def test_lex_expression_strings(): From 47a9a8780d76f82bd0d1609791df5706189d780d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 3 Dec 2022 11:01:20 -0500 Subject: [PATCH 056/342] Reserve `pragma` as a core macro name --- NEWS.rst | 2 ++ docs/api.rst | 6 ++++++ hy/core/result_macros.py | 6 ++++-- tests/compilers/test_ast.py | 5 +++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 95375059a..4f61d7c69 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -14,6 +14,8 @@ Breaking Changes the case for `'`.) * The bitwise NOT operator `~` has been renamed to `bnot`. +* `pragma` is now reserved as a core macro name, although it doesn't + do anything useful (yet). * `hy.cmdline.HyREPL` is now `hy.REPL`. Bug Fixes diff --git a/docs/api.rst b/docs/api.rst index 843508ac7..24c0b099f 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1419,6 +1419,12 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. coroutine, say, if using something fancy like `asyncio `_. +.. hy:function:: (pragma) + + ``pragma`` is reserved as a core macro name for future use, especially for + allowing backwards-compatible addition of new features after the release of Hy + 1.0. Currently, trying to use ``pragma`` is an error. + .. hy:automodule:: hy.core.macros :members: :macros: diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 540c07fcb..86bef3b59 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1895,8 +1895,10 @@ def compile_let(compiler, expr, root, bindings, body): @pattern_macro( - "unquote unquote-splice unpack-mapping except except* finally else".split(), + "pragma unquote unquote-splice unpack-mapping except except* finally else".split(), [many(FORM)], ) def compile_placeholder(compiler, expr, root, body): - raise ValueError(f"`{root}` is not allowed here") + raise ValueError("`{}` is not allowed {}".format( + root, + "in this version of Hy" if root == "pragma" else "here")) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index efa2882f0..1dafc7bc4 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -643,3 +643,8 @@ def test_module_prelude(): x = x[0].names[0] assert x.name == "hy" assert x.asname is None + + +def test_pragma(): + cant_compile("(pragma)") + cant_compile("(pragma :native-code :namespaced-symbols :give-user-a-pony)") From 5a00a0fde38d35ed53b2ee88bd31f23615fe6ad8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 6 Dec 2022 17:04:36 -0500 Subject: [PATCH 057/342] Autoformat --- hy/core/result_macros.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 86bef3b59..42dcab8a1 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1899,6 +1899,8 @@ def compile_let(compiler, expr, root, bindings, body): [many(FORM)], ) def compile_placeholder(compiler, expr, root, body): - raise ValueError("`{}` is not allowed {}".format( - root, - "in this version of Hy" if root == "pragma" else "here")) + raise ValueError( + "`{}` is not allowed {}".format( + root, "in this version of Hy" if root == "pragma" else "here" + ) + ) From d6f6a66f7587c58a23d3b86305a81a081c13edba Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 4 Dec 2022 14:17:42 -0500 Subject: [PATCH 058/342] In `syntax.rst`, show calling a macro-shadowed fn --- docs/syntax.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/syntax.rst b/docs/syntax.rst index 511f2a2da..b2ad201b1 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -357,10 +357,11 @@ Expressions (:class:`Expression `) are denoted by parentheses: ``( … )``. The compiler evaluates expressions by checking the first element. If it's a symbol, and the symbol is the name of a currently defined macro, the macro is called. Otherwise, the expression is compiled into -a Python-level call, with the first element being the calling object. The -remaining forms are understood as arguments. Use :hy:func:`unpack-iterable` or -:hy:func:`unpack-mapping` to break up data structures into individual arguments -at runtime. +a Python-level call, with the first element being the calling object. (So, you +can call a function that has the same name as a macro with an expression like +``((do setv) …)``). The remaining forms are understood as arguments. Use +:hy:func:`unpack-iterable` or :hy:func:`unpack-mapping` to break up data +structures into individual arguments at runtime. The empty expression ``()`` is legal at the reader level, but has no inherent meaning. Trying to compile it is an error. For the empty tuple, use ``#()``. From 571497592c58b1112ea2e017dd79cae59889af86 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 4 Dec 2022 15:39:47 -0500 Subject: [PATCH 059/342] Improve some documentation of the CLI and REPL --- docs/cli.rst | 58 +++++++++++++++++++-------------------------------- docs/repl.rst | 6 ++++-- hy/cmdline.py | 2 +- 3 files changed, 26 insertions(+), 40 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index e34393968..374be98c9 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1,57 +1,41 @@ ====================== -Command Line Interface +Command-Line Interface ====================== -.. _hy: +Hy provides a handful of command-line programs for working with Hy code. + +.. _hy-cli: hy -- -Command Line Options -^^^^^^^^^^^^^^^^^^^^ - -.. cmdoption:: -c - - Execute the Hy code in *command*. - - .. code-block:: bash - - $ hy -c "(print (+ 2 2))" - 4 - -.. cmdoption:: -i - - Execute the Hy code in *command*, then stay in REPL. +``hy`` is a command-line interface for Hy that in general imitates the program +``python`` provided by CPython. For example, ``hy`` without arguments launches +the :ref:`REPL `, whereas ``hy foo.hy a b`` runs the Hy program +``foo.hy`` with ``a`` and ``b`` as command-line arguments. See ``hy --help`` +for a complete list of options and :py:ref:`Python's documentation +` for many details. Here are some Hy-specific details: .. cmdoption:: -m - Execute the Hy code in *module*, including ``defmain`` if defined. - - The :option:`-m` flag terminates the options list so that - all arguments after the *module* name are passed to the module in - ``sys.argv``. + Much like Python's ``-m``, but the input module name will be :ref:`mangled + `. .. cmdoption:: --spy - Print equivalent Python code before executing in REPL. For example:: - - => (defn salutationsnm [name] (print (+ "Hy " name "!"))) - def salutationsnm(name): - return print('Hy ' + name + '!') - => (salutationsnm "YourName") - salutationsnm('YourName') - Hy YourName! - => + Print equivalent Python code before executing each piece of Hy code in the + REPL:: - `--spy` only works on REPL mode. + => (+ 1 2) + 1 + 2 + 3 .. cmdoption:: --repl-output-fn - Format REPL output using specific function (e.g., ``repr``) - -.. cmdoption:: -v - - Print the Hy version number and exit. + Set the :ref:`REPL output function `. This can be the + name of a Python builtin, mostly likely ``repr``, or a dotted name like + ``foo.bar.baz``. In the latter case, Hy will attempt to import the named + object with code like ``(import foo.bar [baz])``. .. _hy2py: diff --git a/docs/repl.rst b/docs/repl.rst index 3df91088d..838f169fd 100644 --- a/docs/repl.rst +++ b/docs/repl.rst @@ -1,3 +1,5 @@ +.. _repl: + =========== The Hy REPL =========== @@ -75,9 +77,9 @@ change the prompts. The following example shows a number of possibilities:: (setv repl-spy True repl-output-fn pformat - ;; We can even add colors to the prompts. - ;; This will set `=>` to green and `...` to red. + ;; Make the REPL prompt `=>` green. sys.ps1 "\x01\x1b[0;32m\x02=> \x01\x1b[0m\x02" + ;; Make the REPL prompt `...` red. sys.ps2 "\x01\x1b[0;31m\x02... \x01\x1b[0m\x02") (defn slurp [path] diff --git a/hy/cmdline.py b/hy/cmdline.py index 965d7de6d..f6093b2e1 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -75,7 +75,7 @@ def run_icommand(source, **kwargs): - program read from stdin [ARG]... - arguments passed to program in sys.argv[1:] + arguments passed to program in (cut sys.argv 1) """ From da0d9f9e8edfb148ab38b3be254c40c59b853abc Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 4 Dec 2022 16:13:38 -0500 Subject: [PATCH 060/342] Clean up the interop chapter of the manual --- docs/interop.rst | 130 ++++++++++++--------------------------------- docs/semantics.rst | 2 + 2 files changed, 35 insertions(+), 97 deletions(-) diff --git a/docs/interop.rst b/docs/interop.rst index 34a68b26f..99f7e38c2 100644 --- a/docs/interop.rst +++ b/docs/interop.rst @@ -1,8 +1,8 @@ .. _interop: -===================== -Hy <-> Python interop -===================== +======================= +Python Interoperability +======================= Despite being a Lisp, Hy aims to be fully compatible with Python. That means every Python module or package can be imported in Hy code, and vice versa. @@ -10,121 +10,57 @@ every Python module or package can be imported in Hy code, and vice versa. :ref:`Mangling ` allows variable names to be spelled differently in Hy and Python. For example, Python's ``str.format_map`` can be written ``str.format-map`` in Hy, and a Hy function named ``valid?`` would be called -``is_valid`` in Python. In Python, you can import Hy's core functions -``mangle`` and ``unmangle`` directly from the ``hy`` package. +``is_valid`` in Python. You can call :hy:func:`hy.mangle` and +:hy:func:`hy.unmangle` from either language. Using Python from Hy ==================== -You can embed Python code directly into a Hy program with the special operators -:hy:func:`py ` and :hy:func:`pys `. +To use a Python module from Hy, just :hy:func:`import` it. No additional +ceremony is required. -Using a Python module from Hy is nice and easy: you just have to :ref:`import` -it. If you have the following in ``greetings.py`` in Python: - -.. code-block:: python - - def greet(name): - print("hello," name) - -You can use it in Hy:: - - (import greetings) - (.greet greetings "foo") ; prints "hello, foo" - -You can also import ``.pyc`` bytecode files, of course. +You can embed Python code directly into a Hy program with the macros +:hy:func:`py ` and :hy:func:`pys `, and you can use standard Python +tools like :func:`eval` or :func:`exec` to execute or manipulate Python code in +strings. Using Hy from Python ==================== -Suppose you have written some useful utilities in Hy, and you want to use them in -regular Python, or to share them with others as a package. Or suppose you work -with somebody else, who doesn't like Hy (!), and only uses Python. - -In any case, you need to know how to use Hy from Python. Fear not, for it is -easy. - -If you save the following in ``greetings.hy``:: - - (setv this-will-have-underscores "See?") - (defn greet [name] (print "Hello from Hy," name)) - -Then you can use it directly from Python, by importing Hy before importing -the module. In Python: - -.. code-block:: python - - import hy - import greetings - - greetings.greet("Foo") # prints "Hello from Hy, Foo" - print(greetings.this_will_have_underscores) # prints "See?" - -If you create a package with Hy code, and you do the ``import hy`` in -``__init__.py``, you can then directly include the package. Of course, Hy still -has to be installed. - -Compiled files --------------- +To use a Hy module from Python, you can just :py:keyword:`import` it, provided +that ``hy`` has already been imported first, whether in the current module or +in some earlier module executed by the current Python process. The ``hy`` +import is necessary to create the hooks that allow importing Hy modules. Note +that you can always have a wrapper Python file (such as a package's +``__init__.py``) do the ``import hy`` for the user; this is a smart thing to do +for a published package. -You can also compile a module with ``hyc``, which gives you a ``.pyc`` file. You -can import that file. Hy does not *really* need to be installed ; however, if in -your code, you use any symbol from :doc:`/api`, a corresponding ``import`` -statement will be generated, and Hy will have to be installed. +No way to import macros or reader macros into a Python module is implemented, +since there's no way to call them in Python anyway. -Even if you do not use a Hy builtin, but just another function or variable with -the name of a Hy builtin, the ``import`` will be generated. For example, the previous code -causes the import of ``name`` from ``hy.core.language``. +You can use :ref:`hy2py` to convert a Hy program to Python. The output will +still import ``hy``, and thus require Hy to be installed in order to run; see +:ref:`implicit-names` for details and workarounds. -**Bottom line: in most cases, Hy has to be installed.** - -.. _repl-from-py: - -Launching a Hy REPL from Python -------------------------------- - -You can use :meth:`hy.REPL.run` to launch the Hy REPL from Python: - -.. code-block:: text - - >>> import hy - >>> hy.REPL(locals = locals()).run() - Hy x.y.z using CPython(default) x.y.z on Linux - => (defn test [] (print "bar")) - => (test) - bar - -Evaluating strings of Hy code from Python ------------------------------------------ - -Evaluating a string (or ``file`` object) containing a Hy expression requires -two separate steps. First, use the ``read`` function to turn the expression -into a Hy model: - -.. code-block:: python - - >>> import hy - >>> expr = hy.read("(- (/ (+ 1 3 88) 2) 8)") - -Then, use the ``hy.eval`` function to evaluate it: - -.. code-block:: python - - >>> hy.eval(expr) - 38.0 +To execute Hy code from a string, use :func:`hy.read` to convert it to +:ref:`models ` and then :func:`hy.eval` to evaluate it. There is no Hy +equivalent of :func:`exec` because :func:`hy.eval` works even when the input +isn't equivalent to a single Python expression. +You can use :meth:`hy.REPL.run` to launch the Hy REPL from Python, as in +``hy.REPL(locals = locals()).run()``. Libraries that expect Python ============================ There are various means by which Hy may interact poorly with a Python library + because the library doesn't account for the possibility of Hy. For example, -when you run the command-line program ``hy``, ``sys.executable`` will be set to +when you run :ref:`hy-cli`, ``sys.executable`` will be set to this program rather than the original Python binary. This is helpful more often than not, but will lead to trouble if e.g. the library tries to call :py:data:`sys.executable` with the ``-c`` option. In this case, you can try setting :py:data:`sys.executable` back to ``hy.sys-executable``, which is a -saved copy of the original value. More generally, you can use ``hy2py``, or you +saved copy of the original value. More generally, you can use :ref:`hy2py`, or you can put a simple Python wrapper script like ``import hy, my_hy_program`` in -front of your code; importing ``hy`` first is necessary here to install the -hooks that allow Python to load your Hy module. +front of your code. diff --git a/docs/semantics.rst b/docs/semantics.rst index 799395ccd..f47cca68b 100644 --- a/docs/semantics.rst +++ b/docs/semantics.rst @@ -5,6 +5,8 @@ Semantics This chapter describes features of Hy semantics that differ from Python's and aren't better categorized elsewhere, such as in the chapter :doc:`macros`. +.. _implicit-names: + Implicit names -------------- From 8e8521bf270417bec3ec3b27427257418120872c Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 4 Dec 2022 16:58:37 -0500 Subject: [PATCH 061/342] Minor edits to the tutorial --- docs/tutorial.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 03f25c76c..314ca4b95 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -77,8 +77,7 @@ comment is functionally equivalent to whitespace. :: (setv password "susan") ; My daughter's name Although ``#`` isn't a comment character in Hy, a Hy program can begin with a -`shebang line `_, which Hy itself -will ignore:: +:ref:`shebang line `, which Hy itself will ignore:: #!/usr/bin/env hy (print "Make me executable, and run me!") @@ -303,12 +302,12 @@ simply:: Our macro ``m`` has an especially simple return value, an integer, which at compile-time is converted to an integer literal. In general, macros can return -arbitrary Hy forms to be executed as code. There are several special operators -and macros that make it easy to construct forms programmatically, such as -:hy:func:`quote` (``'``), :hy:func:`quasiquote` (`````), :hy:func:`unquote` -(``~``), and :hy:func:`defmacro! `. The previous -chapter has :ref:`a simple example ` of using ````` and ``~`` to -define a new control construct ``do-while``. +arbitrary Hy forms to be executed as code. There are several helper macros that +make it easy to construct forms programmatically, such as :hy:func:`quote` +(``'``), :hy:func:`quasiquote` (`````), :hy:func:`unquote` (``~``), and +:hy:func:`defmacro! `. The previous chapter has +:ref:`a simple example ` of using ````` and ``~`` to define a new +control construct ``do-while``. What if you want to use a macro that's defined in a different module? ``import`` won't help, because it merely translates to a Python ``import`` From b605654a48981b0e030362b0ef0d43053910555e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 5 Dec 2022 09:14:42 -0500 Subject: [PATCH 062/342] Improve the documentation of some core macros --- docs/api.rst | 142 +++++++++++++++++---------------------------------- 1 file changed, 48 insertions(+), 94 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 24c0b099f..6e23c492e 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -5,7 +5,7 @@ API Core Macros ----------- -The following macros are auto imported into all Hy modules as their +The following macros are automatically imported into all Hy modules as their base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (annotate [value type]) @@ -105,16 +105,16 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. parameters (also given as symbols). Any further arguments constitute the body of the function:: - (defn name [params] bodyform1 bodyform2...) + (defn name [params] bodyform1 bodyform2…) - An empty body is implicitly ``(return None)``. If there at least two body + An empty body is implicitly ``(return None)``. If there are at least two body forms, and the first of them is a string literal, this string becomes the :term:`py:docstring` of the function. The final body form is implicitly returned; thus, ``(defn f [] 5)`` is equivalent to ``(defn f [] (return 5))``. ``defn`` accepts two additional, optional arguments: a bracketed list of - :term:`decorators ` and an annotation (see :hy:data:`^`) for + :term:`decorators ` and an annotation (see :hy:func:`annotate`) for the return value. These are placed before the function name (in that order, if both are present):: @@ -222,21 +222,14 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (await [obj]) ``await`` creates an :ref:`await expression `. It takes exactly one - argument: the object to wait for. + argument: the object to wait for. :: - - :strong:`Examples` - - :: - - => (import asyncio) - => (defn/a main [] - ... (print "hello") - ... (await (asyncio.sleep 1)) - ... (print "world")) - => (asyncio.run (main)) - hello - world + (import asyncio) + (defn/a main [] + (print "hello") + (await (asyncio.sleep 1)) + (print "world")) + (asyncio.run (main)) .. hy:function:: break @@ -252,8 +245,10 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. In a loop with multiple iteration clauses, such as ``(for [x xs y ys] …)``, ``break`` only breaks out of the innermost iteration, not the whole form. To jump out of the whole form, enclose it in a :hy:func:`block - ` and use ``block-ret`` instead of ``break``, or - enclose it in a function and use :hy:func:`return`. + ` and use ``block-ret`` instead of ``break``. In + the case of :hy:func:`for`, but not :hy:func:`lfor` and the other + comprehension forms, you may also enclose it in a function and use + :hy:func:`return`. .. hy:function:: (chainc [#* args]) @@ -312,20 +307,16 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. ``do`` (called ``progn`` in some Lisps) takes any number of forms, evaluates them, and returns the value of the last one, or ``None`` if no - forms were provided. - - :strong:`Examples` + forms were provided. :: - :: - - => (+ 1 (do (setv x (+ 1 1)) x)) - 3 + (+ 1 (do (setv x (+ 1 1)) x)) ; => 3 .. hy:function:: (for [#* args]) - ``for`` is used to evaluate some forms for each element in an iterable - object, such as a list. The return values of the forms are discarded and - the ``for`` form returns ``None``. + ``for`` compiles to one or more :py:keyword:`for` statements, which + execute code repeatedly for each element of an iterable object. + The return values of the forms are discarded and the ``for`` form + returns ``None``. :: @@ -339,10 +330,10 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. iterating 3 - In its square-bracketed first argument, ``for`` allows the same types of - clauses as :hy:func:`lfor`. + The first argument of ``for``, in square brackets, specifies how to loop. A simple and common case is ``[variable values]``, where ``values`` is a form that evaluates to an iterable object (such as a list) and ``variable`` is a symbol specifiying the name to assign each element to. Subsequent arguments to ``for`` are body forms to be evaluated for each iteration of the loop. - :: + More generally, the first argument of ``for`` allows the same types of + clauses as :hy:func:`lfor`:: => (for [x [1 2 3] :if (!= x 2) y [7 8]] ... (print x y)) @@ -351,7 +342,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. 3 7 3 8 - Furthermore, the last argument of ``for`` can be an ``(else …)`` form. + The last argument of ``for`` can be an ``(else …)`` form. This form is executed after the last iteration of the ``for``\'s outermost iteration clause, but only if that outermost loop terminates normally. If it's jumped out of with e.g. ``break``, the ``else`` is @@ -377,49 +368,32 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (assert [condition [label None]]) - ``assert`` is used to verify conditions while the program is - running. If the condition is not met, an :exc:`AssertionError` is - raised. ``assert`` may take one or two parameters. The first - parameter is the condition to check, and it should evaluate to either - ``True`` or ``False``. The second parameter, optional, is a label for - the assert, and is the string that will be raised with the - :exc:`AssertionError`. For example: - - :strong:`Examples` - - :: - - (assert (= variable expected-value)) - - (assert False) - ; AssertionError + ``assert`` compiles to an :py:keyword:`assert` statement, which checks + whether a condition is true. The first argument, specifying the condition to + check, is mandatory, whereas the second, which will be passed to + :py:class:`AssertionError`, is optional. The whole form is only evaluated + when :py:data:`__debug__` is true, and the second argument is only evaluated + when :py:data:`__debug__` is true and the condition fails. ``assert`` always + returns ``None``. :: (assert (= 1 2) "one should equal two") - ; AssertionError: one should equal two - -.. hy:function:: (global [sym]) - - ``global`` can be used to mark a symbol as global. This allows the programmer to - assign a value to a global symbol. Reading a global symbol does not require the - ``global`` keyword -- only assigning it does. + ; AssertionError: one should equal two - The following example shows how the global symbol ``a`` is assigned a value in a - function and is later on printed in another function. Without the ``global`` - keyword, the second function would have raised a ``NameError``. +.. hy:function:: (global [sym #* syms]) - :strong:`Examples` - - :: + ``global`` compiles to a :py:keyword:`global` statement, which declares one + or more names as referring to global (i.e., module-level) variables. The + arguments are symbols; at least one is required. The return value is always + ``None``. :: - (defn set-a [value] + (setv a 1 b 10) + (print a b) ; => 1 10 + (defn f [] (global a) - (setv a value)) - - (defn print-a [] - (print a)) + (setv a 2 b 20)) + (f) + (print a b) ; => 2 10 - (set-a 5) - (print-a) .. hy:function:: (get [coll key1 #* keys]) @@ -839,29 +813,9 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. => dic {} -.. hy:function:: (nonlocal [object]) - - ``nonlocal`` can be used to mark a symbol as not local to the current scope. - The parameters are the names of symbols to mark as nonlocal. This is necessary - to modify variables through nested ``fn`` scopes: - - :strong:`Examples` - - :: - - (defn some-function [] - (setv x 0) - (register-some-callback - (fn [stuff] - (nonlocal x) - (setv x stuff)))) - - Without the call to ``(nonlocal x)``, the inner function would redefine ``x`` to - ``stuff`` inside its local scope instead of overwriting the ``x`` in the outer - function. +.. hy:function:: (nonlocal [sym #* syms]) - See `PEP3104 `_ for further - information. + As :hy:func:`global`, but the result is a :py:keyword:`nonlocal` statement. .. hy:function:: (py [string]) @@ -876,8 +830,8 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. The code must be given as a single string literal, but you can still use macros, :hy:func:`hy.eval `, and related tools to construct the ``py`` form. If having to backslash-escape internal double quotes is getting you down, try a - :ref:`bracket string `. If you want to evaluate some - Python code that's only defined at run-time, try the standard Python function + :ref:`bracket string `. If you want to evaluate some Python + code that's only defined at run-time, try the standard Python function :func:`eval`. Python code need not syntactically round-trip if you use ``hy2py`` on a Hy From b1918fc52d6aaba130a24b777da9299ca056c2ca Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 10:55:32 -0500 Subject: [PATCH 063/342] Refer to the Python tutorial from our tutorial --- docs/tutorial.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 314ca4b95..b38f0271e 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -364,9 +364,14 @@ Next steps You now know enough to be dangerous with Hy. You may now smile villainously and sneak off to your Hydeaway to do unspeakable things. -Refer to Python's documentation for the details of Python semantics, and the -rest of this manual for Hy-specific features. Like Hy itself, the manual is -incomplete, but :ref:`contributions ` are always welcome. +Refer to Python's documentation for the details of Python semantics. In +particular, :ref:`the Python tutorial ` can be helpful even if +you have no interest in writing your own Python code, because it will introduce +you to the semantics, and you'll need a reading knowledge of Python syntax to +understand example code for Python libraries. + +Refer to the rest of this manual for Hy-specific features. Like Hy itself, the +manual is incomplete, but :ref:`contributions ` are always welcome. Bear in mind that Hy is still unstable, and with each release along the way to Hy 1.0, there are new breaking changes. Refer to `the NEWS file From aee03728c233105a09d0cd1b18290bf90c11d407 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 9 Dec 2022 14:51:23 -0500 Subject: [PATCH 064/342] Remove the duplicate documentation of `get` --- docs/api.rst | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 6e23c492e..eef1b7b81 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -395,36 +395,6 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print a b) ; => 2 10 -.. hy:function:: (get [coll key1 #* keys]) - - ``get`` is used to access single elements in collections. ``get`` takes at - least two parameters: the *data structure* and the *index* or *key* of the - item. It will then return the corresponding value from the collection. If - multiple *index* or *key* values are provided, they are used to access - successive elements in a nested structure. Example usage: - - :strong:`Examples` - - :: - - => (do - ... (setv animals {"dog" "bark" "cat" "meow"} - ... numbers #("zero" "one" "two" "three") - ... nested [0 1 ["a" "b" "c"] 3 4]) - ... (print (get animals "dog")) - ... (print (get numbers 2)) - ... (print (get nested 2 1))) - - bark - two - b - - .. note:: ``get`` raises a KeyError if a dictionary is queried for a - non-existing key. - - .. note:: ``get`` raises an IndexError if a list or a tuple is queried for an - index that is out of bounds. - .. hy:function:: (import [#* forms]) ``import`` compiles to an :py:keyword:`import` statement, which makes objects From ab91cf0edf4e2e48a0ca14c3da60e4fa950c3280 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 09:01:15 -0500 Subject: [PATCH 065/342] Rewrite the docstring of `get` --- hy/pyops.hy | 49 +++++++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/hy/pyops.hy b/hy/pyops.hy index 339ec6f4b..0b09129d9 100644 --- a/hy/pyops.hy +++ b/hy/pyops.hy @@ -207,34 +207,27 @@ evaluate all arguments." (not x)) (defn get [coll key1 #* keys] - "Access item in `coll` indexed by `key1`, with optional `keys` nested-access. - - ``get`` is used to access single elements in collections. ``get`` takes at - least two parameters: the *data structure* and the *index* or *key* of the - item. It will then return the corresponding value from the collection. If - multiple *index* or *key* values are provided, they are used to access - successive elements in a nested structure. - - .. note:: ``get`` raises a KeyError if a dictionary is queried for a - non-existing key. - - .. note:: ``get`` raises an IndexError if a list or a tuple is queried for an - index that is out of bounds. - - Examples: - :: - - => (do - ... (setv animals {\"dog\" \"bark\" \"cat\" \"meow\"} - ... numbers #(\"zero\" \"one\" \"two\" \"three\") - ... nested [0 1 [\"a\" \"b\" \"c\"] 3 4]) - ... (print (get animals \"dog\")) - ... (print (get numbers 2)) - ... (print (get nested 2 1)) - bark - two - b - " + #[[``get`` compiles to one or more :ref:`subscription expressions `, + which select an element of a data structure. The first two arguments are the + collection object and a key; for example, ``(get person name)`` compiles to + ``person[name]``. Subsequent arguments indicate chained subscripts, so ``(get person + name "surname" 0)`` becomes ``person[name]["surname"][0]``. You can assign to a + ``get`` form, as in :: + + (setv real-estate {"price" 1,500,000}) + (setv (get real-estate "price") 0) + + but this doesn't work with the function version of ``get`` from ``hy.pyops``, due to + Python limitations on lvalues. + + If you're looking for the Hy equivalent of Python list slicing, as in ``foo[1:3]``, + note that this is just Python's syntactic sugar for ``foo[slice(1, 3)]``, and Hy + provides its own syntactic sugar for this with a different macro, :hy:func:`cut`. + + Note that ``.`` (:ref:`dot `) forms can also subscript. See also Hyrule's + :hy:func:`assoc ` to easily assign multiple elements of a + single collection.]] + (setv coll (get coll key1)) (for [k keys] (setv coll (get coll k))) From bf71d7294fa290c2485603d0e881424af2a1d61f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 09:00:00 -0500 Subject: [PATCH 066/342] Test the return values of `eval-foo-compile` --- tests/native_tests/language.hy | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index a88ff672a..f4194f253 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -1775,3 +1775,18 @@ cee"} dee" "ey bee\ncee dee")) (defclass Quest [QuestBase :swallow "african"]) (assert (= (. (Quest) swallow) "african"))) + +(defn test-eval-foo-compile-return-values [] + (eval-and-compile (setv jim 0)) + + (setv derrick (eval-and-compile (+= jim 1) 2)) + (assert (= jim 1)) + (assert (= derrick 2)) + + (setv derrick (eval-and-compile)) + (assert (is derrick None)) + + (setv derrick 3) + (setv derrick (eval-when-compile (+= jim 1) 2)) + (assert (= jim 1)) + (assert (is derrick None))) From b97f5e6eee926da195cb482540a51cecf0d90f16 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 11:30:45 -0500 Subject: [PATCH 067/342] Edit the documentation of `eval-foo-compile` --- docs/api.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index eef1b7b81..b83f9049f 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -435,7 +435,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (eval-and-compile [#* body]) - ``eval-and-compile`` is a special form that takes any number of forms. The input forms are evaluated as soon as the ``eval-and-compile`` form is compiled, instead of being deferred until run-time. The input forms are also left in the program so they can be executed at run-time as usual. So, if you compile and immediately execute a program (as calling ``hy foo.hy`` does when ``foo.hy`` doesn't have an up-to-date byte-compiled version), ``eval-and-compile`` forms will be evaluated twice. + ``eval-and-compile`` takes any number of forms as arguments. The input forms are evaluated as soon as the ``eval-and-compile`` form is compiled, instead of being deferred until run-time. The input forms are also left in the program so they can be executed at run-time as usual. So, if you compile and immediately execute a program (as calling ``hy foo.hy`` does when ``foo.hy`` doesn't have an up-to-date byte-compiled version), ``eval-and-compile`` forms will be evaluated twice. The return value is the final argument, as in ``do``. One possible use of ``eval-and-compile`` is to make a function available both at compile-time (so a macro can call it while expanding) and run-time (so it can be called like any other function):: @@ -453,11 +453,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (eval-when-compile [#* body]) - ``eval-when-compile`` is like ``eval-and-compile``, but the code isn't executed at run-time. Hence, ``eval-when-compile`` doesn't directly contribute any code to the final program, although it can still change Hy's state while compiling (e.g., by defining a function). - - :strong:`Examples` - - :: + As ``eval-and-compile``, but the code isn't executed at run-time, and ``None`` is returned. Hence, ``eval-when-compile`` doesn't directly contribute any code to the final program, although it can still change Hy's state while compiling (e.g., by defining a function). :: (eval-when-compile (defn add [x y] From 570078c3796557aca6d339245e8dac584eb36b63 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 09:14:54 -0500 Subject: [PATCH 068/342] Edit the documentation of comprehension macros --- docs/api.rst | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index b83f9049f..cd55ad0f2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -470,10 +470,9 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. The comprehension forms ``lfor``, :hy:func:`sfor`, :hy:func:`dfor`, :hy:func:`gfor`, and :hy:func:`for` are used to produce various kinds of loops, including Python-style :ref:`comprehensions `. ``lfor`` in particular - creates a list comprehension. A simple use of ``lfor`` is:: + can create a list comprehension. A simple use of ``lfor`` is:: - => (lfor x (range 5) (* 2 x)) - [0 2 4 6 8] + (lfor x (range 5) (* 2 x)) ; => [0 2 4 6 8] ``x`` is the name of a new variable, which is bound to each element of ``(range 5)``. Each such element in turn is used to evaluate the value @@ -481,13 +480,13 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Here's a more complex example:: - => (lfor - ... x (range 3) - ... y (range 3) - ... :if (!= x y) - ... :setv total (+ x y) - ... [x y total]) - [[0 1 1] [0 2 2] [1 0 1] [1 2 3] [2 0 2] [2 1 3]] + (lfor + x (range 3) + y (range 3) + :if (!= x y) + :setv total (+ x y) + [x y total]) + ; => [[0 1 1] [0 2 2] [1 0 1] [1 2 3] [2 0 2] [2 1 3]] When there are several iteration clauses (here, the pairs of forms ``x (range 3)`` and ``y (range 3)``), the result works like a nested loop or @@ -517,9 +516,9 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. For ``lfor``, ``sfor``, ``gfor``, and ``dfor``, variables defined by an iteration clause or ``:setv`` are not visible outside the form. - However, variables defined within the body, such as via a ``setx`` + However, variables defined within the body, as with a ``setx`` expression, will be visible outside the form. - By contrast, iteration and ``:setv`` clauses for ``for`` share the + In ``for``, by contrast, iteration and ``:setv`` clauses share the caller's scope and are visible outside the form. .. hy:function:: (dfor [#* args]) @@ -537,11 +536,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. ``gfor`` creates a :ref:`generator expression `. Its syntax is the same as that of :hy:func:`lfor`. The difference is that ``gfor`` returns - an iterator, which evaluates and yields values one at a time. - - :strong:`Examples` - - :: + an iterator, which evaluates and yields values one at a time:: => (import itertools [count take-while]) => (setv accum []) @@ -554,7 +549,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (sfor [#* args]) - ``sfor`` creates a set comprehension. ``(sfor CLAUSES VALUE)`` is + ``sfor`` creates a :ref:`set comprehension `. ``(sfor CLAUSES VALUE)`` is equivalent to ``(set (lfor CLAUSES VALUE))``. See :hy:func:`lfor`. .. hy:function:: (setv [#* args]) From 7271c6bd04eeeb8c0c0bc9d80daabcf3be8d5bc9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 10:10:21 -0500 Subject: [PATCH 069/342] Edit the documentation of `setv` and `setx` --- docs/api.rst | 51 ++++++++++++++++++--------------------------------- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index cd55ad0f2..d5f39b514 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -554,50 +554,35 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (setv [#* args]) - ``setv`` is used to bind a value, object, or function to a symbol. + ``setv`` compiles to an :ref:`assignment statement ` (see :hy:func:`setx` for assignment expressions), which sets the value of a variable or some other assignable expression. It requires an even number of arguments, and always returns ``None``. The most common case is two arguments, where the first is a symbol:: - :strong:`Examples` - - :: - - => (setv names ["Alice" "Bob" "Charlie"]) - => (print names) - ['Alice', 'Bob', 'Charlie'] - - => (setv counter (fn [collection item] (.count collection item))) - => (counter [1 2 3 4 5 2 3] 2) - 2 - - You can provide more than one target–value pair, and the assignments will be made in order:: + (setv websites 103) + (print websites) ; => 103 - => (setv x 1 y x x 2) - => (print x y) - 2 1 + Additional pairs of arguments are equivalent to several two-argument ``setv`` calls, in the given order. Thus, the semantics are like Common Lisp's ``setf`` rather than ``psetf``. :: - You can perform parallel assignments or unpack the source value with square brackets and :hy:func:`unpack-iterable `:: + (setv x 1 y x x 2) + (print x y) ; => 2 1 - => (setv duo ["tim" "eric"]) - => (setv [guy1 guy2] duo) - => (print guy1 guy2) - tim eric + All the same kinds of complex assignment targets are allowed as in Python. So, you can use list assignment to assign in parallel. (As in Python, tuple and list syntax are equivalent for this purpose; Hy differs from Python merely in that its list syntax is shorter than its tuple syntax.) :: - => (setv [letter1 letter2 #* others] "abcdefg") - => (print letter1 letter2 others) - a b ['c', 'd', 'e', 'f', 'g'] + (setv [x y] [y x]) ; Swaps the values of `x` and `y` + Unpacking assignment looks like this (see :hy:func:`unpack-iterable`):: -.. hy:function:: (setx [#* args]) + (setv [letter1 letter2 #* others] "abcdefg") + (print letter1 letter2 (hy.repr others)) + ; => a b ["c" "d" "e" "f" "g"] - Whereas ``setv`` creates an assignment statement, ``setx`` creates an assignment expression (see :pep:`572`). It requires Python 3.8 or later. Only one target–value pair is allowed, and the target must be a bare symbol, but the ``setx`` form returns the assigned value instead of ``None``. + See :hy:func:`let` to simulate more traditionally Lispy block-level scoping. - :strong:`Examples` - - :: +.. hy:function:: (setx [target value]) - => (when (> (setx x (+ 1 2)) 0) - ... (print x "is greater than 0")) - 3 is greater than 0 + ``setx`` compiles to an assignment expression. Thus, unlike :hy:func:`setv`, it returns the assigned value. It takes exactly two arguments, and the target must be a bare symbol. Python 3.8 or later is required. :: + (when (> (setx x (+ 1 2)) 0) + (print x "is greater than 0")) + ; => 3 is greater than 0 .. hy:function:: (let [bindings #* body]) From a3de4095e1385033dba67d806ced9b615fe1e9e6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 10:27:51 -0500 Subject: [PATCH 070/342] Edit the documentation of `match` --- docs/api.rst | 51 +++++++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index d5f39b514..a9193681c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -656,13 +656,13 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. requires Python 3.10 or later. The first argument should be the subject, and any remaining arguments should be pairs of patterns and results. The ``match`` form returns the value of the corresponding result, or - ``None`` if no case matched. For example:: + ``None`` if no case matched. :: - => (match (+ 1 1) - ... 1 "one" - ... 2 "two" - ... 3 "three") - "two" + (match (+ 1 1) + 1 "one" + 2 "two" + 3 "three") + ; => "two" You can use :hy:func:`do` to build a complex result form. Patterns, as in Python match statements, are interpreted specially and can't be @@ -671,26 +671,33 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. value, sequence, mapping, and class patterns. Guards are specified with ``:if FORM``. Here's a more complex example:: - => (match #(100 200) - ... [100 300] "Case 1" - ... [100 200] :if flag "Case 2" - ... [900 y] f"Case 3, y: {y}" - ... [100 (| 100 200) :as y] f"Case 4, y: {y}" - ... _ "Case 5, I match anything!") + (match #(100 200) + [100 300] "Case 1" + [100 200] :if flag "Case 2" + [900 y] f"Case 3, y: {y}" + [100 (| 100 200) :as y] f"Case 4, y: {y}" + _ "Case 5, I match anything!") This will match case 2 if ``flag`` is true and case 4 otherwise. ``match`` can also match against class instances by keyword (or - positionally if its ``__match_args__`` attribute is defined, see - `pep 636 `_):: - - => (import dataclasses [dataclass]) - => (defclass [dataclass] Point [] - ... #^int x - ... #^int y) - => (match (Point 1 2) - ... (Point 1 x) :if (= (% x 2) 0) x) - 2 + positionally if its ``__match_args__`` attribute is defined; see :pep:`636`):: + + (import dataclasses [dataclass]) + (defclass [dataclass] Point [] + #^int x + #^int y) + (match (Point 1 2) + (Point 1 x) :if (= (% x 2) 0) x) ; => 2 + + It's worth emphasizing that ``match`` is a pattern-matching construct + rather than a generic `switch + `_ construct, and + retains all of Python's limitations on match patterns. For example, you + can't match against the value of a variable. For more flexible branching + constructs, see Hyrule's :hy:func:`branch ` and + :hy:func:`case `, or simply use :hy:func:`cond + `. .. hy:function:: (defclass [class-name super-classes #* body]) From 6a0132169663bb06ab5ac012514887bca3f8b00f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 10:55:47 -0500 Subject: [PATCH 071/342] Rewrite the documentation of `defclass` --- docs/api.rst | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index a9193681c..680b3e3c9 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -699,40 +699,28 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. :hy:func:`case `, or simply use :hy:func:`cond `. -.. hy:function:: (defclass [class-name super-classes #* body]) +.. hy:function:: (defclass [arg1 #* args]) - New classes are declared with ``defclass``. It can take optional parameters - in the following order: a list defining (a) possible super class(es) and a - string (:term:`py:docstring`). The class name may also be preceded by a list - of :term:`decorators `, as in :hy:func:`defn`. + ``defclass`` compiles to a :py:keyword:`class` statement, which creates a + new class. It always returns ``None``. Only one argument, specifying the + name of the new class as a symbol, is required. A list of :term:`decorators + ` may be provided before the class name. After the name comes + a list of superclasses (use the empty list ``[]`` for the typical case of no + superclasses) and any number of body forms, the first of which may be a + :term:`py:docstring`. :: - :strong:`Examples` - - :: + (defclass [decorator1 decorator2] MyClass [SuperClass1 SuperClass2] + "A class that does things at times." - => (defclass class-name [super-class-1 super-class-2] - ... "docstring" - ... - ... (setv attribute1 value1) - ... (setv attribute2 value2) - ... - ... (defn method [self] (print "hello!"))) - - Both values and functions can be bound on the new class as shown by the example - below: - - :: + (setv + attribute1 value1 + attribute2 value2) - => (defclass Cat [] - ... (setv age None) - ... (setv colour "white") - ... - ... (defn speak [self] (print "Meow"))) + (defn method1 [self arg1 arg2] + …) - => (setv spot (Cat)) - => (setv spot.colour "Black") - => (.speak spot) - Meow + (defn method2 [self arg1 arg2] + …)) .. hy:function:: (del [object]) From 79862dc8cdba19cf97e670911bc9d229388b0089 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 11:08:53 -0500 Subject: [PATCH 072/342] Rewrite the documentation of `del` --- docs/api.rst | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 680b3e3c9..fc71531e4 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -722,37 +722,12 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (defn method2 [self arg1 arg2] …)) -.. hy:function:: (del [object]) +.. hy:function:: (del [#* args]) - ``del`` removes an object from the current namespace. + ``del`` compiles to a :py:keyword:`del` statement, which deletes variables + or other assignable expressions. It always returns ``None``. :: - :strong:`Examples` - - :: - - => (setv foo 42) - => (del foo) - => foo - Traceback (most recent call last): - File "", line 1, in - NameError: name 'foo' is not defined - - ``del`` can also remove objects from mappings, lists, and more. - - :: - - => (setv test (list (range 10))) - => test - [0 1 2 3 4 5 6 7 8 9] - => (del (cut test 2 4)) ;; remove items from 2 to 4 excluded - => test - [0 1 4 5 6 7 8 9] - => (setv dic {"foo" "bar"}) - => dic - {"foo" "bar"} - => (del (get dic "foo")) - => dic - {} + (del foo (get mydict "mykey") myobj.myattr) .. hy:function:: (nonlocal [sym #* syms]) From b58c810e6d83d864b978b7561b027080efc6cb0d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 11:09:10 -0500 Subject: [PATCH 073/342] Edit the documentation of `pys` --- docs/api.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index fc71531e4..8b4f60118 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -760,14 +760,16 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. As :hy:func:`py `, but the code can consist of zero or more statements, including compound statements such as ``for`` and ``def``. ``pys`` always - returns ``None``. Also, the code string is dedented with - :func:`textwrap.dedent` before parsing, which allows you to intend the code to - match the surrounding Hy code, but significant leading whitespace in embedded - string literals will be removed. :: + returns ``None``. :: (pys "myvar = 5") (print "myvar is" myvar) + The code string is dedented with :func:`textwrap.dedent` before parsing, + which allows you to indent the code to match the surrounding Hy code when + Python would otherwise forbid this, but beware that significant leading + whitespace in embedded string literals will be removed. + .. hy:function:: (quasiquote [form]) ``quasiquote`` allows you to quote a form, but also selectively evaluate From 815e8d521a34f39c00a4c1989ce387a0b912a6f5 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 11:17:39 -0500 Subject: [PATCH 074/342] Edit the documentation of `import` --- docs/api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 8b4f60118..edc205b9d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -398,8 +398,8 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (import [#* forms]) ``import`` compiles to an :py:keyword:`import` statement, which makes objects - in a different module available in the current module. Hy's syntax for the - various kinds of import looks like this:: + in a different module available in the current module. It always returns + ``None``. Hy's syntax for the various kinds of import looks like this:: ;; Import each of these modules ;; Python: import sys, os.path From b09db0f3287ae306952e26926775827a5b882c4d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 10 Dec 2022 11:25:49 -0500 Subject: [PATCH 075/342] Edit the documentation of `return` --- docs/api.rst | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index edc205b9d..e952ec1a1 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -936,35 +936,26 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (return [object]) ``return`` compiles to a :py:keyword:`return` statement. It exits the - current function, returning its argument if provided with one or - ``None`` if not. + current function, returning its argument if provided with one, or + ``None`` if not. :: - :strong:`Examples` - - :: - - => (defn f [x] (for [n (range 10)] (when (> n x) (return n)))) - => (f 3.9) - 4 + (defn f [x] + (for [n (range 10)] + (when (> n x) + (return n)))) + (f 3.9) ; => 4 Note that in Hy, ``return`` is necessary much less often than in Python, since the last form of a function is returned automatically. Hence, an - explicit ``return`` is only necessary to exit a function early. - - :: - - => (defn f [x] (setv y 10) (+ x y)) - => (f 4) - 14 - - To get Python's behavior of returning ``None`` when execution reaches - the end of a function, put ``None`` there yourself. - - :: - - => (defn f [x] (setv y 10) (+ x y) None) - => (print (f 4)) - None + explicit ``return`` is only necessary to exit a function early. To force + Python's behavior of returning ``None`` when execution reaches the end of a + function, you can put ``None`` there yourself:: + + (defn f [x] + (setv y 10) + (print (+ x y)) + None) + (print (f 4)) ; Prints "14" and then "None" .. hy:function:: (cut [coll [start None] [stop None] [step None]) From 9df24b8439caf6cd61e7384e17a1a20a84813323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesc=20Guill=C3=A9n=20Torruella?= Date: Tue, 13 Dec 2022 09:25:34 +0100 Subject: [PATCH 076/342] Update CherryPy example to hy0.25. The example was not a valid. #@ is not supported in 0.25 use [] as first parameter in defn. --- docs/whyhy.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/whyhy.rst b/docs/whyhy.rst index c6c00c898..5fa857624 100644 --- a/docs/whyhy.rst +++ b/docs/whyhy.rst @@ -109,8 +109,8 @@ in Hy:: (import cherrypy) (defclass HelloWorld [] - #@(cherrypy.expose (defn index [self] - "Hello World!"))) + (defn [cherrypy.expose] index [self] + "Hello World!")) (cherrypy.quickstart (HelloWorld)) From 64377d770bdb3bd098ad0937f19ad22d557afeed Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 13 Dec 2022 16:55:04 -0500 Subject: [PATCH 077/342] Document augmented assignment --- hy/pyops.hy | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hy/pyops.hy b/hy/pyops.hy index 0b09129d9..1869d1060 100644 --- a/hy/pyops.hy +++ b/hy/pyops.hy @@ -17,7 +17,17 @@ macro instead of the function. The functions in ``hy.pyops`` have the same semantics as their macro equivalents, with one exception: functions can't short-circuit, so the functions for operators such as ``and`` and ``!=`` unconditionally -evaluate all arguments." +evaluate all arguments. + +Hy also provides macros for :ref:`Python's augmented assignment +operators ` (but no equivalent functions, because Python +semantics don't allow for this). These macros require at least two +arguments even if the parent operator doesn't; for example, ``(-= x)`` +is an error even though ``(- x)`` is legal. On the other hand, +augmented-assignment macros extend to more than two arguments in an +analogous way as the parent operator, following the pattern ``(OP= x a b +c …)`` → ``(OP= x (OP a b c …))``. For example, ``(+= count n1 n2 n3)`` +is equivalent to ``(+= count (+ n1 n2 n3)).``" ;;;; Hy shadow functions From 9f1bd2463c95a0e33fff1a48dcb1af1f6f1336b2 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 14 Dec 2022 14:56:14 -0500 Subject: [PATCH 078/342] Improve the documentation of more core macros --- docs/api.rst | 210 ++++++++++++++++++++------------------------------- 1 file changed, 82 insertions(+), 128 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index e952ec1a1..ac7e1d2ef 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -957,59 +957,33 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. None) (print (f 4)) ; Prints "14" and then "None" -.. hy:function:: (cut [coll [start None] [stop None] [step None]) +.. hy:function:: (cut [coll arg1 arg2 arg3]) - ``cut`` can be used to take a subset of a list and create a new list from it. - The form takes at least one parameter specifying the list to cut. Two - optional parameters can be used to give the start and end position of the - subset. If only one is given, it is taken as the ``stop`` value. - The third optional parameter is used to control the step stride between the elements. + ``cut`` compiles to a :ref:`slicing expression `, which selects multiple elements of a sequential data structure. The first argument is the object to be sliced. The remaining arguments are optional, and understood the same way as in a Python slicing expression. :: - ``cut`` follows the same rules as its Python counterpart. Negative indices are - counted starting from the end of the list. Some example usage: + (setv x "abcdef") + (cut x) ; => "abcdef" + (cut x 3) ; => "abc" + (cut x 3 5) ; => "de" + (cut x -3 None) ; => "def" + (cut x 0 None 2) ; => "ace" - :strong:`Examples` - - :: - - => (setv collection (range 10)) - => (cut collection) - [0 1 2 3 4 5 6 7 8 9] - - => (cut collection 5) - [0 1 2 3 4] - - => (cut collection 2 8) - [2 3 4 5 6 7] - - => (cut collection 2 8 2) - [2 4 6] + A ``cut`` form is a valid target for assignment (with :hy:func:`setv`, ``+=``, etc.) and for deletion (with :hy:func:`del`). - => (cut collection -4 -2) - [6 7] +.. hy:function:: (raise [exception :from other]) -.. hy:function:: (raise [[exception None]]) + ``raise`` compiles to a :py:keyword:`raise` statement, which throws an + exception. With no arguments, the current exception is reraised. With one + argument, an exception, that exception is raised. :: - The ``raise`` form can be used to raise an ``Exception`` at - runtime. Example usage: - - :strong:`Examples` - - :: - - (raise) - ; re-rase the last exception - - (raise IOError) - ; raise an IOError - - (raise (IOError "foobar")) - ; raise an IOError("foobar") - - - ``raise`` can accept a single argument (an ``Exception`` class or instance) - or no arguments to re-raise the last ``Exception``. + (try + (raise KeyError) + (except [KeyError] + (print "gottem"))) + ``raise`` supports one other syntax, ``(raise EXCEPTION_1 :from + EXCEPTION_2)``, which compiles to a Python ``raise … from`` statement like + ``raise EXCEPTION_1 from EXCEPTION_2``. .. hy:function:: (try [#* body]) @@ -1047,7 +1021,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. - ``[]`` to catch any subtype of ``Exception``, like Python's ``except:`` - ``[ETYPE]`` to catch only the single type ``ETYPE``, like Python's - ```except ETYPE:`` + ``except ETYPE:`` - ``[[ETYPE1 ETYPE2 …]]`` to catch any of the named types, like Python's ``except ETYPE1, ETYPE2, …:`` - ``[VAR ETYPE]`` to catch ``ETYPE`` and bind it to ``VAR``, like Python's @@ -1144,19 +1118,17 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:function:: (while [condition #* body]) - ``while`` compiles to a :py:keyword:`while` statement. It is used to execute a - set of forms as long as a condition is met. The first argument to ``while`` is - the condition, and any remaining forms constitute the body. The following - example will output "Hello world!" to the screen indefinitely: - - :: - - (while True (print "Hello world!")) + ``while`` compiles to a :py:keyword:`while` statement, which executes some + code as long as a condition is met. The first argument to ``while`` is the + condition, and any remaining forms constitute the body. It always returns + ``None``. :: - The last form of a ``while`` loop can be an ``else`` clause, which is executed - after the loop terminates, unless it exited abnormally (e.g., with ``break``). So, + (while True + (print "Hello world!")) - :: + The last form of a ``while`` loop can be an ``else`` clause, which is + executed after the loop terminates, unless it exited abnormally (e.g., with + ``break``). So, :: (setv x 2) (while x @@ -1165,9 +1137,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (else (print "In else"))) - prints - - :: + prints :: In body In body @@ -1177,9 +1147,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. loop, it will apply to the very same loop rather than an outer loop, even if execution is yet to ever reach the loop body. (Hy compiles a ``while`` loop with statements in its condition by rewriting it so that the condition is - actually in the body.) So, - - :: + actually in the body.) So, :: (for [x [1]] (print "In outer loop") @@ -1192,95 +1160,81 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print "This won't print, either.")) (print "At end of outer loop")) - prints - - :: + prints :: In outer loop In condition At end of outer loop -.. hy:function:: (with [#* args]) - - Wrap execution of `body` within a context manager given as bracket `args`. - ``with`` is used to wrap the execution of a block within a context manager. The - context manager can then set up the local system and tear it down in a controlled - manner. The archetypical example of using ``with`` is when processing files. - If only a single expression is supplied, or the argument is `_`, then no - variable is bound to the expression, as shown below. - - Examples: - :: - - => (with [arg (expr)] block) - => (with [(expr)] block) - => (with [arg1 (expr1) _ (expr2) arg3 (expr3)] block) - - The following example will open the ``NEWS`` file and print its content to the - screen. The file is automatically closed after it has been processed:: +.. hy:function:: (with [managers #* body]) - => (with [f (open \"NEWS\")] (print (.read f))) + ``with`` compiles to a :py:keyword:`with` statement, which wraps some code + with one or more :ref:`context managers `. The first + argument is a bracketed list of context managers, and the remaining + arguments are body forms. - ``with`` returns the value of its last form, unless it suppresses an exception - (because the context manager's ``__exit__`` method returned true), in which - case it returns ``None``. So, the previous example could also be written:: + The manager list can't be empty. If it has only one item, that item is + evaluated to obtain the context manager to use. If it has two, the first + argument (a symbol) is bound to the result of the second. Thus, ``(with + [(f)] …)`` compiles to ``with f(): …`` and ``(with [x (f)] …)`` compiles to + ``with f() as x: …``. :: - => (print (with [f (open \"NEWS\")] (.read f))) + (with [o (open "file.txt" "rt")] + (print (.read o))) -.. hy:function:: (with/a [#* args]) + If the manager list has more than two items, they're understood as + variable-manager pairs; thus :: - Wrap execution of `body` within a context manager given as bracket `args`. - ``with/a`` behaves like ``with``, but is used to wrap the execution of a block - within an asynchronous context manager. The context manager can then set up - the local system and tear it down in a controlled manner asynchronously. - Examples: + (with [v1 e1 v2 e2 v3 e3] …) - :: - => (with/a [arg (expr)] block) - => (with/a [(expr)] block) - => (with/a [_ (expr) arg (expr) _ (expr)] block) + compiles to - .. note:: - ``with/a`` returns the value of its last form, unless it suppresses an exception - (because the context manager's ``__aexit__`` method returned true), in which - case it returns ``None``. + .. code-block:: python -.. hy:function:: (yield [object]) + with e1 as v1, e2 as v2, e3 as v3: … - ``yield`` is used to create a generator object that returns one or more values. - The generator is iterable and therefore can be used in loops, list - comprehensions and other similar constructs. + The symbol ``_`` is interpreted specially as a variable name in the manager + list: instead of binding the context manager to the variable ``_`` (as + Python's ``with e1 as _: …``), ``with`` will leave it anonymous (as Python's + ``with e1: …``). - The function ``random-numbers`` shows how generators can be used to generate - infinite series without consuming infinite amount of memory. + ``with`` returns the value of its last form, unless it suppresses an + exception (because the context manager's ``__exit__`` method returned true), + in which case it returns ``None``. So, the previous example could also be + written :: - :strong:`Examples` + (print (with [o (open "file.txt" "rt")] (.read o))) - :: +.. hy:function:: (with/a [managers #* body]) - => (defn multiply [bases coefficients] - ... (for [#(base coefficient) (zip bases coefficients)] - ... (yield (* base coefficient)))) + As :hy:func:`with`, but compiles to an :py:keyword:`async with` statement. - => (multiply (range 5) (range 5)) - +.. hy:function:: (yield [value]) - => (list (multiply (range 10) (range 10))) - [0 1 4 9 16 25 36 49 64 81] + ``yield`` compiles to a :ref:`yield expression `, which + returns a value as a generator. As in Python, one argument, the value to + yield, is accepted, and it defaults to ``None``. :: - => (import random) - => (defn random-numbers [low high] - ... (while True (yield (.randint random low high)))) - => (list (take 15 (random-numbers 1 50))) - [7 41 6 22 32 17 5 38 18 38 17 14 23 23 19] + (defn naysayer [] + (while True + (yield "nope"))) + (hy.repr (list (zip "abc" (naysayer)))) + ; => [#("a" "nope") #("b" "nope") #("c" "nope")] + For ``yield from``, see :hy:func:`yield-from`. .. hy:function:: (yield-from [object]) - ``yield-from`` is used to call a subgenerator. This is useful if you - want your coroutine to be able to delegate its processes to another - coroutine, say, if using something fancy like - `asyncio `_. + ``yield-from`` compiles to a :ref:`yield-from expression `, + which returns a value from a subgenerator. The syntax is the same as that of + :hy:func:`yield`. :: + + (defn myrange [] + (setv r (range 10)) + (while True + (yield-from r))) + (hy.repr (list (zip "abc" (myrange)))) + ; => [#("a" 0) #("b" 1) #("c" 2)] .. hy:function:: (pragma) From a674254b329c8c2b0421af7c480286d979f51af3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 30 Dec 2022 12:03:39 -0500 Subject: [PATCH 079/342] Test reading of numbers with leading zeroes --- tests/test_reader.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_reader.py b/tests/test_reader.py index 68813438b..af37fd686 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -263,6 +263,15 @@ def test_lex_digit_separators(): ] +def test_leading_zero(): + assert tokenize("0") == [Integer(0)] + assert tokenize("0000") == [Integer(0)] + assert tokenize("010") == [Integer(10)] + assert tokenize("000010") == [Integer(10)] + assert tokenize("000010.00") == [Float(10)] + assert tokenize("010+000010j") == [Complex(10 + 10j)] + + def test_lex_bad_attrs(): with lexe() as execinfo: tokenize("1.foo") From becc2d1e7e8beec4a9201acbfeef879293e2981c Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 30 Dec 2022 12:06:03 -0500 Subject: [PATCH 080/342] Delegate more of `Integer.__new__` to `int` --- hy/models.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/hy/models.py b/hy/models.py index 18881de65..28f620372 100644 --- a/hy/models.py +++ b/hy/models.py @@ -327,22 +327,13 @@ class Integer(Object, int): """ def __new__(cls, number, *args, **kwargs): - if isinstance(number, str): - number = strip_digit_separators(number) - bases = {"0x": 16, "0o": 8, "0b": 2} - for leader, base in bases.items(): - if number.startswith(leader): - # We've got a string, known leader, set base. - number = int(number, base=base) - break - else: - # We've got a string, no known leader; base 10. - number = int(number, base=10) - else: - # We've got a non-string; convert straight. - number = int(number) - return super().__new__(cls, number) - + return super().__new__(cls, int( + strip_digit_separators(number), + **({'base': 0} + if isinstance(number, str) and not number.isdigit() + # `not number.isdigit()` is necessary because `base = 0` + # fails on decimal integers starting with a leading 0. + else {}))) _wrappers[int] = Integer From 2acb4eceb327bfc0e240dd1f9345f600b919dcd7 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 30 Dec 2022 13:19:40 -0500 Subject: [PATCH 081/342] Remove a test skip for a now-fixed PyPy bug --- tests/test_bin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_bin.py b/tests/test_bin.py index d046ce8e3..18a36d19b 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -522,7 +522,6 @@ def test_macro_require(): assert output.strip() == "abc" -@pytest.mark.skipif(PYPY, reason = 'https://foss.heptapod.net/pypy/pypy/-/issues/3800') def test_tracebacks(): """Make sure the printed tracebacks are correct.""" From 2fd4969468e7341456d2e250bc8f75cb3d63a9b8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 30 Dec 2022 13:20:43 -0500 Subject: [PATCH 082/342] Add a test skip for a new PyPy bug --- tests/test_bin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_bin.py b/tests/test_bin.py index 18a36d19b..bcc8dfc21 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -654,6 +654,7 @@ def test_output_buffering(tmp_path): assert tf.read_text().splitlines() == ["line 1", "line 2"] +@pytest.mark.skipif(PYPY, reason = 'https://foss.heptapod.net/pypy/pypy/-/issues/3881') def test_uufileuu(tmp_path, monkeypatch): # `__file__` should be set the same way as in Python. # https://github.com/hylang/hy/issues/2318 From a48a377ab7d30fe8275c5a3b91d0c4c4381bc1ed Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 2 Jan 2023 18:01:14 -0500 Subject: [PATCH 083/342] Autoformat --- hy/models.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/hy/models.py b/hy/models.py index 28f620372..4a3317880 100644 --- a/hy/models.py +++ b/hy/models.py @@ -327,13 +327,20 @@ class Integer(Object, int): """ def __new__(cls, number, *args, **kwargs): - return super().__new__(cls, int( - strip_digit_separators(number), - **({'base': 0} - if isinstance(number, str) and not number.isdigit() - # `not number.isdigit()` is necessary because `base = 0` - # fails on decimal integers starting with a leading 0. - else {}))) + return super().__new__( + cls, + int( + strip_digit_separators(number), + **( + {"base": 0} + if isinstance(number, str) and not number.isdigit() + # `not number.isdigit()` is necessary because `base = 0` + # fails on decimal integers starting with a leading 0. + else {} + ), + ), + ) + _wrappers[int] = Integer From 625a20ee824b454a7a74a2f29bf831087362d7b9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 10:31:14 -0500 Subject: [PATCH 084/342] Update copyright year --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index a0d3b8150..1247a0ad2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2022 the authors. +Copyright 2023 the authors. Portions of setup.py, copyright 2016 Jason R Coombs . Permission is hereby granted, free of charge, to any person obtaining a From 7e6c055c81a2da5fa5ec0b4cafa6bb518156cc58 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 30 Dec 2022 14:48:37 -0500 Subject: [PATCH 085/342] Use the term "identifier" more consistently --- hy/models.py | 4 ++-- hy/reader/hy_reader.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/hy/models.py b/hy/models.py index 4a3317880..ad3a93db8 100644 --- a/hy/models.py +++ b/hy/models.py @@ -227,9 +227,9 @@ def __new__(cls, s, from_parser=False): if not from_parser: # Check that the symbol is syntactically legal. # import here to prevent circular imports. - from hy.reader.hy_reader import symbol_like + from hy.reader.hy_reader import as_identifier - sym = symbol_like(s) + sym = as_identifier(s) if not isinstance(sym, Symbol): raise ValueError(f"Syntactically illegal symbol: {s!r}") return sym diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index fd77c8470..d166ab1a4 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -34,8 +34,8 @@ def mkexpr(root, *args): return Expression((sym(root) if isinstance(root, str) else root, *args)) -def symbol_like(ident, reader=None): - """Generate a Hy model from an identifier-like string. +def as_identifier(ident, reader=None): + """Generate a Hy model from an identifier. Also verifies the syntax of dot notation and validity of symbol names. @@ -67,7 +67,7 @@ def symbol_like(ident, reader=None): if "." in ident: for chunk in ident.split("."): - if chunk and not isinstance(symbol_like(chunk, reader=reader), Symbol): + if chunk and not isinstance(as_identifier(chunk, reader=reader), Symbol): msg = ( "Cannot access attribute on anything other" " than a name (in order to get attributes of expressions," @@ -116,14 +116,14 @@ def fill_pos(self, model, start): def read_default(self, key): """Default reader handler when nothing in the table matches. - Try to read an identifier/symbol. If there's a double-quote immediately - following, then parse it as a string with the given prefix (e.g., - `r"..."`). Otherwise, parse it as a symbol-like. + Try to read an identifier. If there's a double-quote immediately + following, then instead parse it as a string with the given prefix (e.g., + `r"..."`). """ ident = key + self.read_ident() if self.peek_and_getc('"'): return self.prefixed_string('"', ident) - return symbol_like(ident, reader=self) + return as_identifier(ident, reader=self) def parse(self, stream, filename=None): """Yields all `hy.models.Object`'s in `source` From 94246534a2d25e792928264722e6ce6025ec65e8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 3 Jan 2023 08:16:00 -0500 Subject: [PATCH 086/342] Parse dotted identifiers as expressions --- hy/reader/hy_reader.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index d166ab1a4..70ab75b0b 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -66,17 +66,26 @@ def as_identifier(ident, reader=None): pass if "." in ident: - for chunk in ident.split("."): - if chunk and not isinstance(as_identifier(chunk, reader=reader), Symbol): - msg = ( - "Cannot access attribute on anything other" - " than a name (in order to get attributes of expressions," - " use `(. )` or `(. )`)" - ) - if reader is None: - raise ValueError(msg) - else: - raise LexException.from_reader(msg, reader) + if not ident.strip("."): + # It's all dots. Return it as a symbol. + return sym(ident) + def err(msg): + raise (ValueError(msg) + if reader is None + else LexException.from_reader(msg, reader)) + if ident.lstrip(".").find("..") > 0: + err("In a dotted identifier, multiple dots in a row are only allowed at the start") + if ident.endswith("."): + err("A dotted identifier can't end with a dot") + head = "." * (len(ident) - len(ident.lstrip("."))) + args = [as_identifier(a, reader=reader) + for a in ident.lstrip(".").split(".")] + if any(not isinstance(a, Symbol) for a in args): + err("The parts of a dotted identifier must be symbols") + return ( + mkexpr(sym("."), *args) + if head == '' + else mkexpr(head, Symbol("None"), *args)) if reader is None: if ( From 266f3ef1b59f2f6f8d2eaceba1e18f9b420449b8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 4 Jan 2023 21:49:40 -0500 Subject: [PATCH 087/342] Update various core bits for new dotted parsing --- hy/compiler.py | 80 ++++++++++++++------------------------ hy/core/result_macros.py | 84 ++++++++++++++++++++++------------------ hy/macros.py | 6 +-- hy/reader/mangling.py | 2 +- 4 files changed, 81 insertions(+), 91 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 8db7db2b0..8f6489483 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -532,40 +532,35 @@ def compile_expression(self, expr, *, allow_annotation_expression=False): root = args.pop(0) func = None - if isinstance(root, Symbol) and root.startswith("."): - # (.split "test test") -> "test test".split() - # (.a.b.c x v1 v2) -> (.c (. x a b) v1 v2) -> x.a.b.c(v1, v2) - - # Get the method name (the last named attribute - # in the chain of attributes) - attrs = [ - Symbol(a).replace(root) if a else None for a in root.split(".")[1:] - ] - if not all(attrs): - raise self._syntax_error(expr, "cannot access empty attribute") - root = attrs.pop() - - # Get the object we're calling the method on - # (extracted with the attribute access DSL) - # Skip past keywords and their arguments. - try: - kws, obj, rest = ( - many(KEYWORD + FORM | unpack("mapping")) + FORM + many(FORM) - ).parse(args) - except NoParseError: + if ( + isinstance(root, Expression) and + len(root) >= 2 and + isinstance(root[0], Symbol) and + not str(root[0]).strip(".") and + root[1] == Symbol("None")): + # ((. None a1 a2) obj v1 v2) -> ((. obj a1 a2) v1 v2) + # (The reader already parsed `.a1.a2` as `(. None a1 a2)`.) + + # Find the object we're calling the method on. + i = 0 + while i < len(args): + if isinstance(args[i], Keyword): + if i == 0 and len(args) == 1: + break + i += 2 + elif is_unpack("iterable", args[i]): + raise self._syntax_error(args[i], "can't call a method on an `unpack-iterable` form") + elif is_unpack("mapping", args[i]): + i += 1 + else: + break + else: raise self._syntax_error(expr, "attribute access requires object") - # Reconstruct `args` to exclude `obj`. - args = [x for p in kws for x in p] + list(rest) - if is_unpack("iterable", obj): - raise self._syntax_error( - obj, "can't call a method on an unpacking form" - ) - func = self.compile(Expression([Symbol(".").replace(root), obj] + attrs)) - # And get the method - func += asty.Attribute( - root, value=func.force_expr, attr=mangle(root), ctx=ast.Load() - ) + func = self.compile(Expression([ + Symbol("."), + args.pop(i), + *root[2:]]).replace(root)) if is_annotate_expression(root): # Flatten and compile the annotation expression. @@ -595,24 +590,9 @@ def compile_symbol(self, symbol): else asty.Ellipsis(symbol) ) - if "." in symbol: - glob, local = symbol.rsplit(".", 1) - - if not glob: - raise self._syntax_error( - symbol, - "cannot access attribute on anything other than a name (in order to get attributes of expressions, use `(. {attr})` or `(.{attr} )`)".format( - attr=local - ), - ) - - if not local: - raise self._syntax_error(symbol, "cannot access empty attribute") - - glob = Symbol(glob).replace(symbol) - ret = self.compile_symbol(glob) - - return asty.Attribute(symbol, value=ret, attr=mangle(local), ctx=ast.Load()) + # By this point, `symbol` should be either all dots or + # dot-free. + assert not symbol.strip(".") or "." not in symbol if mangle(symbol) in ("None", "False", "True"): return asty.Constant(symbol, value=ast.literal_eval(mangle(symbol))) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 42dcab8a1..35c3f9569 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -74,6 +74,10 @@ def maybe_annotated(target): return pexpr(sym("annotate") + target + FORM) | target >> (lambda x: (x, None)) +def dotted(name): + return Expression(map(Symbol, [".", *name.split('.')])) + + # ------------------------------------------------ # * Fundamentals # ------------------------------------------------ @@ -181,8 +185,7 @@ def render_quoted_form(compiler, form, level): elif op in ("unquote", "unquote-splice"): level -= 1 - hytype = form.__class__ - name = ".".join((hytype.__module__, hytype.__name__)) + name = form.__class__.__name__ body = [form] if isinstance(form, Sequence): @@ -214,8 +217,9 @@ def render_quoted_form(compiler, form, level): if form.brackets is not None: body.extend([Keyword("brackets"), form.brackets]) - ret = Expression([Symbol(name), *body]).replace(form) - return ret, False + return ( + Expression([dotted("hy.models." + name), *body]).replace(form), + False) # ------------------------------------------------ @@ -447,7 +451,7 @@ def compile_assign( ld_name = compiler.compile(name) - if result.temp_variables and isinstance(name, Symbol) and "." not in name: + if result.temp_variables and isinstance(name, Symbol): result.rename(compiler, compiler._nonconst(name)) if not is_assignment_expr: # Throw away .expr to ensure that (setv ...) returns None. @@ -1170,11 +1174,7 @@ def compile_pattern(compiler, pattern): value, value=compiler.compile(value).force_expr.value, ) - elif ( - isinstance(value, (String, Integer, Float, Complex, Bytes)) - or isinstance(value, Symbol) - and "." in value - ): + elif isinstance(value, (String, Integer, Float, Complex, Bytes)): return asty.MatchValue( value, value=compiler.compile(value).expr, @@ -1230,7 +1230,7 @@ def compile_pattern(compiler, pattern): elif isinstance(value, Keyword): return asty.MatchClass( value, - cls=compiler.compile(Symbol("hy.models.Keyword")).expr, + cls=compiler.compile(dotted("hy.models.Keyword")).expr, patterns=[ asty.MatchValue(value, value=asty.Constant(value, value=value.name)) ], @@ -1476,9 +1476,6 @@ def compile_function_node(compiler, expr, node, decorators, name, args, returns, ], ) def compile_macro_def(compiler, expr, root, name, params, body): - if "." in name: - raise compiler._syntax_error(name, "periods are not allowed in macro names") - ret = Result() + compiler.compile( Expression( [ @@ -1487,7 +1484,7 @@ def compile_macro_def(compiler, expr, root, name, params, body): [ Expression( [ - Symbol("hy.macros.macro"), + dotted("hy.macros.macro"), str(name), ] ), @@ -1680,14 +1677,23 @@ def compile_class_expression(compiler, expr, root, decorators, name, rest): # * `import` and `require` # ------------------------------------------------ +module_name_pattern = SYM | pexpr( + some(lambda x: isinstance(x, Symbol) and not str(x[0]).strip(".")) + + oneplus(SYM)) -def importlike(*name_types): - name = some(lambda x: isinstance(x, name_types) and "." not in x) +def module_name_str(x): return ( - keepsym("*") - | (keepsym(":as") + name) - | brackets(many(name + maybe(sym(":as") + name))) - ) + '.'.join(map(mangle, x[1][x[1][0] == Symbol("None") :])) + if isinstance(x, Expression) + else str(x) + if isinstance(x, Symbol) and not x.strip(".") + else mangle(x)) + +importlike = ( + keepsym("*") + | (keepsym(":as") + SYM) + | brackets(many(SYM + maybe(sym(":as") + SYM))) +) def assignment_shape(module, rest): @@ -1695,13 +1701,13 @@ def assignment_shape(module, rest): assignments = "EXPORTS" if rest is None: # (import foo) - prefix = module + prefix = module_name_str(module) elif rest == Symbol("*"): # (import foo *) pass elif rest[0] == Keyword("as"): # (import foo :as bar) - prefix = rest[1] + prefix = mangle(rest[1]) else: # (import foo [bar baz :as MyBaz bing]) assignments = [(k, v or k) for k, v in rest[0]] @@ -1712,11 +1718,11 @@ def assignment_shape(module, rest): "require", [ many( - SYM + module_name_pattern + times( 0, 2, - (maybe(sym(":macros")) + importlike(Symbol)) + (maybe(sym(":macros")) + importlike) | (keepsym(":readers") + (keepsym("*") | brackets(many(SYM)))), ) ) @@ -1802,31 +1808,25 @@ def compile_require(compiler, expr, root, entries): return ret -@pattern_macro("import", [many(SYM + maybe(importlike(Symbol)))]) +@pattern_macro("import", [many(module_name_pattern + maybe(importlike))]) def compile_import(compiler, expr, root, entries): ret = Result() for entry in entries: - assignments = "EXPORTS" - prefix = "" - module, _ = entry prefix, assignments = assignment_shape(*entry) - ast_module = mangle(module) - - module_name = ast_module.lstrip(".") - level = len(ast_module) - len(module_name) + module_name = module_name_str(module) if assignments == "EXPORTS" and prefix == "": node = asty.ImportFrom names = [asty.alias(module, name="*", asname=None)] elif assignments == "EXPORTS": - compiler.scope.define(mangle(prefix)) + compiler.scope.define(prefix) node = asty.Import names = [ asty.alias( module, - name=ast_module, - asname=mangle(prefix) if prefix != module else None, + name=module_name, + asname=prefix if prefix != module_name else None, ) ] else: @@ -1839,7 +1839,17 @@ def compile_import(compiler, expr, root, entries): module, name=mangle(k), asname=None if v == k else mangle(v) ) ) - ret += node(expr, module=module_name or None, names=names, level=level) + ret += node(expr, + module = module_name if module_name and module_name.strip(".") else None, + names = names, + level = ( + len(module[0]) + if isinstance(module, Expression) and + module[1][0] == Symbol("None") + else len(module) + if isinstance(module, Symbol) and + not module.strip(".") + else 0)) return ret diff --git a/hy/macros.py b/hy/macros.py index 294310564..92c471910 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -48,9 +48,9 @@ def wrapper(hy_compiler, *args): if shadow and any(is_unpack("iterable", x) for x in args): # Try a shadow function call with this name instead. - return Expression([Symbol("hy.pyops." + name), *args]).replace( - hy_compiler.this - ) + return Expression([ + Expression(map(Symbol, [".", "hy", "pyops", name])), + *args]).replace(hy_compiler.this) expr = hy_compiler.this root = unmangle(expr[0]) diff --git a/hy/reader/mangling.py b/hy/reader/mangling.py index c22ac5282..647c2ad26 100644 --- a/hy/reader/mangling.py +++ b/hy/reader/mangling.py @@ -40,7 +40,7 @@ def mangle(s): assert s s = str(s) - if "." in s: + if "." in s and s.strip("."): return ".".join(mangle(x) if x else "" for x in s.split(".")) # Step 1: Remove and save leading underscores From b66aad53c29184e66ef61f092a0317128276f742 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 12:01:44 -0500 Subject: [PATCH 088/342] Update `require`, `macroexpand` for new dot parse --- hy/core/result_macros.py | 21 ++++++++++++--------- hy/macros.py | 7 +++++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 35c3f9569..4ba5b1818 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1746,13 +1746,16 @@ def compile_require(compiler, expr, root, entries): readers = readers and readers[0] prefix, assignments = assignment_shape(module, rest) - ast_module = mangle(module) + module_name = module_name_str(module) + if isinstance(module, Expression) and module[1][0] == Symbol("None"): + # Prepend leading dots to `module_name`. + module_name = str(module[0]) + module_name # we don't want to import all macros as prefixed if we're specifically # importing readers but not macros # (require a-module :readers ["!"]) if (rest or not readers) and require( - ast_module, compiler.module, assignments=assignments, prefix=prefix + module_name, compiler.module, assignments=assignments, prefix=prefix ): # Actually calling `require` is necessary for macro expansions # occurring during compilation. @@ -1761,8 +1764,8 @@ def compile_require(compiler, expr, root, entries): ret += compiler.compile( Expression( [ - Symbol("hy.macros.require"), - String(ast_module), + dotted("hy.macros.require"), + String(module_name), Symbol("None"), Keyword("assignments"), ( @@ -1783,22 +1786,22 @@ def compile_require(compiler, expr, root, entries): if readers == Symbol("*") else ["#" + reader for reader in readers[0]] ) - if require_reader(ast_module, compiler.module, reader_assignments): + if require_reader(module_name, compiler.module, reader_assignments): ret += compiler.compile( mkexpr( "do", mkexpr( - "hy.macros.require-reader", - String(ast_module), + dotted("hy.macros.require-reader"), + String(module_name), "None", [reader_assignments], ), mkexpr( "eval-when-compile", mkexpr( - "hy.macros.enable-readers", + dotted("hy.macros.enable-readers"), "None", - "hy.&reader", + dotted("hy.&reader"), [reader_assignments], ), ), diff --git a/hy/macros.py b/hy/macros.py index 92c471910..70841bbe6 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -393,10 +393,13 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): while isinstance(tree, Expression) and tree: fn = tree[0] - if fn in ("quote", "quasiquote") or not isinstance(fn, Symbol): + if isinstance(fn, Expression) and fn and fn[0] == Symbol("."): + fn = ".".join(map(mangle, fn[1:])) + elif isinstance(fn, Symbol): + fn = mangle(fn) + else: break - fn = mangle(fn) expr_modules = ([] if not hasattr(tree, "module") else [tree.module]) + [module] expr_modules.append(builtins) From ce0bf1866ebc990fd1bdddb5f681b190cd239c25 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 3 Jan 2023 08:19:34 -0500 Subject: [PATCH 089/342] Update some error-message tests --- tests/native_tests/native_macros.hy | 5 ++--- tests/test_reader.py | 8 ++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy index a5a0aa40f..30777494d 100644 --- a/tests/native_tests/native_macros.hy +++ b/tests/native_tests/native_macros.hy @@ -75,9 +75,8 @@ (hy.eval '(defmacro :kw []))) (assert (in "got unexpected token: :kw" e.value.msg)) - (with [e (pytest.raises HySyntaxError)] - (hy.eval '(defmacro foo.bar []))) - (assert (in "periods are not allowed in macro names" e.value.msg))) + (with [(pytest.raises HySyntaxError)] + (hy.eval '(defmacro foo.bar [])))) (defn test-macro-calling-fn [] (assert (= 3 (bar 1 2)))) diff --git a/tests/test_reader.py b/tests/test_reader.py index af37fd686..52a05edd4 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -281,9 +281,7 @@ def test_lex_bad_attrs(): ' File "", line 1', " 1.foo", " ^", - "hy.reader.exceptions.LexException: Cannot access attribute on anything other" - " than a name (in order to get attributes of expressions," - " use `(. )` or `(. )`)", + "hy.reader.exceptions.LexException: The parts of a dotted identifier must be symbols", ], ) @@ -625,9 +623,7 @@ def test_lex_exception_filtering(capsys): ' File "", line 3', " 1.foo", " ^", - "hy.reader.exceptions.LexException: Cannot access attribute on anything other" - " than a name (in order to get attributes of expressions," - " use `(. )` or `(. )`)", + "hy.reader.exceptions.LexException: The parts of a dotted identifier must be symbols", ], ) From a76fbe87c379d200040d541f082a27ba22de1228 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 12:40:41 -0500 Subject: [PATCH 090/342] Update `hy-repr` for new dotted parsing --- hy/core/hy_repr.hy | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/hy/core/hy_repr.hy b/hy/core/hy_repr.hy index b98b36e98..663b80ca1 100644 --- a/hy/core/hy_repr.hy +++ b/hy/core/hy_repr.hy @@ -114,16 +114,34 @@ 'unquote-splice "~@" 'unpack-iterable "#* " 'unpack-mapping "#** "}) + (setv x0 (when x (get x 0))) + (setv x1 (when (> (len x) 1) (get x 1))) + (cond - (and (= (len x) 2) (in (get x 0) syntax)) + + (and + (>= (len x) 3) + (all (gfor e x (is (type e) hy.models.Symbol))) + (or (= x0 '.) (and + (= x1 'None) + (not (.strip (str x0) "."))))) + (+ + (if (= x1 'None) (str x0) "") + (.join "." (map hy-repr (cut + x + (if (= x1 'None) 2 1) + None)))) + + (and (= (len x) 2) (in x0 syntax)) (if (and - (= (get x 0) 'unquote) - (isinstance (get x 1) hy.models.Symbol) - (.startswith (get x 1) "@")) + (= x0 'unquote) + (isinstance x1 hy.models.Symbol) + (.startswith x1 "@")) ; This case is special because `~@b` would be wrongly ; interpreted as `(unquote-splice b)` instead of `(unquote @b)`. - (+ "~ " (hy-repr (get x 1))) - (+ (get syntax (get x 0)) (hy-repr (get x 1)))) + (+ "~ " (hy-repr x1)) + (+ (get syntax x0) (hy-repr x1))) + True (+ "(" (_cat x) ")")))) From f45e284dd05b41fae19ada6744de2ed201817b75 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 12:41:09 -0500 Subject: [PATCH 091/342] Add more `hy-repr` tests with dots --- tests/resources/hy_repr_str_tests.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/resources/hy_repr_str_tests.txt b/tests/resources/hy_repr_str_tests.txt index 668eb4592..f61568966 100644 --- a/tests/resources/hy_repr_str_tests.txt +++ b/tests/resources/hy_repr_str_tests.txt @@ -29,6 +29,10 @@ NaN+NaNj 'mysymbol 'my♥symbol? +'. +'.. +'... +'.... :mykeyword :my♥keyword? : @@ -37,6 +41,14 @@ NaN+NaNj ; quote (which is equivalent), so it's tested in ; `test-hy-repr-roundtrip-from-value`. +;; * Dotted identifiers + +'foo.bar +'.foo.bar +'..foo.bar +'...foo.bar +'....foo.bar + ;; * Stringy thingies "" @@ -90,6 +102,15 @@ b"\"double \" quotes\"" '(+ 1 2) '(f a b) '(f #* args #** kwargs) +'(.real 3) +'(.a.b.c foo) +'(math.sqrt 25) +'(. a) +'(. None) +'(. 1 2) +; `(. a b)` will render as `a.b`. +'(.. 1 2) +'(.. a b) [1 [2 3] #(4 #('mysymbol :mykeyword)) {"a" b"hello"} '(f #* a #** b)] '(quote) From c6f2b1dd049f4e187132216622615f4bab6832b6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 14:05:51 -0500 Subject: [PATCH 092/342] Edit the docstring of `hy.mangle` --- hy/reader/mangling.py | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/hy/reader/mangling.py b/hy/reader/mangling.py index 647c2ad26..ce9f85822 100644 --- a/hy/reader/mangling.py +++ b/hy/reader/mangling.py @@ -7,34 +7,26 @@ def mangle(s): - """Stringify the argument and convert it to a valid Python identifier - according to :ref:`Hy's mangling rules `. + """Stringify the argument (with :class:`str`, not :func:`repr` or + :hy:func:`hy.repr`) and convert it to a valid Python identifier according + to :ref:`Hy's mangling rules `. :: - If the argument is already both legal as a Python identifier and normalized - according to Unicode normalization form KC (NFKC), it will be returned - unchanged. Thus, ``mangle`` is idempotent. + (hy.mangle 'foo-bar?) ; => "is_foo_bar" + (hy.mangle "🦑") ; => "hyx_squid" - Examples: - :: - - => (hy.mangle 'foo-bar) - "foo_bar" - - => (hy.mangle 'foo-bar?) - "is_foo_bar" - - => (hy.mangle '*) - "hyx_XasteriskX" - - => (hy.mangle '_foo/a?) - "_hyx_is_fooXsolidusXa" + If the stringified argument is already both legal as a Python identifier + and normalized according to Unicode normalization form KC (NFKC), it will + be returned unchanged. Thus, ``hy.mangle`` is idempotent. :: - => (hy.mangle '-->) - "hyx_XhyphenHminusX_XgreaterHthan_signX" + (setv x '♦-->♠) + (= (hy.mangle (hy.mangle x)) (hy.mangle x)) ; => True - => (hy.mangle '<--) - "hyx_XlessHthan_signX__" + Generally, the stringifed input is expected to be parsable as a symbol. As + a convenience, it can also have the syntax of a :ref:`dotted identifier + `, and ``hy.mangle`` will mangle the dot-delimited + parts separately. :: + (hy.mangle "a.b?.c!.d") ; => "a.is_b.hyx_cXexclamation_markX.d" """ assert s From e6fe1a50dac9a801961e37e08236b5c4cae57f59 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 4 Jan 2023 10:13:00 -0500 Subject: [PATCH 093/342] Document the syntax of dotted identifiers --- docs/syntax.rst | 59 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/docs/syntax.rst b/docs/syntax.rst index b2ad201b1..6eede3c78 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -132,8 +132,9 @@ Identifiers are a broad class of syntax in Hy, comprising not only variable names, but any nonempty sequence of characters that aren't ASCII whitespace nor one of the following: ``()[]{};"'`~``. The reader will attempt to read each identifier as a :ref:`numeric literal `, then attempt to read -it as a :ref:`keyword ` if that fails, then fall back on reading it -as a :ref:`symbol ` if that fails. +it as a :ref:`keyword ` if that fails, then attempt to read it as a +:ref:`dotted identifier ` if that fails, then fall back on +reading it as a :ref:`symbol ` if that fails. .. _numeric-literals: @@ -195,6 +196,34 @@ function call with an empty keyword argument. .. autoclass:: hy.models.Keyword :members: __bool__, __call__ +.. _dotted-identifiers: + +Dotted identifiers +~~~~~~~~~~~~~~~~~~ + +Dotted identifiers are named for their use of the dot character ``.``, also +known as a period or full stop. They don't have their own model type because +they're actually syntactic sugar for :ref:`expressions `. Syntax +like ``foo.bar.baz`` is equivalent to ``(. foo bar baz)``. The general rule is +that a dotted identifier looks like two or more :ref:`symbols ` +(themselves not containing any dots) separated by single dots. The result is an +expression with the symbol ``.`` as its first element and the constituent +symbols as the remaining elements. + +A dotted identifier may also begin with one or more dots, as in ``.foo.bar`` or +``..foo.bar``, in which case the resulting expression has the appropriate head +(``.`` or ``..`` or whatever) and the symbol ``None`` as the following element. +Thus, ``..foo.bar`` is equivalent to ``(.. None foo bar)``. In the leading-dot +case, you may also use only one constitutent symbol. Thus, ``.foo`` is a legal +dotted identifier, and equivalent to ``(. None foo)``. + +See :ref:`the dot macro ` for what these expressions typically compile to. +See also the special behavior for :ref:`expressions ` that begin +with a dotted identifier that itself begins with a dot. Note that Hy provides +definitions of ``.`` and ``...`` by default, but not ``..``, ``....``, +``.....``, etc., so ``..foo.bar`` won't do anything useful by default outside +of macros that treat it specially, like :hy:func:`import`. + .. _symbols: Symbols @@ -208,6 +237,10 @@ the :class:`Symbol ` constructor (thus, :class:`Symbol Lisps). Some example symbols are ``hello``, ``+++``, ``3fiddy``, ``$40``, ``just✈wrong``, and ``🦑``. +Dots are only allowed in a symbol if every character in the symbol is a dot. +Thus, ``a..b`` and ``a.`` are neither dotted identifiers nor symbols; they're +simply illegal syntax. + As a special case, the symbol ``...`` compiles to the :class:`Ellipsis` object, as in Python. @@ -355,13 +388,21 @@ Expressions Expressions (:class:`Expression `) are denoted by parentheses: ``( … )``. The compiler evaluates expressions by checking the -first element. If it's a symbol, and the symbol is the name of a currently -defined macro, the macro is called. Otherwise, the expression is compiled into -a Python-level call, with the first element being the calling object. (So, you -can call a function that has the same name as a macro with an expression like -``((do setv) …)``). The remaining forms are understood as arguments. Use -:hy:func:`unpack-iterable` or :hy:func:`unpack-mapping` to break up data -structures into individual arguments at runtime. +first element. + +- If it's a symbol, and the symbol is the name of a currently defined macro, + the macro is called. +- If it is itself an expression of the form ``(. None …)`` (typically produced + with a :ref:`dotted identifier ` like ``.add``), it's used + to construct a method call with the element after ``None`` as the object: + thus, ``(.add my-set 5)`` is equivalent to ``((. my-set add) 5)``, which + becomes ``my_set.add(5)`` in Python. +- Otherwise, the expression is compiled into a Python-level call, with the + first element being the calling object. (So, you can call a function that has + the same name as a macro with an expression like ``((do setv) …)``.) The + remaining forms are understood as arguments. Use :hy:func:`unpack-iterable` + or :hy:func:`unpack-mapping` to break up data structures into individual + arguments at runtime. The empty expression ``()`` is legal at the reader level, but has no inherent meaning. Trying to compile it is an error. For the empty tuple, use ``#()``. From ce7748cf3ca5119ad97aefbf41ccb0db7a369dc3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 4 Jan 2023 10:34:26 -0500 Subject: [PATCH 094/342] Document the magical use of `pyops` with unpacking --- docs/syntax.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/syntax.rst b/docs/syntax.rst index 6eede3c78..0bace6238 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -392,6 +392,15 @@ first element. - If it's a symbol, and the symbol is the name of a currently defined macro, the macro is called. + + - Exception: if the symbol is also the name of a function in + :hy:mod:`hy.pyops`, and one of the arguments is an + :hy:func:`unpack-iterable` form, the ``pyops`` function is called instead + of the macro. This makes reasonable-looking expressions work that would + otherwise fail. For example, ``(+ #* summands)`` is understood as + ``(hy.pyops.+ #* summands)``, because Python provides no way to sum a list + of unknown length with a real addition expression. + - If it is itself an expression of the form ``(. None …)`` (typically produced with a :ref:`dotted identifier ` like ``.add``), it's used to construct a method call with the element after ``None`` as the object: From b26fbf434a75eca99be0a8acf6ddf4f77bafa2d4 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 4 Jan 2023 10:07:35 -0500 Subject: [PATCH 095/342] Correct an Intersphinx link --- docs/syntax.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/syntax.rst b/docs/syntax.rst index 0bace6238..70d9b2ced 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -241,7 +241,7 @@ Dots are only allowed in a symbol if every character in the symbol is a dot. Thus, ``a..b`` and ``a.`` are neither dotted identifiers nor symbols; they're simply illegal syntax. -As a special case, the symbol ``...`` compiles to the :class:`Ellipsis` object, +As a special case, the symbol ``...`` compiles to the :data:`Ellipsis` object, as in Python. .. autoclass:: hy.models.Symbol From 836766cc1c8b8bb1e914663f53ab2b0fedf678c8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 15:09:32 -0500 Subject: [PATCH 096/342] Rewrite the documentation of the dot macro --- docs/api.rst | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index ac7e1d2ef..e05545f05 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -58,30 +58,31 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:data:: . - ``.`` is used to perform attribute access on objects. It uses a small DSL - to allow quick access to attributes and items in a nested data structure. + The dot macro ``.`` compiles to one or more :ref:`attribute references + `, which select an attribute of an object. The + first argument, which is required, can be an arbitrary form. With no further + arguments, ``.`` is a no-op. Additional symbol arguments are understood as a + chain of attributes, so ``(. foo bar)`` compiles to ``foo.bar``, and ``(. a b + c d)`` compiles to ``a.b.c.d``. - :strong:`Examples` - - :: + As a convenience, ``.`` supports two other kinds of arguments in place of a + plain attribute. A parenthesized expression is understood as a method call: + ``(. foo (bar a b))`` compiles to ``x.foo.bar(a, b)``. A bracketed form is + understood as a subscript: ``(. foo ["bar"])`` compiles to ``foo["bar"]``. + All these options can be mixed and matched in a single ``.`` call, so :: - (. foo (bar "qux") baz [(+ 1 2)] frob) + (. a (b 1 2) c [d] [(e)]) - Compiles down to: + compiles to .. code-block:: python - foo.bar("qux").baz[1 + 2].frob - - ``.`` compiles its first argument (in the example, *foo*) as the object on - which to do the attribute dereference. It uses bare symbols as attributes - to access (in the example, *baz*, *frob*), Expressions as method calls (as in *bar*), - and compiles the contents of lists (in the example, ``[(+ 1 2)]``) for indexation. - Other arguments raise a compilation error. + a.b(1, 2).c[d][e()] - Access to unknown attributes raises an :exc:`AttributeError`. Access to - unknown keys raises an :exc:`IndexError` (on lists and tuples) or a - :exc:`KeyError` (on dictionaries). + :ref:`Dotted identifiers ` provide syntactic sugar for + common uses of this macro. In particular, syntax like ``foo.bar`` ends up + meaning the same thing in Hy as in Python. Also, :hy:func:`get + ` is another way to subscript in Hy. .. hy:function:: (fn [args]) From 6a3f6a980ae9db1769561131506cb4822989883f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 16:28:06 -0500 Subject: [PATCH 097/342] Move a test --- tests/native_tests/language.hy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index f4194f253..2f66953ed 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -1213,6 +1213,11 @@ cee"} dee" "ey bee\ncee dee")) (assert (= dn dirname))) +(defn test-relative-import [] + (import ..resources [tlib]) + (assert (= tlib.SECRET-MESSAGE "Hello World"))) + + (defn test-lambda-keyword-lists [] (defn foo [x #* xs #** kw] [x xs kw]) (assert (= (foo 10 20 30) [10 #(20 30) {}]))) @@ -1564,10 +1569,6 @@ cee"} dee" "ey bee\ncee dee")) (assert (= m.__doc__ "This is the module docstring.")) (assert (= m.foo 5))) -(defn test-relative-import [] - (import ..resources [tlib]) - (assert (= tlib.SECRET-MESSAGE "Hello World"))) - (defn test-exception-cause [] (assert (is NameError (type (. From d91917134d32717ccf509d26dc2990c9467aadce Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 16:48:37 -0500 Subject: [PATCH 098/342] Add more relative-import tests --- tests/native_tests/language.hy | 7 +++++-- tests/resources/__init__.py | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 2f66953ed..9d5a4164e 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -1214,8 +1214,11 @@ cee"} dee" "ey bee\ncee dee")) (defn test-relative-import [] - (import ..resources [tlib]) - (assert (= tlib.SECRET-MESSAGE "Hello World"))) + (import ..resources [tlib in-init]) + (assert (= tlib.SECRET-MESSAGE "Hello World")) + (assert (= in-init "chippy")) + (import .. [resources]) + (assert (= resources.in-init "chippy"))) (defn test-lambda-keyword-lists [] diff --git a/tests/resources/__init__.py b/tests/resources/__init__.py index fe698d49a..5fec6dbb9 100644 --- a/tests/resources/__init__.py +++ b/tests/resources/__init__.py @@ -1,3 +1,6 @@ +in_init = "chippy" + + def kwtest(*args, **kwargs): return kwargs From d4090658e0b03fd81a2af03da5b41144686acb32 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 17:10:54 -0500 Subject: [PATCH 099/342] Update tests of illegal uses of dots --- tests/compilers/test_ast.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 1dafc7bc4..58011dde8 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -524,16 +524,14 @@ def test_attribute_access(): cant_compile("(. foo bar :baz [0] quux [frob])") cant_compile("(. foo bar baz (0) quux [frob])") cant_compile("(. foo bar baz [0] quux {frob})") - cant_compile("(.. foo bar baz)") -def test_attribute_empty(): - """Ensure using dot notation with a non-expression is an error""" - cant_compile(".") +def test_misplaced_dots(): cant_compile("foo.") - cant_compile(".foo") - cant_compile('"bar".foo') - cant_compile("[2].foo") + cant_compile("foo..") + cant_compile("foo.bar.") + cant_compile("foo.bar..") + cant_compile("foo..bar") def test_bad_setv(): From d77eb8a254c228aa2f38fd025d013369f81e6bd2 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 17:15:21 -0500 Subject: [PATCH 100/342] Test parsing dotted identifiers --- tests/test_reader.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_reader.py b/tests/test_reader.py index 52a05edd4..db7272e0b 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -272,6 +272,17 @@ def test_leading_zero(): assert tokenize("010+000010j") == [Complex(10 + 10j)] +def test_dotted_identifiers(): + t = tokenize + + assert t("foo.bar") == t("(. foo bar)") + assert t("foo.bar.baz") == t("(. foo bar baz)") + assert t(".foo") == t("(. None foo)") + assert t(".foo.bar.baz") == t("(. None foo bar baz)") + assert t("..foo") == t("(.. None foo)") + assert t("..foo.bar.baz") == t("(.. None foo bar baz)") + + def test_lex_bad_attrs(): with lexe() as execinfo: tokenize("1.foo") From 75e5f9601563fb41d454be26980d13e3e3288ae5 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Jan 2023 17:20:55 -0500 Subject: [PATCH 101/342] Test defining `..` and `....` --- tests/native_tests/language.hy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 9d5a4164e..efb57791c 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -438,6 +438,18 @@ (assert (= (.__str__ :foo) ":foo"))) +(defn test-multidot [] + (setv a 1 b 2 c 3) + + (defn .. [#* args] + (.join "~" (map str args))) + (assert (= ..a.b.c "None~1~2~3")) + + (defmacro .... [#* args] + (.join "@" (map str args))) + (assert (= ....uno.dos.tres "None@uno@dos@tres"))) + + (defn test-do [] (do)) From e7a5f4db3689682ea9051199f32b969bff0d35f6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 3 Jan 2023 08:09:57 -0500 Subject: [PATCH 102/342] Update NEWS --- NEWS.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 4f61d7c69..9302226b5 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -14,6 +14,10 @@ Breaking Changes the case for `'`.) * The bitwise NOT operator `~` has been renamed to `bnot`. +* Dotted identifiers like `foo.bar` and `.sqrt` now parse as + expressions (like `(. foo bar)` and `(. None sqrt)`) instead of + symbols. Some odd cases like `foo.` and `foo..bar` are now + syntactically illegal. * `pragma` is now reserved as a core macro name, although it doesn't do anything useful (yet). * `hy.cmdline.HyREPL` is now `hy.REPL`. @@ -29,6 +33,8 @@ Bug Fixes New Features ------------------------------ +* `.`, `..`, etc. are now usable as ordinary symbols (with the + remaining special rule that `...` compiles to `Ellipsis`) * On Pythons ≥ 3.7, Hy modules can now be imported from ZIP archives in the same way as Python modules, via `zipimport`_. From e637fa57b5f42a822106a16644c0d161b9102deb Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 4 Jan 2023 21:49:54 -0500 Subject: [PATCH 103/342] Autoformat --- hy/compiler.py | 22 ++++++++++++---------- hy/core/result_macros.py | 40 +++++++++++++++++++++------------------- hy/macros.py | 6 +++--- hy/reader/hy_reader.py | 20 +++++++++++++------- 4 files changed, 49 insertions(+), 39 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 8f6489483..3775051e0 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -533,11 +533,12 @@ def compile_expression(self, expr, *, allow_annotation_expression=False): func = None if ( - isinstance(root, Expression) and - len(root) >= 2 and - isinstance(root[0], Symbol) and - not str(root[0]).strip(".") and - root[1] == Symbol("None")): + isinstance(root, Expression) + and len(root) >= 2 + and isinstance(root[0], Symbol) + and not str(root[0]).strip(".") + and root[1] == Symbol("None") + ): # ((. None a1 a2) obj v1 v2) -> ((. obj a1 a2) v1 v2) # (The reader already parsed `.a1.a2` as `(. None a1 a2)`.) @@ -549,7 +550,9 @@ def compile_expression(self, expr, *, allow_annotation_expression=False): break i += 2 elif is_unpack("iterable", args[i]): - raise self._syntax_error(args[i], "can't call a method on an `unpack-iterable` form") + raise self._syntax_error( + args[i], "can't call a method on an `unpack-iterable` form" + ) elif is_unpack("mapping", args[i]): i += 1 else: @@ -557,10 +560,9 @@ def compile_expression(self, expr, *, allow_annotation_expression=False): else: raise self._syntax_error(expr, "attribute access requires object") - func = self.compile(Expression([ - Symbol("."), - args.pop(i), - *root[2:]]).replace(root)) + func = self.compile( + Expression([Symbol("."), args.pop(i), *root[2:]]).replace(root) + ) if is_annotate_expression(root): # Flatten and compile the annotation expression. diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 4ba5b1818..31c0a758d 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -75,7 +75,7 @@ def maybe_annotated(target): def dotted(name): - return Expression(map(Symbol, [".", *name.split('.')])) + return Expression(map(Symbol, [".", *name.split(".")])) # ------------------------------------------------ @@ -217,9 +217,7 @@ def render_quoted_form(compiler, form, level): if form.brackets is not None: body.extend([Keyword("brackets"), form.brackets]) - return ( - Expression([dotted("hy.models." + name), *body]).replace(form), - False) + return (Expression([dotted("hy.models." + name), *body]).replace(form), False) # ------------------------------------------------ @@ -1678,16 +1676,19 @@ def compile_class_expression(compiler, expr, root, decorators, name, rest): # ------------------------------------------------ module_name_pattern = SYM | pexpr( - some(lambda x: isinstance(x, Symbol) and not str(x[0]).strip(".")) + - oneplus(SYM)) + some(lambda x: isinstance(x, Symbol) and not str(x[0]).strip(".")) + oneplus(SYM) +) + def module_name_str(x): return ( - '.'.join(map(mangle, x[1][x[1][0] == Symbol("None") :])) - if isinstance(x, Expression) + ".".join(map(mangle, x[1][x[1][0] == Symbol("None") :])) + if isinstance(x, Expression) else str(x) - if isinstance(x, Symbol) and not x.strip(".") - else mangle(x)) + if isinstance(x, Symbol) and not x.strip(".") + else mangle(x) + ) + importlike = ( keepsym("*") @@ -1842,17 +1843,18 @@ def compile_import(compiler, expr, root, entries): module, name=mangle(k), asname=None if v == k else mangle(v) ) ) - ret += node(expr, - module = module_name if module_name and module_name.strip(".") else None, - names = names, - level = ( + ret += node( + expr, + module=module_name if module_name and module_name.strip(".") else None, + names=names, + level=( len(module[0]) - if isinstance(module, Expression) and - module[1][0] == Symbol("None") + if isinstance(module, Expression) and module[1][0] == Symbol("None") else len(module) - if isinstance(module, Symbol) and - not module.strip(".") - else 0)) + if isinstance(module, Symbol) and not module.strip(".") + else 0 + ), + ) return ret diff --git a/hy/macros.py b/hy/macros.py index 70841bbe6..d04202495 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -48,9 +48,9 @@ def wrapper(hy_compiler, *args): if shadow and any(is_unpack("iterable", x) for x in args): # Try a shadow function call with this name instead. - return Expression([ - Expression(map(Symbol, [".", "hy", "pyops", name])), - *args]).replace(hy_compiler.this) + return Expression( + [Expression(map(Symbol, [".", "hy", "pyops", name])), *args] + ).replace(hy_compiler.this) expr = hy_compiler.this root = unmangle(expr[0]) diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 70ab75b0b..025adab7e 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -69,23 +69,29 @@ def as_identifier(ident, reader=None): if not ident.strip("."): # It's all dots. Return it as a symbol. return sym(ident) + def err(msg): - raise (ValueError(msg) + raise ( + ValueError(msg) if reader is None - else LexException.from_reader(msg, reader)) + else LexException.from_reader(msg, reader) + ) + if ident.lstrip(".").find("..") > 0: - err("In a dotted identifier, multiple dots in a row are only allowed at the start") + err( + "In a dotted identifier, multiple dots in a row are only allowed at the start" + ) if ident.endswith("."): err("A dotted identifier can't end with a dot") head = "." * (len(ident) - len(ident.lstrip("."))) - args = [as_identifier(a, reader=reader) - for a in ident.lstrip(".").split(".")] + args = [as_identifier(a, reader=reader) for a in ident.lstrip(".").split(".")] if any(not isinstance(a, Symbol) for a in args): err("The parts of a dotted identifier must be symbols") return ( mkexpr(sym("."), *args) - if head == '' - else mkexpr(head, Symbol("None"), *args)) + if head == "" + else mkexpr(head, Symbol("None"), *args) + ) if reader is None: if ( From fbcb8de5ad46fa852e9739184b98c4a45625a2e7 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 9 Jan 2023 15:33:56 -0500 Subject: [PATCH 104/342] Rename some test files --- tests/native_tests/{native_macros.hy => macros.hy} | 0 tests/native_tests/{with_test.hy => with.hy} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/native_tests/{native_macros.hy => macros.hy} (100%) rename tests/native_tests/{with_test.hy => with.hy} (100%) diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/macros.hy similarity index 100% rename from tests/native_tests/native_macros.hy rename to tests/native_tests/macros.hy diff --git a/tests/native_tests/with_test.hy b/tests/native_tests/with.hy similarity index 100% rename from tests/native_tests/with_test.hy rename to tests/native_tests/with.hy From 4a8ed71651d63bde70f1908e032b9010dc95c802 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 9 Jan 2023 15:18:00 -0500 Subject: [PATCH 105/342] Remove `test-exec` and `test-filter` As they stand, these are really just tests of Python standard functions. I think they were more meaningful tests of Hy back when we supported Python 2. --- tests/native_tests/core.hy | 63 -------------------------------------- 1 file changed, 63 deletions(-) diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy index d70e7573a..af4cd3df0 100644 --- a/tests/native_tests/core.hy +++ b/tests/native_tests/core.hy @@ -1,5 +1,4 @@ (import - itertools [repeat cycle islice] pytest) ;;;; some simple helpers @@ -51,68 +50,6 @@ (assert-equal (set (itertools.permutations [2 1 3])) p) (assert-equal (set (foopermutations [2 3 1])) p)) -(setv globalvar 1) -(defn test-exec [] - (setv localvar 1) - (setv code " -result['localvar in locals'] = 'localvar' in locals() -result['localvar in globals'] = 'localvar' in globals() -result['globalvar in locals'] = 'globalvar' in locals() -result['globalvar in globals'] = 'globalvar' in globals() -result['x in locals'] = 'x' in locals() -result['x in globals'] = 'x' in globals() -result['y in locals'] = 'y' in locals() -result['y in globals'] = 'y' in globals()") - - (setv result {}) - (exec code) - (assert-true (get result "localvar in locals")) - (assert-false (get result "localvar in globals")) - (assert-false (get result "globalvar in locals")) - (assert-true (get result "globalvar in globals")) - (assert-false (or - (get result "x in locals") (get result "x in globals") - (get result "y in locals") (get result "y in globals"))) - - (setv result {}) - (exec code {"x" 1 "result" result}) - (assert-false (or - (get result "localvar in locals") (get result "localvar in globals") - (get result "globalvar in locals") (get result "globalvar in globals"))) - (assert-true (and - (get result "x in locals") (get result "x in globals"))) - (assert-false (or - (get result "y in locals") (get result "y in globals"))) - - (setv result {}) - (exec code {"x" 1 "result" result} {"y" 1}) - (assert-false (or - (get result "localvar in locals") (get result "localvar in globals") - (get result "globalvar in locals") (get result "globalvar in globals"))) - (assert-false (get result "x in locals")) - (assert-true (get result "x in globals")) - (assert-true (get result "y in locals")) - (assert-false (get result "y in globals"))) - -(defn test-filter [] - (setv res (list (filter (fn [x] (> x 0)) [ 1 2 3 -4 5]))) - (assert-equal res [ 1 2 3 5 ]) - ;; test with iter - (setv res (list (filter (fn [x] (> x 0)) (iter [ 1 2 3 -4 5 -6])))) - (assert-equal res [ 1 2 3 5]) - (setv res (list (filter (fn [x] (< x 0)) [ -1 -4 5 3 4]))) - (assert-false (= res [1 2])) - ;; test with empty list - (setv res (list (filter (fn [x] (< x 0)) []))) - (assert-equal res []) - ;; test with None in the list - (setv res (list - (filter (fn [x] (not (% x 2))) - (filter (fn [x] (isinstance x int)) - [1 2 None 3 4 None 4 6])))) - (assert-equal res [2 4 4 6]) - (setv res (list (filter (fn [x] (is x None)) [1 2 None 3 4 None 4 6]))) - (assert-equal res [None None])) (defn test-gensym [] (setv s1 (hy.gensym)) From 55d1449e527385d5a150414670be127b44dae018 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 5 Jan 2023 15:06:10 -0500 Subject: [PATCH 106/342] Use plain `assert` in tests intead of `assert-*` --- tests/native_tests/core.hy | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy index af4cd3df0..934140d70 100644 --- a/tests/native_tests/core.hy +++ b/tests/native_tests/core.hy @@ -1,20 +1,6 @@ (import pytest) -;;;; some simple helpers - -(defn assert-true [x] - (assert (= True x))) - -(defn assert-false [x] - (assert (= False x))) - -(defn assert-equal [x y] - (assert (= x y))) - -(defn assert-none [x] - (assert (is x None))) - (defn assert-requires-num [f] (for [x ["foo" [] None]] (try (f x) @@ -24,15 +10,15 @@ (defn test-setv [] (setv x 1) (setv y 1) - (assert-equal x y) + (assert (= x y)) (setv y 12) (setv x y) - (assert-equal x 12) - (assert-equal y 12) + (assert (= x 12)) + (assert (= y 12)) (setv y (fn [x] 9)) (setv x y) - (assert-equal (x y) 9) - (assert-equal (y x) 9) + (assert (= (x y) 9)) + (assert (= (y x) 9)) (try (do (setv a.b 1) (assert False)) (except [e [NameError]] (assert (in "name 'a' is not defined" (str e))))) (try (do (setv b.a (fn [x] x)) (assert False)) @@ -40,15 +26,15 @@ (import itertools) (setv foopermutations (fn [x] (itertools.permutations x))) (setv p (set [#(1 3 2) #(3 2 1) #(2 1 3) #(3 1 2) #(1 2 3) #(2 3 1)])) - (assert-equal (set (itertools.permutations [1 2 3])) p) - (assert-equal (set (foopermutations [3 1 2])) p) + (assert (= (set (itertools.permutations [1 2 3])) p)) + (assert (= (set (foopermutations [3 1 2])) p)) (setv permutations- itertools.permutations) (setv itertools.permutations (fn [x] 9)) - (assert-equal (itertools.permutations p) 9) - (assert-equal (foopermutations foopermutations) 9) + (assert (= (itertools.permutations p) 9)) + (assert (= (foopermutations foopermutations) 9)) (setv itertools.permutations permutations-) - (assert-equal (set (itertools.permutations [2 1 3])) p) - (assert-equal (set (foopermutations [2 3 1])) p)) + (assert (= (set (itertools.permutations [2 1 3])) p)) + (assert (= (set (foopermutations [2 3 1])) p))) (defn test-gensym [] From a1ccb9dc78b9a4cee3a773b5e200203e5819e238 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 9 Jan 2023 15:17:39 -0500 Subject: [PATCH 107/342] Remove an unused helper function --- tests/native_tests/core.hy | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy index 934140d70..36859b231 100644 --- a/tests/native_tests/core.hy +++ b/tests/native_tests/core.hy @@ -1,11 +1,6 @@ (import pytest) -(defn assert-requires-num [f] - (for [x ["foo" [] None]] - (try (f x) - (except [TypeError] True) - (else (assert False))))) (defn test-setv [] (setv x 1) From ec5a7a1184294bc5d6daddb220f8c652a67251fb Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 9 Jan 2023 15:22:54 -0500 Subject: [PATCH 108/342] Convert some native tests to reader tests I just deleted `test-sets`, because `test_reader.py` already has a more thorough `test_sets`. --- tests/native_tests/language.hy | 33 --------------------------------- tests/test_reader.py | 22 +++++++++++++++++++--- 2 files changed, 19 insertions(+), 36 deletions(-) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index efb57791c..a8bdc2a69 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -24,33 +24,6 @@ (assert (isinstance sys.argv list))) -(defn test-hex [] - (assert (= 0x80 128))) - - -(defn test-octal [] - (assert (= 0o1232 666))) - - -(defn test-binary [] - (assert (= 0b1011101 93))) - - -(defn test-lists [] - (assert (= [1 2 3 4] (+ [1 2] [3 4])))) - - -(defn test-dicts [] - (assert (= {1 2 3 4} {3 4 1 2})) - (assert (= {1 2 3 4} {1 (+ 1 1) 3 (+ 2 2)}))) - - -(defn test-sets [] - (assert (= #{1 2 3 4} (| #{1 2} #{3 4}))) - (assert (= (type #{1 2 3 4}) set)) - (assert (= #{} (set)))) - - (defn test-setv-get [] (setv foo [0 1 2]) (setv (get foo 0) 12) @@ -1562,12 +1535,6 @@ cee"} dee" "ey bee\ncee dee")) (= (identify-keywords 1 "bloo" :foo) ["other" "other" "keyword"]))) -(defn test-underscore_variables [] - ; https://github.com/hylang/hy/issues/1340 - (defclass XYZ [] - (setv _42 6)) - (setv x (XYZ)) - (assert (= (. x _42) 6))) (defn test-docstrings [] (defn f [] "docstring" 5) diff --git a/tests/test_reader.py b/tests/test_reader.py index db7272e0b..29f98649a 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -178,9 +178,10 @@ def test_lex_bracket_strings(): def test_lex_integers(): - """Make sure that integers are valid expressions""" - objs = tokenize("42 ") - assert objs == [Integer(42)] + assert tokenize("42") == [Integer(42)] + assert tokenize("0x80") == [Integer(128)] + assert tokenize("0o1232") == [Integer(666)] + assert tokenize("0b1011101") == [Integer(93)] def test_lex_expression_float(): @@ -242,6 +243,8 @@ def test_lex_digit_separators(): assert tokenize("1,000,000") == [Integer(1000000)] assert tokenize("1,000_000") == [Integer(1000000)] assert tokenize("1_000,000") == [Integer(1000000)] + # https://github.com/hylang/hy/issues/1340 + assert tokenize("_42") == [Symbol("_42")] assert tokenize("0x_af") == [Integer(0xAF)] assert tokenize("0x,af") == [Integer(0xAF)] @@ -313,6 +316,19 @@ def test_lex_bad_attrs(): tokenize(":hello.foo") +def test_lists(): + assert tokenize("[1 2 3 4]") == [List(map(Integer, (1, 2, 3, 4)))] + + +def test_dicts(): + assert tokenize("{1 2 3 4}") == [Dict(map(Integer, (1, 2, 3, 4)))] + assert tokenize("{1 (+ 1 1) 3 (+ 2 2)}") == [Dict(( + Integer(1), + Expression((Symbol("+"), Integer(1), Integer(1))), + Integer(3), + Expression((Symbol("+"), Integer(2), Integer(2)))))] + + def test_lex_column_counting(): entry = tokenize("(foo (one two))")[0] assert entry.start_line == 1 From dd4a5adaa9740596b4cbc89a2f1a4aefee46ec05 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 8 Jan 2023 08:47:58 -0500 Subject: [PATCH 109/342] Remove `test-index` It's basically redundant with `op-and-shadow-test [get]`. --- tests/native_tests/language.hy | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index a8bdc2a69..96bc21921 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -330,19 +330,6 @@ (defn test-if [] (assert (= 1 (if 0 -1 1)))) -(defn test-index [] - (assert (= (get {"one" "two"} "one") "two")) - (assert (= (get [1 2 3 4 5] 1) 2)) - (assert (= (get {"first" {"second" {"third" "level"}}} - "first" "second" "third") - "level")) - (assert (= (get ((fn [] {"first" {"second" {"third" "level"}}})) - "first" "second" "third") - "level")) - (assert (= (get {"first" {"second" {"third" "level"}}} - ((fn [] "first")) "second" "third") - "level"))) - (defn test-fn [] (setv square (fn [x] (* x x))) From c5f52b39555f9dceeb2c72f3274c652078c92cad Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 9 Jan 2023 16:41:34 -0500 Subject: [PATCH 110/342] Remove `test-sys-argv` Command-line arguments are tested better elsewhere (like in `test_bin.py`), and inline comments are used all over the place in the native tests. --- tests/native_tests/language.hy | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 96bc21921..0f9887e28 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -19,11 +19,6 @@ `(~@expr ~x)) -(defn test-sys-argv [] - ;; BTW, this also tests inline comments. Which suck to implement. - (assert (isinstance sys.argv list))) - - (defn test-setv-get [] (setv foo [0 1 2]) (setv (get foo 0) 12) From d3d8fe5c62d9366cb8e7dec4929d0d93704ac33a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 9 Jan 2023 16:53:03 -0500 Subject: [PATCH 111/342] Reorganize lots of native tests Having smaller number of tests per file should make it easier to investigate compilation crashes. --- .../{language_beside.hy => beside.hy} | 2 +- tests/native_tests/break_continue.hy | 12 + tests/native_tests/comprehensions.hy | 51 + tests/native_tests/conditional.hy | 262 +++ tests/native_tests/core.hy | 79 - tests/native_tests/cut.hy | 5 + tests/native_tests/decorators.hy | 13 + tests/native_tests/defclass.hy | 36 + tests/native_tests/del.hy | 16 + tests/native_tests/do.hy | 7 + tests/native_tests/doc.hy | 35 + tests/native_tests/dots.hy | 79 + tests/native_tests/eval_foo_compile.hy | 17 + tests/native_tests/functions.hy | 280 +++ tests/native_tests/hy_eval.hy | 86 + tests/native_tests/hy_misc.hy | 88 + tests/native_tests/import.hy | 304 +++ tests/native_tests/keywords.hy | 99 + tests/native_tests/language.hy | 1757 ----------------- tests/native_tests/logic_short_circuit.hy | 83 + tests/native_tests/macros.hy | 192 +- tests/native_tests/operators.hy | 6 + tests/native_tests/other.hy | 52 + tests/native_tests/setv.hy | 91 + tests/native_tests/strings.hy | 166 ++ tests/native_tests/try.hy | 219 ++ tests/native_tests/unpack.hy | 41 + tests/native_tests/when.hy | 10 - tests/native_tests/with.hy | 49 +- 29 files changed, 2105 insertions(+), 2032 deletions(-) rename tests/native_tests/{language_beside.hy => beside.hy} (69%) create mode 100644 tests/native_tests/break_continue.hy create mode 100644 tests/native_tests/conditional.hy delete mode 100644 tests/native_tests/core.hy create mode 100644 tests/native_tests/cut.hy create mode 100644 tests/native_tests/del.hy create mode 100644 tests/native_tests/do.hy create mode 100644 tests/native_tests/doc.hy create mode 100644 tests/native_tests/dots.hy create mode 100644 tests/native_tests/eval_foo_compile.hy create mode 100644 tests/native_tests/functions.hy create mode 100644 tests/native_tests/hy_eval.hy create mode 100644 tests/native_tests/hy_misc.hy create mode 100644 tests/native_tests/import.hy create mode 100644 tests/native_tests/keywords.hy delete mode 100644 tests/native_tests/language.hy create mode 100644 tests/native_tests/logic_short_circuit.hy create mode 100644 tests/native_tests/other.hy create mode 100644 tests/native_tests/setv.hy create mode 100644 tests/native_tests/strings.hy create mode 100644 tests/native_tests/try.hy create mode 100644 tests/native_tests/unpack.hy delete mode 100644 tests/native_tests/when.hy diff --git a/tests/native_tests/language_beside.hy b/tests/native_tests/beside.hy similarity index 69% rename from tests/native_tests/language_beside.hy rename to tests/native_tests/beside.hy index 6594c5a85..cbe3a3206 100644 --- a/tests/native_tests/language_beside.hy +++ b/tests/native_tests/beside.hy @@ -1,5 +1,5 @@ ;; This file has no tests of its own, and only exists to be required -;; by `language.hy`, and sit in the same directory to test +;; by `import.hy`, and sit in the same directory to test ;; single-period relative `require`. (defmacro xyzzy [] diff --git a/tests/native_tests/break_continue.hy b/tests/native_tests/break_continue.hy new file mode 100644 index 000000000..5a422f613 --- /dev/null +++ b/tests/native_tests/break_continue.hy @@ -0,0 +1,12 @@ +(defn test-break-breaking [] + (defn holy-grail [] (for [x (range 10)] (when (= x 5) (break))) x) + (assert (= (holy-grail) 5))) + + +(defn test-continue-continuation [] + (setv y []) + (for [x (range 10)] + (when (!= x 5) + (continue)) + (.append y x)) + (assert (= y [5]))) diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy index e3bfc4bef..abeb10634 100644 --- a/tests/native_tests/comprehensions.hy +++ b/tests/native_tests/comprehensions.hy @@ -1,5 +1,6 @@ (import types + asyncio pytest) @@ -359,3 +360,53 @@ (assert (= x 2))) (bar) (assert (= x 19)))) + + +(defn test-for-do [] + (do (do (do (do (do (do (do (do (do (setv #(x y) #(0 0))))))))))) + (for [- [1 2]] + (do + (setv x (+ x 1)) + (setv y (+ y 1)))) + (assert (= y x 2))) + + +(defn test-for-else [] + (setv x 0) + (for [a [1 2]] + (setv x (+ x a)) + (else (setv x (+ x 50)))) + (assert (= x 53)) + + (setv x 0) + (for [a [1 2]] + (setv x (+ x a)) + (else)) + (assert (= x 3))) + + +(defn test-for-async [] + (defn/a numbers [] + (for [i [1 2]] + (yield i))) + + (asyncio.run + ((fn/a [] + (setv x 0) + (for [:async a (numbers)] + (setv x (+ x a))) + (assert (= x 3)))))) + + +(defn test-for-async-else [] + (defn/a numbers [] + (for [i [1 2]] + (yield i))) + + (asyncio.run + ((fn/a [] + (setv x 0) + (for [:async a (numbers)] + (setv x (+ x a)) + (else (setv x (+ x 50)))) + (assert (= x 53)))))) diff --git a/tests/native_tests/conditional.hy b/tests/native_tests/conditional.hy new file mode 100644 index 000000000..97d2cbaca --- /dev/null +++ b/tests/native_tests/conditional.hy @@ -0,0 +1,262 @@ +;; Tests of `if`, `cond`, `when`, and `while` + +(import + pytest) + + +(defn test-branching [] + (if True + (assert (= 1 1)) + (assert (= 2 1)))) + + +(defn test-branching-with-do [] + (if False + (assert (= 2 1)) + (do + (assert (= 1 1)) + (assert (= 1 1)) + (assert (= 1 1))))) + + +(defn test-branching-expr-count-with-do [] + "Ensure we execute the right number of expressions in a branch." + (setv counter 0) + (if False + (assert (= 2 1)) + (do + (setv counter (+ counter 1)) + (setv counter (+ counter 1)) + (setv counter (+ counter 1)))) + (assert (= counter 3))) + + +(defn test-cond [] + (cond + (= 1 2) (assert (is True False)) + (is None None) (do (setv x True) (assert x))) + (assert (is (cond) None)) + + (assert (= (cond + False 1 + [] 2 + True 8) 8)) + + (setv x 0) + (assert (is (cond False 1 [] 2 x 3) None)) + + (with [e (pytest.raises hy.errors.HyMacroExpansionError)] + (hy.eval '(cond 1))) + (assert (in "needs an even number of arguments" e.value.msg)) + + ; Make sure each test is only evaluated once, and `cond` + ; short-circuits. + (setv x 1) + (assert (= "first" (cond + (do (*= x 2) True) (do (*= x 3) "first") + (do (*= x 5) True) (do (*= x 7) "second")))) + (assert (= x 6))) + + +(defn test-if [] + (assert (= 1 (if 0 -1 1)))) + + +(defn test-returnable-ifs [] + (assert (= True (if True True True)))) + + +(defn test-if-return-branching [] + ; thanks, kirbyfan64 + (defn f [] + (if True (setv x 1) 2) + 1) + + (assert (= 1 (f)))) + + +(defn test-nested-if [] + (for [x (range 10)] + (if (in "foo" "foobar") + (do + (if True True True)) + (do + (if False False False))))) + + +(defn test-if-in-if [] + (assert (= 42 + (if (if 1 True False) + 42 + 43))) + (assert (= 43 + (if (if 0 True False) + 42 + 43)))) + + +(defn test-when [] + (assert (= (when True 1) 1)) + (assert (= (when True 1 2) 2)) + (assert (= (when True 1 3) 3)) + (assert (= (when False 2) None)) + (assert (= (when (= 1 2) 42) None)) + (assert (= (when (= 2 2) 42) 42)) + + (assert (is (when (do (setv x 3) True)) None)) + (assert (= x 3))) + + +(defn test-while-loop [] + (setv count 5) + (setv fact 1) + (while (> count 0) + (setv fact (* fact count)) + (setv count (- count 1))) + (assert (= count 0)) + (assert (= fact 120)) + + (setv l []) + (defn f [] + (.append l 1) + (len l)) + (while (!= (f) 4)) + (assert (= l [1 1 1 1])) + + (setv l []) + (defn f [] + (.append l 1) + (len l)) + (while (!= (f) 4) (do)) + (assert (= l [1 1 1 1])) + + ; only compile the condition once + ; https://github.com/hylang/hy/issues/1790 + (global while-cond-var) + (setv while-cond-var 10) + (hy.eval + '(do + (defmacro while-cond [] + (global while-cond-var) + (assert (= while-cond-var 10)) + (+= while-cond-var 1) + `(do + (setv x 3) + False)) + (while (while-cond)) + (assert (= x 3))))) + + +(defn test-while-loop-else [] + (setv count 5) + (setv fact 1) + (setv myvariable 18) + (while (> count 0) + (setv fact (* fact count)) + (setv count (- count 1)) + (else (setv myvariable 26))) + (assert (= count 0)) + (assert (= fact 120)) + (assert (= myvariable 26)) + + ; multiple statements in a while loop should work + (setv count 5) + (setv fact 1) + (setv myvariable 18) + (setv myothervariable 15) + (while (> count 0) + (setv fact (* fact count)) + (setv count (- count 1)) + (else (setv myvariable 26) + (setv myothervariable 24))) + (assert (= count 0)) + (assert (= fact 120)) + (assert (= myvariable 26)) + (assert (= myothervariable 24)) + + ; else clause shouldn't get run after a break + (while True + (break) + (else (setv myvariable 53))) + (assert (= myvariable 26)) + + ; don't be fooled by constructs that look like else clauses + (setv x 2) + (setv a []) + (setv else True) + (while x + (.append a x) + (-= x 1) + [else (.append a "e")]) + (assert (= a [2 "e" 1 "e"])) + + (setv x 2) + (setv a []) + (with [(pytest.raises TypeError)] + (while x + (.append a x) + (-= x 1) + ("else" (.append a "e")))) + (assert (= a [2 "e"]))) + + +(defn test-while-multistatement-condition [] + + ; The condition should be executed every iteration, before the body. + ; `else` should be executed last. + (setv s "") + (setv x 2) + (while (do (+= s "a") x) + (+= s "b") + (-= x 1) + (else + (+= s "z"))) + (assert (= s "ababaz")) + + ; `else` should still be skipped after `break`. + (setv s "") + (setv x 2) + (while (do (+= s "a") x) + (+= s "b") + (-= x 1) + (when (= x 0) + (break)) + (else + (+= s "z"))) + (assert (= s "abab")) + + ; `continue` should jump to the condition. + (setv s "") + (setv x 2) + (setv continued? False) + (while (do (+= s "a") x) + (+= s "b") + (when (and (= x 1) (not continued?)) + (+= s "c") + (setv continued? True) + (continue)) + (-= x 1) + (else + (+= s "z"))) + (assert (= s "ababcabaz")) + + ; `break` in a condition applies to the `while`, not an outer loop. + (setv s "") + (for [x "123"] + (+= s x) + (setv y 0) + (while (do (when (and (= x "2") (= y 1)) (break)) (< y 3)) + (+= s "y") + (+= y 1))) + (assert (= s "1yyy2y3yyy")) + + ; The condition is still tested appropriately if its last variable + ; is set to a false value in the loop body. + (setv out []) + (setv x 0) + (setv a [1 1]) + (while (do (.append out 2) (setv x (and a (.pop a))) x) + (setv x 0) + (.append out x)) + (assert (= out [2 0 2 0 2])) + (assert (is x a))) diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy deleted file mode 100644 index 36859b231..000000000 --- a/tests/native_tests/core.hy +++ /dev/null @@ -1,79 +0,0 @@ -(import - pytest) - - -(defn test-setv [] - (setv x 1) - (setv y 1) - (assert (= x y)) - (setv y 12) - (setv x y) - (assert (= x 12)) - (assert (= y 12)) - (setv y (fn [x] 9)) - (setv x y) - (assert (= (x y) 9)) - (assert (= (y x) 9)) - (try (do (setv a.b 1) (assert False)) - (except [e [NameError]] (assert (in "name 'a' is not defined" (str e))))) - (try (do (setv b.a (fn [x] x)) (assert False)) - (except [e [NameError]] (assert (in "name 'b' is not defined" (str e))))) - (import itertools) - (setv foopermutations (fn [x] (itertools.permutations x))) - (setv p (set [#(1 3 2) #(3 2 1) #(2 1 3) #(3 1 2) #(1 2 3) #(2 3 1)])) - (assert (= (set (itertools.permutations [1 2 3])) p)) - (assert (= (set (foopermutations [3 1 2])) p)) - (setv permutations- itertools.permutations) - (setv itertools.permutations (fn [x] 9)) - (assert (= (itertools.permutations p) 9)) - (assert (= (foopermutations foopermutations) 9)) - (setv itertools.permutations permutations-) - (assert (= (set (itertools.permutations [2 1 3])) p)) - (assert (= (set (foopermutations [2 3 1])) p))) - - -(defn test-gensym [] - (setv s1 (hy.gensym)) - (assert (isinstance s1 hy.models.Symbol)) - (assert (= 0 (.find s1 "_G\uffff"))) - (setv s2 (hy.gensym "xx")) - (setv s3 (hy.gensym "xx")) - (assert (= 0 (.find s2 "_xx\uffff"))) - (assert (not (= s2 s3))) - (assert (not (= (str s2) (str s3))))) - -(defn test-import-init-hy [] - (import tests.resources.bin) - (assert (in "_null_fn_for_import_test" (dir tests.resources.bin)))) - -(defreader some-tag - "Some tag macro" - '1) - -(defn test-doc [capsys] - ;; https://github.com/hylang/hy/issues/1970 - ;; Let's first make sure we can doc the builtin macros - ;; before we create the user macros. - (doc doc) - (setv [out err] (.readouterr capsys)) - (assert (in "Gets help for a macro function" out)) - - (doc "#some-tag") - (setv [out err] (.readouterr capsys)) - (assert (in "Some tag macro" out)) - - (defmacro <-mangle-> [] - "a fancy docstring" - '(+ 2 2)) - (doc <-mangle->) - (setv [out err] (.readouterr capsys)) - ;; https://github.com/hylang/hy/issues/1946 - (assert (.startswith (.strip out) - f"Help on function {(hy.mangle '<-mangle->)} in module ")) - (assert (in "a fancy docstring" out)) - (assert (not err)) - - ;; make sure doc raises an error instead of - ;; presenting a default value help screen - (with [(pytest.raises NameError)] - (doc does-not-exist))) diff --git a/tests/native_tests/cut.hy b/tests/native_tests/cut.hy new file mode 100644 index 000000000..999a5246c --- /dev/null +++ b/tests/native_tests/cut.hy @@ -0,0 +1,5 @@ +(defn test-cut [] + (assert (= (cut [1 2 3 4 5] 3) [1 2 3])) + (assert (= (cut [1 2 3 4 5] 1 None) [2 3 4 5])) + (assert (= (cut [1 2 3 4 5] 1 3) [2 3])) + (assert (= (cut [1 2 3 4 5]) [1 2 3 4 5]))) diff --git a/tests/native_tests/decorators.hy b/tests/native_tests/decorators.hy index 4b79d1f4d..9997c6b6d 100644 --- a/tests/native_tests/decorators.hy +++ b/tests/native_tests/decorators.hy @@ -1,3 +1,7 @@ +(import + asyncio) + + (defn test-decorated-1line-function [] (defn foodec [func] (fn [] (+ (func) 1))) @@ -46,3 +50,12 @@ (.append l "bar body") arg) ; Body (.append l (bar)) (assert (= l ["dec" "arg" "foo" "foo fn" "bar body" 1]))) + + +(defn test-decorated-defn/a [] + (defn decorator [func] (fn/a [] (/ (await (func)) 2))) + + (defn/a [decorator] coro-test [] + (await (asyncio.sleep 0)) + 42) + (assert (= (asyncio.run (coro-test)) 21))) diff --git a/tests/native_tests/defclass.hy b/tests/native_tests/defclass.hy index ef9b4f29d..30aae9732 100644 --- a/tests/native_tests/defclass.hy +++ b/tests/native_tests/defclass.hy @@ -112,3 +112,39 @@ (set-sentinel)) (assert set-sentinel.set)) + + +(defn test-pep-3115 [] + (defclass member-table [dict] + (defn __init__ [self] + (setv self.member-names [])) + + (defn __setitem__ [self key value] + (when (not-in key self) + (.append self.member-names key)) + (dict.__setitem__ self key value))) + + (defclass OrderedClass [type] + (setv __prepare__ (classmethod (fn [metacls name bases] + (member-table)))) + + (defn __new__ [cls name bases classdict] + (setv result (type.__new__ cls name bases (dict classdict))) + (setv result.member-names classdict.member-names) + result)) + + (defclass MyClass [:metaclass OrderedClass] + (defn method1 [self] (pass)) + (defn method2 [self] (pass))) + + (assert (= (. (MyClass) member-names) + ["__module__" "__qualname__" "method1" "method2"]))) + + +(defn test-pep-487 [] + (defclass QuestBase [] + (defn __init-subclass__ [cls swallow #** kwargs] + (setv cls.swallow swallow))) + + (defclass Quest [QuestBase :swallow "african"]) + (assert (= (. (Quest) swallow) "african"))) diff --git a/tests/native_tests/del.hy b/tests/native_tests/del.hy new file mode 100644 index 000000000..6dc927db9 --- /dev/null +++ b/tests/native_tests/del.hy @@ -0,0 +1,16 @@ +(import + pytest) + + +(defn test-del [] + (setv foo 42) + (assert (= foo 42)) + (del foo) + (with [(pytest.raises NameError)] + foo) + (setv test (list (range 5))) + (del (get test 4)) + (assert (= test [0 1 2 3])) + (del (get test 2)) + (assert (= test [0 1 3])) + (assert (= (del) None))) diff --git a/tests/native_tests/do.hy b/tests/native_tests/do.hy new file mode 100644 index 000000000..957d87095 --- /dev/null +++ b/tests/native_tests/do.hy @@ -0,0 +1,7 @@ +(defn test-do [] + (do)) + + +(defn test-pass [] + (if True (do) (do)) + (assert (= 1 1))) diff --git a/tests/native_tests/doc.hy b/tests/native_tests/doc.hy new file mode 100644 index 000000000..147ef3962 --- /dev/null +++ b/tests/native_tests/doc.hy @@ -0,0 +1,35 @@ +(import + pytest) + + +(defreader some-tag + "Some tag macro" + '1) + +(defn test-doc [capsys] + ;; https://github.com/hylang/hy/issues/1970 + ;; Let's first make sure we can doc the builtin macros + ;; before we create the user macros. + (doc doc) + (setv [out err] (.readouterr capsys)) + (assert (in "Gets help for a macro function" out)) + + (doc "#some-tag") + (setv [out err] (.readouterr capsys)) + (assert (in "Some tag macro" out)) + + (defmacro <-mangle-> [] + "a fancy docstring" + '(+ 2 2)) + (doc <-mangle->) + (setv [out err] (.readouterr capsys)) + ;; https://github.com/hylang/hy/issues/1946 + (assert (.startswith (.strip out) + f"Help on function {(hy.mangle '<-mangle->)} in module ")) + (assert (in "a fancy docstring" out)) + (assert (not err)) + + ;; make sure doc raises an error instead of + ;; presenting a default value help screen + (with [(pytest.raises NameError)] + (doc does-not-exist))) diff --git a/tests/native_tests/dots.hy b/tests/native_tests/dots.hy new file mode 100644 index 000000000..e3fa2a29a --- /dev/null +++ b/tests/native_tests/dots.hy @@ -0,0 +1,79 @@ +(import + os) + + +(defn dotted [] + (assert (= (.join " " ["one" "two"]) "one two")) + + (defclass X [object] []) + (defclass M [object] + (defn meth [self #* args #** kwargs] + (.join " " (+ #("meth") args + (tuple (map (fn [k] (get kwargs k)) (sorted (.keys kwargs)))))))) + + (setv x (X)) + (setv m (M)) + + (assert (= (.meth m) "meth")) + (assert (= (.meth m "foo" "bar") "meth foo bar")) + (assert (= (.meth :b "1" :a "2" m "foo" "bar") "meth foo bar 2 1")) + (assert (= (.meth m #* ["foo" "bar"]) "meth foo bar")) + + (setv x.p m) + (assert (= (.p.meth x) "meth")) + (assert (= (.p.meth x "foo" "bar") "meth foo bar")) + (assert (= (.p.meth :b "1" :a "2" x "foo" "bar") "meth foo bar 2 1")) + (assert (= (.p.meth x #* ["foo" "bar"]) "meth foo bar")) + + (setv x.a (X)) + (setv x.a.b m) + (assert (= (.a.b.meth x) "meth")) + (assert (= (.a.b.meth x "foo" "bar") "meth foo bar")) + (assert (= (.a.b.meth :b "1" :a "2" x "foo" "bar") "meth foo bar 2 1")) + (assert (= (.a.b.meth x #* ["foo" "bar"]) "meth foo bar")) + + (assert (= (.__str__ :foo) ":foo"))) + + +(defn test-attribute-access [] + (defclass mycls [object]) + + (setv foo [(mycls) (mycls) (mycls)]) + (assert (is (. foo) foo)) + (assert (is (. foo [0]) (get foo 0))) + (assert (is (. foo [0] __class__) mycls)) + (assert (is (. foo [1] __class__) mycls)) + (assert (is (. foo [(+ 1 1)] __class__) mycls)) + (assert (= (. foo [(+ 1 1)] __class__ __name__ [0]) "m")) + (assert (= (. foo [(+ 1 1)] __class__ __name__ [1]) "y")) + (assert (= (. os (getcwd) (isalpha) __class__ __name__ [0]) "b")) + (assert (= (. "ab hello" (strip "ab ") (upper)) "HELLO")) + (assert (= (. "hElLO\twoRld" (expandtabs :tabsize 4) (lower)) "hello world")) + + (setv bar (mycls)) + (setv (. foo [1]) bar) + (assert (is bar (get foo 1))) + (setv (. foo [1] test) "hello") + (assert (= (getattr (. foo [1]) "test") "hello"))) + + +(defn test-multidot [] + (setv a 1 b 2 c 3) + + (defn .. [#* args] + (.join "~" (map str args))) + (assert (= ..a.b.c "None~1~2~3")) + + (defmacro .... [#* args] + (.join "@" (map str args))) + (assert (= ....uno.dos.tres "None@uno@dos@tres"))) + + +(defn test-ellipsis [] + (global Ellipsis) + (assert (is ... Ellipsis)) + (setv e Ellipsis) + (setv Ellipsis 14) + (assert (= Ellipsis 14)) + (assert (!= ... 14)) + (assert (is ... e))) diff --git a/tests/native_tests/eval_foo_compile.hy b/tests/native_tests/eval_foo_compile.hy new file mode 100644 index 000000000..bb14cab6e --- /dev/null +++ b/tests/native_tests/eval_foo_compile.hy @@ -0,0 +1,17 @@ +;; Tests of `eval-when-compile` and `eval-and-compile` + + +(defn test-eval-foo-compile-return-values [] + (eval-and-compile (setv jim 0)) + + (setv derrick (eval-and-compile (+= jim 1) 2)) + (assert (= jim 1)) + (assert (= derrick 2)) + + (setv derrick (eval-and-compile)) + (assert (is derrick None)) + + (setv derrick 3) + (setv derrick (eval-when-compile (+= jim 1) 2)) + (assert (= jim 1)) + (assert (is derrick None))) diff --git a/tests/native_tests/functions.hy b/tests/native_tests/functions.hy new file mode 100644 index 000000000..bff25a0b3 --- /dev/null +++ b/tests/native_tests/functions.hy @@ -0,0 +1,280 @@ +;; Tests of `fn`, `defn`, `return`, and `yield` + +(import + asyncio + typing [List] + pytest) + + +(defn test-fn [] + (setv square (fn [x] (* x x))) + (assert (= 4 (square 2))) + (setv lambda_list (fn [test #* args] #(test args))) + (assert (= #(1 #(2 3)) (lambda_list 1 2 3)))) + + +(defn test-immediately-call-lambda [] + (assert (= 2 ((fn [] (+ 1 1)))))) + + +(defn test-fn-return [] + (setv fn-test ((fn [] (fn [] (+ 1 1))))) + (assert (= (fn-test) 2)) + (setv fn-test (fn [])) + (assert (= (fn-test) None))) + + +(defn test-fn/a [] + (assert (= (asyncio.run ((fn/a [] (await (asyncio.sleep 0)) [1 2 3]))) + [1 2 3]))) + + +(defn test-defn-evaluation-order [] + (setv acc []) + (defn my-fun [] + (.append acc "Foo") + (.append acc "Bar") + (.append acc "Baz")) + (my-fun) + (assert (= acc ["Foo" "Bar" "Baz"]))) + + +(defn test-defn-return [] + (defn my-fun [x] + (+ x 1)) + (assert (= 43 (my-fun 42)))) + + +(defn test-defn-lambdakey [] + "Test defn with a `&symbol` function name." + (defn &hy [] 1) + (assert (= (&hy) 1))) + + +(defn test-defn-evaluation-order-with-do [] + (setv acc []) + (defn my-fun [] + (do + (.append acc "Foo") + (.append acc "Bar") + (.append acc "Baz"))) + (my-fun) + (assert (= acc ["Foo" "Bar" "Baz"]))) + + +(defn test-defn-do-return [] + (defn my-fun [x] + (do + (+ x 42) ; noop + (+ x 1))) + (assert (= 43 (my-fun 42)))) + + +(defn test-defn-dunder-name [] + "`defn` should preserve `__name__`." + + (defn phooey [x] + (+ x 1)) + (assert (= phooey.__name__ "phooey")) + + (defn mooey [x] + (+= x 1) + x) + (assert (= mooey.__name__ "mooey"))) + + +(defn test-defn-annotations [] + + (defn #^int f [#^(get List int) p1 p2 #^str p3 #^str [o1 None] #^int [o2 0] + #^str #* rest #^str k1 #^int [k2 0] #^bool #** kwargs]) + + (assert (is (. f __annotations__ ["return"]) int)) + (for [[k v] (.items (dict + :p1 (get List int) :p3 str :o1 str :o2 int + :k1 str :k2 int :kwargs bool))] + (assert (= (. f __annotations__ [k]) v)))) + + +(defn test-lambda-keyword-lists [] + (defn foo [x #* xs #** kw] [x xs kw]) + (assert (= (foo 10 20 30) [10 #(20 30) {}]))) + + +(defn test-optional-arguments [] + (defn foo [a b [c None] [d 42]] [a b c d]) + (assert (= (foo 1 2) [1 2 None 42])) + (assert (= (foo 1 2 3) [1 2 3 42])) + (assert (= (foo 1 2 3 4) [1 2 3 4]))) + + +(defn test-kwonly [] + ;; keyword-only with default works + (defn kwonly-foo-default-false [* [foo False]] foo) + (assert (= (kwonly-foo-default-false) False)) + (assert (= (kwonly-foo-default-false :foo True) True)) + ;; keyword-only without default ... + (defn kwonly-foo-no-default [* foo] foo) + (with [e (pytest.raises TypeError)] + (kwonly-foo-no-default)) + (assert (in "missing 1 required keyword-only argument: 'foo'" + (. e value args [0]))) + ;; works + (assert (= (kwonly-foo-no-default :foo "quux") "quux")) + ;; keyword-only with other arg types works + (defn function-of-various-args [a b #* args foo #** kwargs] + #(a b args foo kwargs)) + (assert (= (function-of-various-args 1 2 3 4 :foo 5 :bar 6 :quux 7) + #(1 2 #(3 4) 5 {"bar" 6 "quux" 7})))) + + +(defn test-only-parse-lambda-list-in-defn [] + (with [(pytest.raises NameError)] + (setv x [#* spam] y 1))) + + +(defn test-defn/a [] + (defn/a coro-test [] + (await (asyncio.sleep 0)) + [1 2 3]) + (assert (= (asyncio.run (coro-test)) [1 2 3]))) + + +(defn test-return [] + + ; `return` in main line + (defn f [x] + (return (+ x "a")) + (+ x "b")) + (assert (= (f "q") "qa")) + + ; Nullary `return` + (defn f [x] + (return) + 5) + (assert (is (f "q") None)) + + ; `return` in `when` + (defn f [x] + (when (< x 3) + (return [x 1])) + [x 2]) + (assert (= (f 2) [2 1])) + (assert (= (f 4) [4 2])) + + ; `return` in a loop + (setv accum []) + (defn f [x] + (while True + (when (= x 0) + (return)) + (.append accum x) + (-= x 1)) + (.append accum "this should never be appended") + 1) + (assert (is (f 5) None)) + (assert (= accum [5 4 3 2 1])) + + ; `return` of a `do` + (setv accum []) + (defn f [] + (return (do + (.append accum 1) + 3)) + 4) + (assert (= (f) 3)) + (assert (= accum [1])) + + ; `return` of an `if` that will need to be compiled to a statement + (setv accum []) + (defn f [x] + (return (if (= x 1) + (do + (.append accum 1) + "a") + (do + (.append accum 2) + "b"))) + "c") + (assert (= (f 2) "b")) + (assert (= accum [2]))) + + +(defn test-yield-from [] + (defn yield-from-test [] + (for [i (range 3)] + (yield i)) + (yield-from [1 2 3])) + (assert (= (list (yield-from-test)) [0 1 2 1 2 3]))) + + +(defn test-yield-from-exception-handling [] + (defn yield-from-subgenerator-test [] + (yield 1) + (yield 2) + (yield 3) + (/ 1 0)) + (defn yield-from-test [] + (for [i (range 3)] + (yield i)) + (try + (yield-from (yield-from-subgenerator-test)) + (except [e ZeroDivisionError] + (yield 4)))) + (assert (= (list (yield-from-test)) [0 1 2 1 2 3 4]))) + + +(defn test-yield [] + (defn gen [] (for [x [1 2 3 4]] (yield x))) + (setv ret 0) + (for [y (gen)] (setv ret (+ ret y))) + (assert (= ret 10))) + + +(defn test-yield-with-return [] + (defn gen [] (yield 3) "goodbye") + (setv gg (gen)) + (assert (= 3 (next gg))) + (with [e (pytest.raises StopIteration)] + (next gg)) + (assert (= e.value.value "goodbye"))) + + +(defn test-yield-in-try [] + (defn gen [] + (setv x 1) + (try (yield x) + (finally (print x)))) + (setv output (list (gen))) + (assert (= [1] output))) + + +(defn test-midtree-yield [] + "Test yielding with a returnable." + (defn kruft [] (yield) (+ 1 1))) + + +(defn test-midtree-yield-in-for [] + "Test yielding in a for with a return." + (defn kruft-in-for [] + (for [i (range 5)] + (yield i)) + (+ 1 2))) + + +(defn test-midtree-yield-in-while [] + "Test yielding in a while with a return." + (defn kruft-in-while [] + (setv i 0) + (while (< i 5) + (yield i) + (setv i (+ i 1))) + (+ 2 3))) + + +(defn test-multi-yield [] + (defn multi-yield [] + (for [i (range 3)] + (yield i)) + (yield "a") + (yield "end")) + (assert (= (list (multi-yield)) [0 1 2 "a" "end"]))) diff --git a/tests/native_tests/hy_eval.hy b/tests/native_tests/hy_eval.hy new file mode 100644 index 000000000..ff664f617 --- /dev/null +++ b/tests/native_tests/hy_eval.hy @@ -0,0 +1,86 @@ +(import + re + pytest) + + +(defn test-eval [] + (assert (= 2 (hy.eval (quote (+ 1 1))))) + (setv x 2) + (assert (= 4 (hy.eval (quote (+ x 2))))) + (setv test-payload (quote (+ x 2))) + (setv x 4) + (assert (= 6 (hy.eval test-payload))) + (assert (= 9 ((hy.eval (quote (fn [x] (+ 3 3 x)))) 3))) + (assert (= 1 (hy.eval (quote 1)))) + (assert (= "foobar" (hy.eval (quote "foobar")))) + (setv x (quote 42)) + (assert (= 42 (hy.eval x))) + (assert (= 27 (hy.eval (+ (quote (*)) (* [(quote 3)] 3))))) + (assert (= None (hy.eval (quote (print ""))))) + + ;; https://github.com/hylang/hy/issues/1041 + (assert (is (hy.eval 're) re)) + (assert (is ((fn [] (hy.eval 're))) re))) + + +(defn test-eval-false [] + (assert (is (hy.eval 'False) False)) + (assert (is (hy.eval 'None) None)) + (assert (= (hy.eval '0) 0)) + (assert (= (hy.eval '"") "")) + (assert (= (hy.eval 'b"") b"")) + (assert (= (hy.eval ':) :)) + (assert (= (hy.eval '[]) [])) + (assert (= (hy.eval '#()) #())) + (assert (= (hy.eval '{}) {})) + (assert (= (hy.eval '#{}) #{}))) + + +(defn test-eval-global-dict [] + (assert (= 'bar (hy.eval (quote foo) {"foo" 'bar}))) + (assert (= 1 (do (setv d {}) (hy.eval '(setv x 1) d) (hy.eval (quote x) d)))) + (setv d1 {} d2 {}) + (hy.eval '(setv x 1) d1) + (with [e (pytest.raises NameError)] + (hy.eval (quote x) d2))) + + +(defn test-eval-failure [] + ; yo dawg + (with [(pytest.raises TypeError)] (hy.eval '(hy.eval))) + (defclass C) + (with [(pytest.raises TypeError)] (hy.eval (C))) + (with [(pytest.raises TypeError)] (hy.eval 'False [])) + (with [(pytest.raises TypeError)] (hy.eval 'False {} 1))) + + +(defn test-eval-quasiquote [] + ; https://github.com/hylang/hy/issues/1174 + + (for [x [ + None False True + 5 5.1 + 5j 5.1j 2+1j 1.2+3.4j + "" b"" + "apple bloom" b"apple bloom" "⚘" b"\x00" + [] #{} {} + [1 2 3] #{1 2 3} {"a" 1 "b" 2}]] + (assert (= (hy.eval `(get [~x] 0)) x)) + (assert (= (hy.eval x) x))) + + (setv kw :mykeyword) + (assert (= (get (hy.eval `[~kw]) 0) kw)) + (assert (= (hy.eval kw) kw)) + + (assert (= (hy.eval #()) #())) + (assert (= (hy.eval #(1 2 3)) #(1 2 3))) + + (assert (= (hy.eval `(+ "a" ~(+ "b" "c"))) "abc")) + + (setv l ["a" "b"]) + (setv n 1) + (assert (= (hy.eval `(get ~l ~n) "b"))) + + (setv d {"a" 1 "b" 2}) + (setv k "b") + (assert (= (hy.eval `(get ~d ~k)) 2))) diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy new file mode 100644 index 000000000..1f283823d --- /dev/null +++ b/tests/native_tests/hy_misc.hy @@ -0,0 +1,88 @@ +;; Tests of `hy.gensym`, `hy.macroexpand`, `hy.macroexpand-1`, +;; `hy.disassemble`, and `hy.read` + +(import + pytest) + + +(defn test-gensym [] + (setv s1 (hy.gensym)) + (assert (isinstance s1 hy.models.Symbol)) + (assert (= 0 (.find s1 "_G\uffff"))) + (setv s2 (hy.gensym "xx")) + (setv s3 (hy.gensym "xx")) + (assert (= 0 (.find s2 "_xx\uffff"))) + (assert (not (= s2 s3))) + (assert (not (= (str s2) (str s3))))) + + +(defmacro mac [x expr] + `(~@expr ~x)) + + +(defn test-macroexpand [] + (assert (= (hy.macroexpand '(mac (a b) (x y))) + '(x y (a b)))) + (assert (= (hy.macroexpand '(mac (a b) (mac 5))) + '(a b 5)))) + + +(defn test-macroexpand-with-named-import [] + ; https://github.com/hylang/hy/issues/1207 + (defmacro m-with-named-import [] + (import math [pow]) + (pow 2 3)) + (assert (= (hy.macroexpand '(m-with-named-import)) (hy.models.Float (** 2 3))))) + + +(defn test-macroexpand-1 [] + (assert (= (hy.macroexpand-1 '(mac (a b) (mac 5))) + '(mac 5 (a b))))) + + +(defn test-disassemble [] + (import re) + (defn nos [x] (re.sub r"\s" "" x)) + (assert (= (nos (hy.disassemble '(do (leaky) (leaky) (macros)))) + (nos (.format + "Module( + body=[Expr(value=Call(func=Name(id='leaky', ctx=Load()), args=[], keywords=[])), + Expr(value=Call(func=Name(id='leaky', ctx=Load()), args=[], keywords=[])), + Expr(value=Call(func=Name(id='macros', ctx=Load()), args=[], keywords=[]))]{})" + (if hy._compat.PY3_8 ",type_ignores=[]" ""))))) + (assert (= (nos (hy.disassemble '(do (leaky) (leaky) (macros)) True)) + "leaky()leaky()macros()")) + (assert (= (re.sub r"[()\n ]" "" (hy.disassemble `(+ ~(+ 1 1) 40) True)) + "2+40"))) + + +(defn test-read-file-object [] + (import io [StringIO]) + + (setv stdin-buffer (StringIO "(+ 2 2)\n(- 2 2)")) + (assert (= (hy.eval (hy.read stdin-buffer)) 4)) + (assert (isinstance (hy.read stdin-buffer) hy.models.Expression)) + + ; Multiline test + (setv stdin-buffer (StringIO "(\n+\n41\n1\n)\n(-\n2\n1\n)")) + (assert (= (hy.eval (hy.read stdin-buffer)) 42)) + (assert (= (hy.eval (hy.read stdin-buffer)) 1)) + + ; EOF test + (setv stdin-buffer (StringIO "(+ 2 2)")) + (hy.read stdin-buffer) + (with [(pytest.raises EOFError)] + (hy.read stdin-buffer))) + + +(defn test-read-str [] + (assert (= (hy.read "(print 1)") '(print 1))) + (assert (is (type (hy.read "(print 1)")) (type '(print 1)))) + + ; Watch out for false values: https://github.com/hylang/hy/issues/1243 + (assert (= (hy.read "\"\"") '"")) + (assert (is (type (hy.read "\"\"")) (type '""))) + (assert (= (hy.read "[]") '[])) + (assert (is (type (hy.read "[]")) (type '[]))) + (assert (= (hy.read "0") '0)) + (assert (is (type (hy.read "0")) (type '0)))) diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy new file mode 100644 index 000000000..00a9b1991 --- /dev/null +++ b/tests/native_tests/import.hy @@ -0,0 +1,304 @@ +;; Tests of `import`, `require`, and `export` + +(import + importlib + os.path + os.path [exists isdir isfile] + sys :as systest + sys + pytest) + + +(defn test-imported-bits [] + (assert (is (exists ".") True)) + (assert (is (isdir ".") True)) + (assert (is (isfile ".") False))) + + +(defn test-importas [] + (assert (!= (len systest.path) 0))) + + +(defn test-import-syntax [] + ;; Simple import + (import sys os) + + ;; from os.path import basename + (import os.path [basename]) + (assert (= (basename "/some/path") "path")) + + ;; import os.path as p + (import os.path :as p) + (assert (= p.basename basename)) + + ;; from os.path import basename as bn + (import os.path [basename :as bn]) + (assert (= bn basename)) + + ;; Multiple stuff to import + (import sys + os.path [dirname] + os.path :as op + os.path [dirname :as dn]) + (assert (= (dirname "/some/path") "/some")) + (assert (= op.dirname dirname)) + (assert (= dn dirname))) + + +(defn test-relative-import [] + (import ..resources [tlib in-init]) + (assert (= tlib.SECRET-MESSAGE "Hello World")) + (assert (= in-init "chippy")) + (import .. [resources]) + (assert (= resources.in-init "chippy"))) + + +(defn test-import-init-hy [] + (import tests.resources.bin) + (assert (in "_null_fn_for_import_test" (dir tests.resources.bin)))) + + +(defn test-require [] + (with [(pytest.raises NameError)] + (qplah 1 2 3 4)) + (with [(pytest.raises NameError)] + (parald 1 2 3 4)) + (with [(pytest.raises NameError)] + (✈ 1 2 3 4)) + (with [(pytest.raises NameError)] + (hyx_XairplaneX 1 2 3 4)) + + (require tests.resources.tlib [qplah]) + (assert (= (qplah 1 2 3) [8 1 2 3])) + (with [(pytest.raises NameError)] + (parald 1 2 3 4)) + + (require tests.resources.tlib) + (assert (= (tests.resources.tlib.parald 1 2 3) [9 1 2 3])) + (assert (= (tests.resources.tlib.✈ "silly") "plane silly")) + (assert (= (tests.resources.tlib.hyx_XairplaneX "foolish") "plane foolish")) + (with [(pytest.raises NameError)] + (parald 1 2 3 4)) + + (require tests.resources.tlib :as T) + (assert (= (T.parald 1 2 3) [9 1 2 3])) + (assert (= (T.✈ "silly") "plane silly")) + (assert (= (T.hyx_XairplaneX "foolish") "plane foolish")) + (with [(pytest.raises NameError)] + (parald 1 2 3 4)) + + (require tests.resources.tlib [parald :as p]) + (assert (= (p 1 2 3) [9 1 2 3])) + (with [(pytest.raises NameError)] + (parald 1 2 3 4)) + + (require tests.resources.tlib *) + (assert (= (parald 1 2 3) [9 1 2 3])) + (assert (= (✈ "silly") "plane silly")) + (assert (= (hyx_XairplaneX "foolish") "plane foolish")) + + (require tests.resources [tlib macros :as m exports-none]) + (assert (in "tlib.qplah" __macros__)) + (assert (in (hy.mangle "m.test-macro") __macros__)) + (assert (in (hy.mangle "exports-none.cinco") __macros__)) + (require os [path]) + (with [(pytest.raises hy.errors.HyRequireError)] + (hy.eval '(require tests.resources [does-not-exist]))) + + (require tests.resources.exports *) + (assert (= (casey 1 2 3) [11 1 2 3])) + (assert (= (☘ 1 2 3) [13 1 2 3])) + (with [(pytest.raises NameError)] + (brother 1 2 3 4))) + + +(defn test-require-native [] + (with [(pytest.raises NameError)] + (test-macro-2)) + (import tests.resources.macros) + (with [(pytest.raises NameError)] + (test-macro-2)) + (require tests.resources.macros [test-macro-2]) + (test-macro-2) + (assert (= qup 2))) + + +(defn test-relative-require [] + (require ..resources.macros [test-macro]) + (assert (in "test_macro" __macros__)) + + (require .beside [xyzzy]) + (assert (in "xyzzy" __macros__)) + + (require . [beside :as b]) + (assert (in "b.xyzzy" __macros__))) + + +(defn test-macro-namespace-resolution [] + "Confirm that local versions of macro-macro dependencies do not shadow the +versions from the macro's own module, but do resolve unbound macro references +in expansions." + + ;; `nonlocal-test-macro` is a macro used within + ;; `tests.resources.macro-with-require.test-module-macro`. + ;; Here, we introduce an equivalently named version in local scope that, when + ;; used, will expand to a different output string. + (defmacro nonlocal-test-macro [x] + (print "this is the local version of `nonlocal-test-macro`!")) + + ;; Was the above macro created properly? + (assert (in "nonlocal_test_macro" __macros__)) + + (setv nonlocal-test-macro (get __macros__ "nonlocal_test_macro")) + + (require tests.resources.macro-with-require *) + + (setv module-name-var "tests.native_tests.native_macros.test-macro-namespace-resolution") + (assert (= (+ "This macro was created in tests.resources.macros, " + "expanded in tests.native_tests.native_macros.test-macro-namespace-resolution " + "and passed the value 2.") + (test-module-macro 2))) + + ;; Now, let's use a `require`d macro that depends on another macro defined only + ;; in this scope. + (defmacro local-test-macro [x] + (.format "This is the local version of `nonlocal-test-macro` returning {}!" (int x))) + + (assert (= "This is the local version of `nonlocal-test-macro` returning 3!" + (test-module-macro-2 3)))) + + +(defn test-requires-pollutes-core [] + ;; https://github.com/hylang/hy/issues/1978 + ;; Macros loaded from an external module should not pollute `__macros__` + ;; with macros from core. + + (setv pyc-file (importlib.util.cache-from-source + (os.path.realpath + (os.path.join + "tests" "resources" "macros.hy")))) + + ;; Remove any cached byte-code, so that this runs from source and + ;; gets evaluated in this module. + (when (os.path.isfile pyc-file) + (os.unlink pyc-file) + (.clear sys.path_importer_cache) + (when (in "tests.resources.macros" sys.modules) + (del (get sys.modules "tests.resources.macros")) + (__macros__.clear))) + + ;; Ensure that bytecode isn't present when we require this module. + (assert (not (os.path.isfile pyc-file))) + + (defn require-macros [] + (require tests.resources.macros :as m) + + (assert (in (hy.mangle "m.test-macro") __macros__)) + (for [macro-name __macros__] + (assert (not (and (in "with" macro-name) + (!= "with" macro-name)))))) + + (require-macros) + + ;; Now that bytecode is present, reload the module, clear the `require`d + ;; macros and tags, and rerun the tests. + (assert (os.path.isfile pyc-file)) + + ;; Reload the module and clear the local macro context. + (.clear sys.path_importer_cache) + (del (get sys.modules "tests.resources.macros")) + (.clear __macros__) + + (require-macros)) + + +(defn [(pytest.mark.xfail)] test-macro-from-module [] + " + Macros loaded from an external module, which itself `require`s macros, should + work without having to `require` the module's macro dependencies (due to + [minimal] macro namespace resolution). + + In doing so we also confirm that a module's `__macros__` attribute is correctly + loaded and used. + + Additionally, we confirm that `require` statements are executed via loaded bytecode. + " + + (setv pyc-file (importlib.util.cache-from-source + (os.path.realpath + (os.path.join + "tests" "resources" "macro_with_require.hy")))) + + ;; Remove any cached byte-code, so that this runs from source and + ;; gets evaluated in this module. + (when (os.path.isfile pyc-file) + (os.unlink pyc-file) + (.clear sys.path_importer_cache) + (when (in "tests.resources.macro_with_require" sys.modules) + (del (get sys.modules "tests.resources.macro_with_require")) + (__macros__.clear))) + + ;; Ensure that bytecode isn't present when we require this module. + (assert (not (os.path.isfile pyc-file))) + + (defn test-requires-and-macros [] + (require tests.resources.macro-with-require + [test-module-macro]) + + ;; Make sure that `require` didn't add any of its `require`s + (assert (not (in (hy.mangle "nonlocal-test-macro") __macros__))) + ;; and that it didn't add its tags. + (assert (not (in (hy.mangle "#test-module-tag") __macros__))) + + ;; Now, require everything. + (require tests.resources.macro-with-require *) + + ;; Again, make sure it didn't add its required macros and/or tags. + (assert (not (in (hy.mangle "nonlocal-test-macro") __macros__))) + + ;; Its tag(s) should be here now. + (assert (in (hy.mangle "#test-module-tag") __macros__)) + + ;; The test macro expands to include this symbol. + (setv module-name-var "tests.native_tests.native_macros") + (assert (= (+ "This macro was created in tests.resources.macros, " + "expanded in tests.native_tests.native_macros " + "and passed the value 1.") + (test-module-macro 1)))) + + (test-requires-and-macros) + + ;; Now that bytecode is present, reload the module, clear the `require`d + ;; macros and tags, and rerun the tests. + (assert (os.path.isfile pyc-file)) + + ;; Reload the module and clear the local macro context. + (.clear sys.path_importer_cache) + (del (get sys.modules "tests.resources.macro_with_require")) + (.clear __macros__) + + ;; There doesn't seem to be a way--via standard import mechanisms--to + ;; ensure that an imported module used the cached bytecode. We'll simply have + ;; to trust that the .pyc loading convention was followed. + (test-requires-and-macros)) + + +(defn test-recursive-require-star [] + "(require foo *) should pull in macros required by `foo`." + (require tests.resources.macro-with-require *) + + (test-macro) + (assert (= blah 1))) + + +(defn test-export-objects [] + ; We use `hy.eval` here because of a Python limitation that + ; importing `*` is only allowed at the module level. + (hy.eval '(do + (import tests.resources.exports *) + (assert (= (jan) 21)) + (assert (= (♥) 23)) + (with [(pytest.raises NameError)] + (wayne)) + (import tests.resources.exports [wayne]) + (assert (= (wayne) 22))))) diff --git a/tests/native_tests/keywords.hy b/tests/native_tests/keywords.hy new file mode 100644 index 000000000..981e39b3d --- /dev/null +++ b/tests/native_tests/keywords.hy @@ -0,0 +1,99 @@ +(import + pickle + pytest + tests.resources [kwtest]) + + +(defn test-keyword [] + (assert (= :foo :foo)) + (assert (= :foo ':foo)) + (setv x :foo) + (assert (is (type x) (type ':foo))) + (assert (= (get {:foo "bar"} :foo) "bar")) + (assert (= (get {:bar "quux"} (get {:foo :bar} :foo)) "quux"))) + + +(defn test-keyword-clash [] + "Keywords shouldn't clash with normal strings." + + (assert (= (get {:foo "bar" ":foo" "quux"} :foo) "bar")) + (assert (= (get {:foo "bar" ":foo" "quux"} ":foo") "quux"))) + + +(defn test-empty-keyword [] + (assert (= : :)) + (assert (isinstance ': hy.models.Keyword)) + (assert (!= : ":")) + (assert (= (. ': name) ""))) + + +(defn test-pickling-keyword [] + ; https://github.com/hylang/hy/issues/1754 + (setv x :test-keyword) + (for [protocol (range 0 (+ pickle.HIGHEST-PROTOCOL 1))] + (assert (= x + (pickle.loads (pickle.dumps x :protocol protocol)))))) + + +(defn test-keyword-get [] + + (assert (= (:foo (dict :foo "test")) "test")) + (setv f :foo) + (assert (= (f (dict :foo "test")) "test")) + + (assert (= (:foo-bar (dict :foo-bar "baz")) "baz")) + (assert (= (:♥ (dict :♥ "heart")) "heart")) + (defclass C [] + (defn __getitem__ [self k] + k)) + (assert (= (:♥ (C)) "hyx_Xblack_heart_suitX")) + + (with [(pytest.raises KeyError)] (:foo (dict :a 1 :b 2))) + (assert (= (:foo (dict :a 1 :b 2) 3) 3)) + (assert (= (:foo (dict :a 1 :b 2 :foo 5) 3) 5)) + + (with [(pytest.raises TypeError)] (:foo "Hello World")) + (with [(pytest.raises TypeError)] (:foo (object))) + + ; The default argument should work regardless of the collection type. + (defclass G [object] + (defn __getitem__ [self k] + (raise KeyError))) + (assert (= (:foo (G) 15) 15))) + + +(defn test-keyword-creation [] + (assert (= (hy.models.Keyword "foo") :foo)) + (assert (= (hy.models.Keyword "foo_bar") :foo_bar)) + (assert (= (hy.models.Keyword "foo-bar") :foo-bar)) + (assert (!= :foo_bar :foo-bar)) + (assert (= (hy.models.Keyword "") :))) + + +(defn test-keywords-in-fn-calls [] + (assert (= (kwtest) {})) + (assert (= (kwtest :key "value") {"key" "value"})) + (assert (= (kwtest :key-with-dashes "value") {"key_with_dashes" "value"})) + (assert (= (kwtest :result (+ 1 1)) {"result" 2})) + (assert (= (kwtest :key (kwtest :key2 "value")) {"key" {"key2" "value"}})) + (assert (= ((get (kwtest :key (fn [x] (* x 2))) "key") 3) 6))) + + +(defn test-kwargs [] + (assert (= (kwtest :one "two") {"one" "two"})) + (setv mydict {"one" "three"}) + (assert (= (kwtest #** mydict) mydict)) + (assert (= (kwtest #** ((fn [] {"one" "two"}))) {"one" "two"}))) + + +(defmacro identify-keywords [#* elts] + `(list + (map + (fn [x] (if (isinstance x hy.models.Keyword) "keyword" "other")) + ~elts))) + +(defn test-keywords-and-macros [] + "Macros should still be able to handle keywords as they best see fit." + (assert + (= (identify-keywords 1 "bloo" :foo) + ["other" "other" "keyword"]))) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy deleted file mode 100644 index 0f9887e28..000000000 --- a/tests/native_tests/language.hy +++ /dev/null @@ -1,1757 +0,0 @@ -(import tests.resources [kwtest function-with-a-dash AsyncWithTest] - os.path [exists isdir isfile] - os - sys :as systest - re - operator [or_] - itertools [repeat count islice] - pickle - typing [get-type-hints List Dict] - asyncio - hy.errors [HyLanguageError HySyntaxError] - pytest) -(import sys) - -(import hy._compat [PY3_8]) - - -(defmacro mac [x expr] - `(~@expr ~x)) - - -(defn test-setv-get [] - (setv foo [0 1 2]) - (setv (get foo 0) 12) - (assert (= (get foo 0) 12))) - - -(defn test-setv-pairs [] - (setv a 1 b 2) - (assert (= a 1)) - (assert (= b 2)) - (setv y 0 x 1 y x) - (assert (= y 1)) - (with [(pytest.raises HyLanguageError)] - (hy.eval '(setv a 1 b)))) - - -(defn test-setv-returns-none [] - - (defn an [x] - (assert (is x None))) - - (an (setv)) - (an (setv x 1)) - (assert (= x 1)) - (an (setv x 2)) - (assert (= x 2)) - (an (setv y 2 z 3)) - (assert (= y 2)) - (assert (= z 3)) - (an (setv [y z] [7 8])) - (assert (= y 7)) - (assert (= z 8)) - (an (setv #(y z) [9 10])) - (assert (= y 9)) - (assert (= z 10)) - - (setv p 11) - (setv p (setv q 12)) - (assert (= q 12)) - (an p) - - (an (setv x (defn phooey [] (setv p 1) (+ p 6)))) - (an (setv x (defclass C))) - (an (setv x (for [i (range 3)] i (+ i 1)))) - (an (setv x (assert True))) - - (an (setv x (with [(open "README.md" "r")] 3))) - (assert (= x 3)) - (an (setv x (try (/ 1 2) (except [ZeroDivisionError] "E1")))) - (assert (= x .5)) - (an (setv x (try (/ 1 0) (except [ZeroDivisionError] "E2")))) - (assert (= x "E2")) - - ; https://github.com/hylang/hy/issues/1052 - (an (setv (get {} "x") 42)) - (setv l []) - (defclass Foo [object] - (defn __setattr__ [self attr val] - (.append l [attr val]))) - (setv x (Foo)) - (an (setv x.eggs "ham")) - (assert (not (hasattr x "eggs"))) - (assert (= l [["eggs" "ham"]]))) - - -(defn test-illegal-assignments [] - (for [form '[ - (setv (do 1 2) 1) - (setv 1 1) - (setv {1 2} 1) - (del 1 1) - ; https://github.com/hylang/hy/issues/1780 - (setv None 1) - (setv False 1) - (setv True 1) - (defn None [] (print "hello")) - (defn True [] (print "hello")) - (defn f [True] (print "hello")) - (for [True [1 2 3]] (print "hello")) - (lfor True [1 2 3] True) - (lfor :setv True 1 True) - (with [True x] (print "hello")) - (try 1 (except [True AssertionError] 2)) - (defclass True [])]] - (with [e (pytest.raises HyLanguageError)] - (hy.eval form)) - (assert (in "Can't assign" e.value.msg)))) - - -(defn test-no-str-as-sym [] - "Don't treat strings as symbols in the calling position" - (with [(pytest.raises TypeError)] ("setv" True 3)) ; A special form - (with [(pytest.raises TypeError)] ("abs" -2)) ; A function - (with [(pytest.raises TypeError)] ("when" 1 2))) ; A macro - - -(defn test-while-loop [] - (setv count 5) - (setv fact 1) - (while (> count 0) - (setv fact (* fact count)) - (setv count (- count 1))) - (assert (= count 0)) - (assert (= fact 120)) - - (setv l []) - (defn f [] - (.append l 1) - (len l)) - (while (!= (f) 4)) - (assert (= l [1 1 1 1])) - - (setv l []) - (defn f [] - (.append l 1) - (len l)) - (while (!= (f) 4) (do)) - (assert (= l [1 1 1 1])) - - ; only compile the condition once - ; https://github.com/hylang/hy/issues/1790 - (global while-cond-var) - (setv while-cond-var 10) - (hy.eval - '(do - (defmacro while-cond [] - (global while-cond-var) - (assert (= while-cond-var 10)) - (+= while-cond-var 1) - `(do - (setv x 3) - False)) - (while (while-cond)) - (assert (= x 3))))) - -(defn test-while-loop-else [] - (setv count 5) - (setv fact 1) - (setv myvariable 18) - (while (> count 0) - (setv fact (* fact count)) - (setv count (- count 1)) - (else (setv myvariable 26))) - (assert (= count 0)) - (assert (= fact 120)) - (assert (= myvariable 26)) - - ; multiple statements in a while loop should work - (setv count 5) - (setv fact 1) - (setv myvariable 18) - (setv myothervariable 15) - (while (> count 0) - (setv fact (* fact count)) - (setv count (- count 1)) - (else (setv myvariable 26) - (setv myothervariable 24))) - (assert (= count 0)) - (assert (= fact 120)) - (assert (= myvariable 26)) - (assert (= myothervariable 24)) - - ; else clause shouldn't get run after a break - (while True - (break) - (else (setv myvariable 53))) - (assert (= myvariable 26)) - - ; don't be fooled by constructs that look like else clauses - (setv x 2) - (setv a []) - (setv else True) - (while x - (.append a x) - (-= x 1) - [else (.append a "e")]) - (assert (= a [2 "e" 1 "e"])) - - (setv x 2) - (setv a []) - (with [(pytest.raises TypeError)] - (while x - (.append a x) - (-= x 1) - ("else" (.append a "e")))) - (assert (= a [2 "e"]))) - - -(defn test-while-multistatement-condition [] - - ; The condition should be executed every iteration, before the body. - ; `else` should be executed last. - (setv s "") - (setv x 2) - (while (do (+= s "a") x) - (+= s "b") - (-= x 1) - (else - (+= s "z"))) - (assert (= s "ababaz")) - - ; `else` should still be skipped after `break`. - (setv s "") - (setv x 2) - (while (do (+= s "a") x) - (+= s "b") - (-= x 1) - (when (= x 0) - (break)) - (else - (+= s "z"))) - (assert (= s "abab")) - - ; `continue` should jump to the condition. - (setv s "") - (setv x 2) - (setv continued? False) - (while (do (+= s "a") x) - (+= s "b") - (when (and (= x 1) (not continued?)) - (+= s "c") - (setv continued? True) - (continue)) - (-= x 1) - (else - (+= s "z"))) - (assert (= s "ababcabaz")) - - ; `break` in a condition applies to the `while`, not an outer loop. - (setv s "") - (for [x "123"] - (+= s x) - (setv y 0) - (while (do (when (and (= x "2") (= y 1)) (break)) (< y 3)) - (+= s "y") - (+= y 1))) - (assert (= s "1yyy2y3yyy")) - - ; The condition is still tested appropriately if its last variable - ; is set to a false value in the loop body. - (setv out []) - (setv x 0) - (setv a [1 1]) - (while (do (.append out 2) (setv x (and a (.pop a))) x) - (setv x 0) - (.append out x)) - (assert (= out [2 0 2 0 2])) - (assert (is x a))) - - -(defn test-branching [] - (if True - (assert (= 1 1)) - (assert (= 2 1)))) - - -(defn test-branching-with-do [] - (if False - (assert (= 2 1)) - (do - (assert (= 1 1)) - (assert (= 1 1)) - (assert (= 1 1))))) - -(defn test-branching-expr-count-with-do [] - "Ensure we execute the right number of expressions in a branch." - (setv counter 0) - (if False - (assert (= 2 1)) - (do - (setv counter (+ counter 1)) - (setv counter (+ counter 1)) - (setv counter (+ counter 1)))) - (assert (= counter 3))) - - -(defn test-cond [] - (cond - (= 1 2) (assert (is True False)) - (is None None) (do (setv x True) (assert x))) - (assert (is (cond) None)) - - (assert (= (cond - False 1 - [] 2 - True 8) 8)) - - (setv x 0) - (assert (is (cond False 1 [] 2 x 3) None)) - - (with [e (pytest.raises hy.errors.HyMacroExpansionError)] - (hy.eval '(cond 1))) - (assert (in "needs an even number of arguments" e.value.msg)) - - ; Make sure each test is only evaluated once, and `cond` - ; short-circuits. - (setv x 1) - (assert (= "first" (cond - (do (*= x 2) True) (do (*= x 3) "first") - (do (*= x 5) True) (do (*= x 7) "second")))) - (assert (= x 6))) - - -(defn test-if [] - (assert (= 1 (if 0 -1 1)))) - - -(defn test-fn [] - (setv square (fn [x] (* x x))) - (assert (= 4 (square 2))) - (setv lambda_list (fn [test #* args] #(test args))) - (assert (= #(1 #(2 3)) (lambda_list 1 2 3)))) - - -(defn test-imported-bits [] - (assert (is (exists ".") True)) - (assert (is (isdir ".") True)) - (assert (is (isfile ".") False))) - - -(defn test-star-unpacking [] - ; Python 3-only forms of unpacking are in py3_only_tests.hy - (setv l [1 2 3]) - (setv d {"a" "x" "b" "y"}) - (defn fun [[x1 None] [x2 None] [x3 None] [x4 None] [a None] [b None] [c None]] - [x1 x2 x3 x4 a b c]) - (assert (= (fun 5 #* l) [5 1 2 3 None None None])) - (assert (= (+ #* l) 6)) - (assert (= (fun 5 #** d) [5 None None None "x" "y" None])) - (assert (= (fun 5 #* l #** d) [5 1 2 3 "x" "y" None]))) - - - -(defn test-kwargs [] - (assert (= (kwtest :one "two") {"one" "two"})) - (setv mydict {"one" "three"}) - (assert (= (kwtest #** mydict) mydict)) - (assert (= (kwtest #** ((fn [] {"one" "two"}))) {"one" "two"}))) - - - -(defn test-dotted [] - (assert (= (.join " " ["one" "two"]) "one two")) - - (defclass X [object] []) - (defclass M [object] - (defn meth [self #* args #** kwargs] - (.join " " (+ #("meth") args - (tuple (map (fn [k] (get kwargs k)) (sorted (.keys kwargs)))))))) - - (setv x (X)) - (setv m (M)) - - (assert (= (.meth m) "meth")) - (assert (= (.meth m "foo" "bar") "meth foo bar")) - (assert (= (.meth :b "1" :a "2" m "foo" "bar") "meth foo bar 2 1")) - (assert (= (.meth m #* ["foo" "bar"]) "meth foo bar")) - - (setv x.p m) - (assert (= (.p.meth x) "meth")) - (assert (= (.p.meth x "foo" "bar") "meth foo bar")) - (assert (= (.p.meth :b "1" :a "2" x "foo" "bar") "meth foo bar 2 1")) - (assert (= (.p.meth x #* ["foo" "bar"]) "meth foo bar")) - - (setv x.a (X)) - (setv x.a.b m) - (assert (= (.a.b.meth x) "meth")) - (assert (= (.a.b.meth x "foo" "bar") "meth foo bar")) - (assert (= (.a.b.meth :b "1" :a "2" x "foo" "bar") "meth foo bar 2 1")) - (assert (= (.a.b.meth x #* ["foo" "bar"]) "meth foo bar")) - - (assert (= (.__str__ :foo) ":foo"))) - - -(defn test-multidot [] - (setv a 1 b 2 c 3) - - (defn .. [#* args] - (.join "~" (map str args))) - (assert (= ..a.b.c "None~1~2~3")) - - (defmacro .... [#* args] - (.join "@" (map str args))) - (assert (= ....uno.dos.tres "None@uno@dos@tres"))) - - -(defn test-do [] - (do)) - - -(defn test-try [] - - (try (do) (except [])) - - (try (do) (except [IOError]) (except [])) - - ; test that multiple statements in a try get evaluated - (setv value 0) - (try (+= value 1) (+= value 2) (except [IOError]) (except [])) - (assert (= value 3)) - - ; test that multiple expressions in a try get evaluated - ; https://github.com/hylang/hy/issues/1584 - (setv l []) - (defn f [] (.append l 1)) - (try (f) (f) (f) (except [IOError])) - (assert (= l [1 1 1])) - (setv l []) - (try (f) (f) (f) (except [IOError]) (else (f))) - (assert (= l [1 1 1 1])) - - ;; Test correct (raise) - (setv passed False) - (try - (try - (do) - (raise IndexError) - (except [IndexError] (raise))) - (except [IndexError] - (setv passed True))) - (assert passed) - - ;; Test incorrect (raise) - (setv passed False) - (try - (raise) - (except [RuntimeError] - (setv passed True))) - (assert passed) - - ;; Test (finally) - (setv passed False) - (try - (do) - (finally (setv passed True))) - (assert passed) - - ;; Test (finally) + (raise) - (setv passed False) - (try - (raise Exception) - (except []) - (finally (setv passed True))) - (assert passed) - - - ;; Test (finally) + (raise) + (else) - (setv passed False - not-elsed True) - (try - (raise Exception) - (except []) - (else (setv not-elsed False)) - (finally (setv passed True))) - (assert passed) - (assert not-elsed) - - (try - (raise (KeyError)) - (except [[IOError]] (assert False)) - (except [e [KeyError]] (assert e))) - - (try - (raise (KeyError)) - (except [[IOError]] (assert False)) - (except [e [KeyError]] (assert e))) - - (try - (get [1] 3) - (except [IndexError] (assert True)) - (except [IndexError] (do))) - - (try - (print foobar42ofthebaz) - (except [IndexError] (assert False)) - (except [NameError] (do))) - - (try - (get [1] 3) - (except [e IndexError] (assert (isinstance e IndexError)))) - - (try - (get [1] 3) - (except [e [IndexError NameError]] (assert (isinstance e IndexError)))) - - (try - (print foobar42ofthebaz) - (except [e [IndexError NameError]] (assert (isinstance e NameError)))) - - (try - (print foobar42) - (except [[IndexError NameError]] (do))) - - (try - (get [1] 3) - (except [[IndexError NameError]] (do))) - - (try - (print foobar42ofthebaz) - (except [])) - - (try - (print foobar42ofthebaz) - (except [] (do))) - - (try - (print foobar42ofthebaz) - (except [] - (setv foobar42ofthebaz 42) - (assert (= foobar42ofthebaz 42)))) - - (setv passed False) - (try - (try (do) (except []) (else (bla))) - (except [NameError] (setv passed True))) - (assert passed) - - (setv x 0) - (try - (raise IOError) - (except [IOError] - (setv x 45)) - (else (setv x 44))) - (assert (= x 45)) - - (setv x 0) - (try - (raise KeyError) - (except [] - (setv x 45)) - (else (setv x 44))) - (assert (= x 45)) - - (setv x 0) - (try - (try - (raise KeyError) - (except [IOError] - (setv x 45)) - (else (setv x 44))) - (except [])) - (assert (= x 0)) - - ; test that [except ...] and ("except" ...) aren't treated like (except ...), - ; and that the code there is evaluated normally - (setv x 0) - (try - (+= x 1) - ("except" [IOError] (+= x 1)) - (except [])) - - (assert (= x 2)) - - (setv x 0) - (try - (+= x 1) - [except [IOError] (+= x 1)] - (except [])) - - (assert (= x 2))) - - -(defn test-pass [] - (if True (do) (do)) - (assert (= 1 1))) - - -(defn test-yield [] - (defn gen [] (for [x [1 2 3 4]] (yield x))) - (setv ret 0) - (for [y (gen)] (setv ret (+ ret y))) - (assert (= ret 10))) - -(defn test-yield-with-return [] - (defn gen [] (yield 3) "goodbye") - (setv gg (gen)) - (assert (= 3 (next gg))) - (with [e (pytest.raises StopIteration)] - (next gg)) - (assert (= e.value.value "goodbye"))) - - -(defn test-yield-in-try [] - (defn gen [] - (setv x 1) - (try (yield x) - (finally (print x)))) - (setv output (list (gen))) - (assert (= [1] output))) - - -(defn test-ellipsis [] - (global Ellipsis) - (assert (is ... Ellipsis)) - (setv e Ellipsis) - (setv Ellipsis 14) - (assert (= Ellipsis 14)) - (assert (!= ... 14)) - (assert (is ... e))) - - -(defn test-cut [] - (assert (= (cut [1 2 3 4 5] 3) [1 2 3])) - (assert (= (cut [1 2 3 4 5] 1 None) [2 3 4 5])) - (assert (= (cut [1 2 3 4 5] 1 3) [2 3])) - (assert (= (cut [1 2 3 4 5]) [1 2 3 4 5]))) - - -(defn test-importas [] - (assert (!= (len systest.path) 0))) - - -(defn test-context [] - (with [fd (open "README.md" "r")] (assert fd)) - (with [(open "README.md" "r")] (do))) - - -(defn test-with-return [] - (defn read-file [filename] - (with [fd (open filename "r")] (.read fd))) - (assert (!= 0 (len (read-file "README.md"))))) - - -(defn test-for-do [] - (do (do (do (do (do (do (do (do (do (setv #(x y) #(0 0))))))))))) - (for [- [1 2]] - (do - (setv x (+ x 1)) - (setv y (+ y 1)))) - (assert (= y x 2))) - - -(defn test-for-else [] - (setv x 0) - (for [a [1 2]] - (setv x (+ x a)) - (else (setv x (+ x 50)))) - (assert (= x 53)) - - (setv x 0) - (for [a [1 2]] - (setv x (+ x a)) - (else)) - (assert (= x 3))) - - -(defn test-defn-evaluation-order [] - (setv acc []) - (defn my-fun [] - (.append acc "Foo") - (.append acc "Bar") - (.append acc "Baz")) - (my-fun) - (assert (= acc ["Foo" "Bar" "Baz"]))) - - -(defn test-defn-return [] - (defn my-fun [x] - (+ x 1)) - (assert (= 43 (my-fun 42)))) - - -(defn test-defn-lambdakey [] - "Test defn with a `&symbol` function name." - (defn &hy [] 1) - (assert (= (&hy) 1))) - - -(defn test-defn-evaluation-order-with-do [] - (setv acc []) - (defn my-fun [] - (do - (.append acc "Foo") - (.append acc "Bar") - (.append acc "Baz"))) - (my-fun) - (assert (= acc ["Foo" "Bar" "Baz"]))) - - -(defn test-defn-do-return [] - (defn my-fun [x] - (do - (+ x 42) ; noop - (+ x 1))) - (assert (= 43 (my-fun 42)))) - - -(defn test-defn-dunder-name [] - "`defn` should preserve `__name__`." - - (defn phooey [x] - (+ x 1)) - (assert (= phooey.__name__ "phooey")) - - (defn mooey [x] - (+= x 1) - x) - (assert (= mooey.__name__ "mooey"))) - - -(defn test-defn-annotations [] - - (defn #^int f [#^(get List int) p1 p2 #^str p3 #^str [o1 None] #^int [o2 0] - #^str #* rest #^str k1 #^int [k2 0] #^bool #** kwargs]) - - (assert (is (. f __annotations__ ["return"]) int)) - (for [[k v] (.items (dict - :p1 (get List int) :p3 str :o1 str :o2 int - :k1 str :k2 int :kwargs bool))] - (assert (= (. f __annotations__ [k]) v)))) - - -(defn test-return [] - - ; `return` in main line - (defn f [x] - (return (+ x "a")) - (+ x "b")) - (assert (= (f "q") "qa")) - - ; Nullary `return` - (defn f [x] - (return) - 5) - (assert (is (f "q") None)) - - ; `return` in `when` - (defn f [x] - (when (< x 3) - (return [x 1])) - [x 2]) - (assert (= (f 2) [2 1])) - (assert (= (f 4) [4 2])) - - ; `return` in a loop - (setv accum []) - (defn f [x] - (while True - (when (= x 0) - (return)) - (.append accum x) - (-= x 1)) - (.append accum "this should never be appended") - 1) - (assert (is (f 5) None)) - (assert (= accum [5 4 3 2 1])) - - ; `return` of a `do` - (setv accum []) - (defn f [] - (return (do - (.append accum 1) - 3)) - 4) - (assert (= (f) 3)) - (assert (= accum [1])) - - ; `return` of an `if` that will need to be compiled to a statement - (setv accum []) - (defn f [x] - (return (if (= x 1) - (do - (.append accum 1) - "a") - (do - (.append accum 2) - "b"))) - "c") - (assert (= (f 2) "b")) - (assert (= accum [2]))) - - -(defn test-immediately-call-lambda [] - (assert (= 2 ((fn [] (+ 1 1)))))) - - -(defn test-fn-return [] - (setv fn-test ((fn [] (fn [] (+ 1 1))))) - (assert (= (fn-test) 2)) - (setv fn-test (fn [])) - (assert (= (fn-test) None))) - - -(defn test-returnable-ifs [] - (assert (= True (if True True True)))) - - -(defn test-macro-call-in-called-lambda [] - (assert (= ((fn [] (mac 2 (- 10 1)))) 7))) - - -(defn test-and [] - - (setv and123 (and 1 2 3) - and-false (and 1 False 3) - and-true (and) - and-single (and 1)) - (assert (= and123 3)) - (assert (= and-false False)) - (assert (= and-true True)) - (assert (= and-single 1)) - ; short circuiting - (setv a 1) - (and 0 (setv a 2)) - (assert (= a 1))) - -(defn test-and-#1151-do [] - (setv a (and 0 (do 2 3))) - (assert (= a 0)) - (setv a (and 1 (do 2 3))) - (assert (= a 3))) - -(defn test-and-#1151-for [] - (setv l []) - (setv x (and 0 (for [n [1 2]] (.append l n)))) - (assert (= x 0)) - (assert (= l [])) - (setv x (and 15 (for [n [1 2]] (.append l n)))) - (assert (= l [1 2]))) - -(defn test-and-#1151-del [] - (setv l ["a" "b"]) - (setv x (and 0 (del (get l 1)))) - (assert (= x 0)) - (assert (= l ["a" "b"])) - (setv x (and 15 (del (get l 1)))) - (assert (= l ["a"]))) - - -(defn test-or [] - (setv or-all-true (or 1 2 3 True "string") - or-some-true (or False "hello") - or-none-true (or False False) - or-false (or) - or-single (or 1)) - (assert (= or-all-true 1)) - (assert (= or-some-true "hello")) - (assert (= or-none-true False)) - (assert (= or-false None)) - (assert (= or-single 1)) - ; short circuiting - (setv a 1) - (or 1 (setv a 2)) - (assert (= a 1))) - -(defn test-or-#1151-do [] - (setv a (or 1 (do 2 3))) - (assert (= a 1)) - (setv a (or 0 (do 2 3))) - (assert (= a 3))) - -(defn test-or-#1151-for [] - (setv l []) - (setv x (or 15 (for [n [1 2]] (.append l n)))) - (assert (= x 15)) - (assert (= l [])) - (setv x (or 0 (for [n [1 2]] (.append l n)))) - (assert (= l [1 2]))) - -(defn test-or-#1151-del [] - (setv l ["a" "b"]) - (setv x (or 15 (del (get l 1)))) - (assert (= x 15)) - (assert (= l ["a" "b"])) - (setv x (or 0 (del (get l 1)))) - (assert (= l ["a"]))) - -(defn test-if-return-branching [] - ; thanks, kirbyfan64 - (defn f [] - (if True (setv x 1) 2) - 1) - - (assert (= 1 (f)))) - - -(defn test-keyword [] - - (assert (= :foo :foo)) - (assert (= :foo ':foo)) - (setv x :foo) - (assert (is (type x) (type ':foo))) - (assert (= (get {:foo "bar"} :foo) "bar")) - (assert (= (get {:bar "quux"} (get {:foo :bar} :foo)) "quux"))) - - -(defn test-keyword-clash [] - "Keywords shouldn't clash with normal strings." - - (assert (= (get {:foo "bar" ":foo" "quux"} :foo) "bar")) - (assert (= (get {:foo "bar" ":foo" "quux"} ":foo") "quux"))) - - -(defn test-empty-keyword [] - (assert (= : :)) - (assert (isinstance ': hy.models.Keyword)) - (assert (!= : ":")) - (assert (= (. ': name) ""))) - -(defn test-pickling-keyword [] - ; https://github.com/hylang/hy/issues/1754 - (setv x :test-keyword) - (for [protocol (range 0 (+ pickle.HIGHEST-PROTOCOL 1))] - (assert (= x - (pickle.loads (pickle.dumps x :protocol protocol)))))) - -(defn test-nested-if [] - (for [x (range 10)] - (if (in "foo" "foobar") - (do - (if True True True)) - (do - (if False False False))))) - - -(defn test-eval [] - (assert (= 2 (hy.eval (quote (+ 1 1))))) - (setv x 2) - (assert (= 4 (hy.eval (quote (+ x 2))))) - (setv test-payload (quote (+ x 2))) - (setv x 4) - (assert (= 6 (hy.eval test-payload))) - (assert (= 9 ((hy.eval (quote (fn [x] (+ 3 3 x)))) 3))) - (assert (= 1 (hy.eval (quote 1)))) - (assert (= "foobar" (hy.eval (quote "foobar")))) - (setv x (quote 42)) - (assert (= 42 (hy.eval x))) - (assert (= 27 (hy.eval (+ (quote (*)) (* [(quote 3)] 3))))) - (assert (= None (hy.eval (quote (print ""))))) - - ;; https://github.com/hylang/hy/issues/1041 - (assert (is (hy.eval 're) re)) - (assert (is ((fn [] (hy.eval 're))) re))) - - -(defn test-eval-false [] - (assert (is (hy.eval 'False) False)) - (assert (is (hy.eval 'None) None)) - (assert (= (hy.eval '0) 0)) - (assert (= (hy.eval '"") "")) - (assert (= (hy.eval 'b"") b"")) - (assert (= (hy.eval ':) :)) - (assert (= (hy.eval '[]) [])) - (assert (= (hy.eval '#()) #())) - (assert (= (hy.eval '{}) {})) - (assert (= (hy.eval '#{}) #{}))) - - -(defn test-eval-global-dict [] - (assert (= 'bar (hy.eval (quote foo) {"foo" 'bar}))) - (assert (= 1 (do (setv d {}) (hy.eval '(setv x 1) d) (hy.eval (quote x) d)))) - (setv d1 {} d2 {}) - (hy.eval '(setv x 1) d1) - (with [e (pytest.raises NameError)] - (hy.eval (quote x) d2))) - -(defn test-eval-failure [] - ; yo dawg - (with [(pytest.raises TypeError)] (hy.eval '(hy.eval))) - (defclass C) - (with [(pytest.raises TypeError)] (hy.eval (C))) - (with [(pytest.raises TypeError)] (hy.eval 'False [])) - (with [(pytest.raises TypeError)] (hy.eval 'False {} 1))) - -(defn test-eval-quasiquote [] - ; https://github.com/hylang/hy/issues/1174 - - (for [x [ - None False True - 5 5.1 - 5j 5.1j 2+1j 1.2+3.4j - "" b"" - "apple bloom" b"apple bloom" "⚘" b"\x00" - [] #{} {} - [1 2 3] #{1 2 3} {"a" 1 "b" 2}]] - (assert (= (hy.eval `(get [~x] 0)) x)) - (assert (= (hy.eval x) x))) - - (setv kw :mykeyword) - (assert (= (get (hy.eval `[~kw]) 0) kw)) - (assert (= (hy.eval kw) kw)) - - (assert (= (hy.eval #()) #())) - (assert (= (hy.eval #(1 2 3)) #(1 2 3))) - - (assert (= (hy.eval `(+ "a" ~(+ "b" "c"))) "abc")) - - (setv l ["a" "b"]) - (setv n 1) - (assert (= (hy.eval `(get ~l ~n) "b"))) - - (setv d {"a" 1 "b" 2}) - (setv k "b") - (assert (= (hy.eval `(get ~d ~k)) 2))) - - -(defn test-quote-bracket-string-delim [] - (assert (= (. '#[my delim[hello world]my delim] brackets) "my delim")) - (assert (= (. '#[[squid]] brackets) "")) - (assert (is (. '"squid" brackets) None))) - - -(defn test-format-strings [] - (assert (= f"hello world" "hello world")) - (assert (= f"hello {(+ 1 1)} world" "hello 2 world")) - (assert (= f"a{ (.upper (+ "g" "k")) }z" "aGKz")) - (assert (= f"a{1}{2}b" "a12b")) - - ; Referring to a variable - (setv p "xyzzy") - (assert (= f"h{p}j" "hxyzzyj")) - - ; Including a statement and setting a variable - (assert (= f"a{(do (setv floop 4) (* floop 2))}z" "a8z")) - (assert (= floop 4)) - - ; Comments - (assert (= f"a{(+ 1 - 2 ; This is a comment. - 3)}z" "a6z")) - - ; Newlines in replacement fields - (assert (= f"ey {"bee -cee"} dee" "ey bee\ncee dee")) - - ; Conversion characters and format specifiers - (setv p:9 "other") - (setv !r "bar") - (assert (= f"a{p !r}" "a'xyzzy'")) - (assert (= f"a{p :9}" "axyzzy ")) - (assert (= f"a{p:9}" "aother")) - (assert (= f"a{p !r :9}" "a'xyzzy' ")) - (assert (= f"a{p !r:9}" "a'xyzzy' ")) - (assert (= f"a{p:9 :9}" "aother ")) - (assert (= f"a{!r}" "abar")) - (assert (= f"a{!r !r}" "a'bar'")) - - ; Fun with `r` - (assert (= f"hello {r"\n"}" r"hello \n")) - (assert (= f"hello {"\n"}" "hello \n")) - - ; Braces escaped via doubling - (assert (= f"ab{{cde" "ab{cde")) - (assert (= f"ab{{cde}}}}fg{{{{{{" "ab{cde}}fg{{{")) - (assert (= f"ab{{{(+ 1 1)}}}" "ab{2}")) - - ; Nested replacement fields - (assert (= f"{2 :{(+ 2 2)}}" " 2")) - (setv value 12.34 width 10 precision 4) - (assert (= f"result: {value :{width}.{precision}}" "result: 12.34")) - - ; Nested replacement fields with ! and : - (defclass C [object] - (defn __format__ [self format-spec] - (+ "C[" format-spec "]"))) - (assert (= f"{(C) : {(str (+ 1 1)) !r :x<5}}" "C[ '2'xx]")) - - ; \N sequences - ; https://github.com/hylang/hy/issues/2321 - (setv ampersand "wich") - (assert (= f"sand{ampersand} \N{ampersand} chips" "sandwich & chips")) - - ; Format bracket strings - (assert (= #[f[a{p !r :9}]f] "a'xyzzy' ")) - (assert (= #[f-string[result: {value :{width}.{precision}}]f-string] - "result: 12.34")) - - ; Quoting shouldn't evaluate the f-string immediately - ; https://github.com/hylang/hy/issues/1844 - (setv quoted 'f"hello {world}") - (assert (isinstance quoted hy.models.FString)) - (with [(pytest.raises NameError)] - (hy.eval quoted)) - (setv world "goodbye") - (assert (= (hy.eval quoted) "hello goodbye")) - - ;; '=' debugging syntax. - (setv foo "bar") - (assert (= f"{foo =}" "foo ='bar'")) - - ;; Whitespace is preserved. - (assert (= f"xyz{ foo = }" "xyz foo = 'bar'")) - - ;; Explicit conversion is applied. - (assert (= f"{ foo = !s}" " foo = bar")) - - ;; Format spec supercedes implicit conversion. - (setv pi 3.141593 fill "_") - (assert (= f"{pi = :{fill}^8.2f}" "pi = __3.14__")) - - ;; Format spec doesn't clobber the explicit conversion. - (with [(pytest.raises - ValueError - :match r"Unknown format code '?f'? for object of type 'str'")] - f"{pi =!s:.3f}") - - ;; Nested "=" is parsed, but fails at runtime, like Python. - (setv width 7) - (with [(pytest.raises - ValueError - :match r"I|invalid format spec(?:ifier)?")] - f"{pi =:{fill =}^{width =}.2f}")) - - -(defn test-format-string-repr-roundtrip [] - (for [orig [ - 'f"hello {(+ 1 1)} world" - 'f"a{p !r:9}" - 'f"{ foo = !s}"]] - (setv new (eval (repr orig))) - (assert (= (len new) (len orig))) - (for [[n o] (zip new orig)] - (when (hasattr o "conversion") - (assert (= n.conversion o.conversion))) - (assert (= n o))))) - - -(defn test-repr-with-brackets [] - (assert (= (repr '"foo") "hy.models.String('foo')")) - (assert (= (repr '#[[foo]]) "hy.models.String('foo', brackets='')")) - (assert (= (repr '#[xx[foo]xx]) "hy.models.String('foo', brackets='xx')")) - (assert (= (repr '#[xx[]xx]) "hy.models.String('', brackets='xx')")) - - (for [g [repr str]] - (defn f [x] (re.sub r"\n\s+" "" (g x) :count 1)) - (assert (= (f 'f"foo") - "hy.models.FString([hy.models.String('foo')])")) - (assert (= (f '#[f[foo]f]) - "hy.models.FString([hy.models.String('foo')], brackets='f')")) - (assert (= (f '#[f-x[foo]f-x]) - "hy.models.FString([hy.models.String('foo')], brackets='f-x')")) - (assert (= (f '#[f-x[]f-x]) - "hy.models.FString(brackets='f-x')")))) - - -(defn test-import-syntax [] - ;; Simple import - (import sys os) - - ;; from os.path import basename - (import os.path [basename]) - (assert (= (basename "/some/path") "path")) - - ;; import os.path as p - (import os.path :as p) - (assert (= p.basename basename)) - - ;; from os.path import basename as bn - (import os.path [basename :as bn]) - (assert (= bn basename)) - - ;; Multiple stuff to import - (import sys - os.path [dirname] - os.path :as op - os.path [dirname :as dn]) - (assert (= (dirname "/some/path") "/some")) - (assert (= op.dirname dirname)) - (assert (= dn dirname))) - - -(defn test-relative-import [] - (import ..resources [tlib in-init]) - (assert (= tlib.SECRET-MESSAGE "Hello World")) - (assert (= in-init "chippy")) - (import .. [resources]) - (assert (= resources.in-init "chippy"))) - - -(defn test-lambda-keyword-lists [] - (defn foo [x #* xs #** kw] [x xs kw]) - (assert (= (foo 10 20 30) [10 #(20 30) {}]))) - - -(defn test-optional-arguments [] - (defn foo [a b [c None] [d 42]] [a b c d]) - (assert (= (foo 1 2) [1 2 None 42])) - (assert (= (foo 1 2 3) [1 2 3 42])) - (assert (= (foo 1 2 3 4) [1 2 3 4]))) - - -(defn test-undefined-name [] - (with [(pytest.raises NameError)] - xxx)) - - -(defn test-if-in-if [] - (assert (= 42 - (if (if 1 True False) - 42 - 43))) - (assert (= 43 - (if (if 0 True False) - 42 - 43)))) - - -(defn test-try-except-return [] - "Ensure we can return from an `except` form." - (assert (= ((fn [] (try xxx (except [NameError] (+ 1 1))))) 2)) - (setv foo (try xxx (except [NameError] (+ 1 1)))) - (assert (= foo 2)) - (setv foo (try (+ 2 2) (except [NameError] (+ 1 1)))) - (assert (= foo 4))) - - -(defn test-try-else-return [] - "Ensure we can return from the `else` clause of a `try`." - ; https://github.com/hylang/hy/issues/798 - - (assert (= "ef" ((fn [] - (try (+ "a" "b") - (except [NameError] (+ "c" "d")) - (else (+ "e" "f"))))))) - - (setv foo - (try (+ "A" "B") - (except [NameError] (+ "C" "D")) - (else (+ "E" "F")))) - (assert (= foo "EF")) - - ; Check that the lvalue isn't assigned in the main `try` body - ; there's an `else`. - (setv x 1) - (setv y 0) - (setv x - (try (+ "G" "H") - (except [NameError] (+ "I" "J")) - (else - (setv y 1) - (assert (= x 1)) - (+ "K" "L")))) - (assert (= x "KL")) - (assert (= y 1))) - - -(defn test-require [] - (with [(pytest.raises NameError)] - (qplah 1 2 3 4)) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) - (with [(pytest.raises NameError)] - (✈ 1 2 3 4)) - (with [(pytest.raises NameError)] - (hyx_XairplaneX 1 2 3 4)) - - (require tests.resources.tlib [qplah]) - (assert (= (qplah 1 2 3) [8 1 2 3])) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) - - (require tests.resources.tlib) - (assert (= (tests.resources.tlib.parald 1 2 3) [9 1 2 3])) - (assert (= (tests.resources.tlib.✈ "silly") "plane silly")) - (assert (= (tests.resources.tlib.hyx_XairplaneX "foolish") "plane foolish")) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) - - (require tests.resources.tlib :as T) - (assert (= (T.parald 1 2 3) [9 1 2 3])) - (assert (= (T.✈ "silly") "plane silly")) - (assert (= (T.hyx_XairplaneX "foolish") "plane foolish")) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) - - (require tests.resources.tlib [parald :as p]) - (assert (= (p 1 2 3) [9 1 2 3])) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) - - (require tests.resources.tlib *) - (assert (= (parald 1 2 3) [9 1 2 3])) - (assert (= (✈ "silly") "plane silly")) - (assert (= (hyx_XairplaneX "foolish") "plane foolish")) - - (require tests.resources [tlib macros :as m exports-none]) - (assert (in "tlib.qplah" __macros__)) - (assert (in (hy.mangle "m.test-macro") __macros__)) - (assert (in (hy.mangle "exports-none.cinco") __macros__)) - (require os [path]) - (with [(pytest.raises hy.errors.HyRequireError)] - (hy.eval '(require tests.resources [does-not-exist]))) - - (require tests.resources.exports *) - (assert (= (casey 1 2 3) [11 1 2 3])) - (assert (= (☘ 1 2 3) [13 1 2 3])) - (with [(pytest.raises NameError)] - (brother 1 2 3 4))) - - -(defn test-require-native [] - (with [(pytest.raises NameError)] - (test-macro-2)) - (import tests.resources.macros) - (with [(pytest.raises NameError)] - (test-macro-2)) - (require tests.resources.macros [test-macro-2]) - (test-macro-2) - (assert (= qup 2))) - - -(defn test-relative-require [] - (require ..resources.macros [test-macro]) - (assert (in "test_macro" __macros__)) - - (require .language-beside [xyzzy]) - (assert (in "xyzzy" __macros__)) - - (require . [language-beside :as lb]) - (assert (in "lb.xyzzy" __macros__))) - - -(defn test-export-objects [] - ; We use `hy.eval` here because of a Python limitation that - ; importing `*` is only allowed at the module level. - (hy.eval '(do - (import tests.resources.exports *) - (assert (= (jan) 21)) - (assert (= (♥) 23)) - (with [(pytest.raises NameError)] - (wayne)) - (import tests.resources.exports [wayne]) - (assert (= (wayne) 22))))) - - -(defn test-encoding-nightmares [] - (assert (= (len "ℵℵℵ♥♥♥\t♥♥\r\n") 11))) - - -(defn test-keyword-get [] - - (assert (= (:foo (dict :foo "test")) "test")) - (setv f :foo) - (assert (= (f (dict :foo "test")) "test")) - - (assert (= (:foo-bar (dict :foo-bar "baz")) "baz")) - (assert (= (:♥ (dict :♥ "heart")) "heart")) - (defclass C [] - (defn __getitem__ [self k] - k)) - (assert (= (:♥ (C)) "hyx_Xblack_heart_suitX")) - - (with [(pytest.raises KeyError)] (:foo (dict :a 1 :b 2))) - (assert (= (:foo (dict :a 1 :b 2) 3) 3)) - (assert (= (:foo (dict :a 1 :b 2 :foo 5) 3) 5)) - - (with [(pytest.raises TypeError)] (:foo "Hello World")) - (with [(pytest.raises TypeError)] (:foo (object))) - - ; The default argument should work regardless of the collection type. - (defclass G [object] - (defn __getitem__ [self k] - (raise KeyError))) - (assert (= (:foo (G) 15) 15))) - - -(defn test-break-breaking [] - (defn holy-grail [] (for [x (range 10)] (when (= x 5) (break))) x) - (assert (= (holy-grail) 5))) - - -(defn test-continue-continuation [] - (setv y []) - (for [x (range 10)] - (when (!= x 5) - (continue)) - (.append y x)) - (assert (= y [5]))) - - -(defn test-del [] - (setv foo 42) - (assert (= foo 42)) - (del foo) - (with [(pytest.raises NameError)] - foo) - (setv test (list (range 5))) - (del (get test 4)) - (assert (= test [0 1 2 3])) - (del (get test 2)) - (assert (= test [0 1 3])) - (assert (= (del) None))) - - -(defn test-macroexpand [] - (assert (= (hy.macroexpand '(mac (a b) (x y))) - '(x y (a b)))) - (assert (= (hy.macroexpand '(mac (a b) (mac 5))) - '(a b 5)))) - -(defn test-macroexpand-with-named-import [] - ; https://github.com/hylang/hy/issues/1207 - (defmacro m-with-named-import [] - (import math [pow]) - (pow 2 3)) - (assert (= (hy.macroexpand '(m-with-named-import)) (hy.models.Float (** 2 3))))) - -(defn test-macroexpand-1 [] - (assert (= (hy.macroexpand-1 '(mac (a b) (mac 5))) - '(mac 5 (a b))))) - -(defn test-disassemble [] - (defn nos [x] (re.sub r"\s" "" x)) - (assert (= (nos (hy.disassemble '(do (leaky) (leaky) (macros)))) - (nos (.format - "Module( - body=[Expr(value=Call(func=Name(id='leaky', ctx=Load()), args=[], keywords=[])), - Expr(value=Call(func=Name(id='leaky', ctx=Load()), args=[], keywords=[])), - Expr(value=Call(func=Name(id='macros', ctx=Load()), args=[], keywords=[]))]{})" - (if PY3_8 ",type_ignores=[]" ""))))) - (assert (= (nos (hy.disassemble '(do (leaky) (leaky) (macros)) True)) - "leaky()leaky()macros()")) - (assert (= (re.sub r"[()\n ]" "" (hy.disassemble `(+ ~(+ 1 1) 40) True)) - "2+40"))) - - -(defn test-attribute-access [] - (defclass mycls [object]) - - (setv foo [(mycls) (mycls) (mycls)]) - (assert (is (. foo) foo)) - (assert (is (. foo [0]) (get foo 0))) - (assert (is (. foo [0] __class__) mycls)) - (assert (is (. foo [1] __class__) mycls)) - (assert (is (. foo [(+ 1 1)] __class__) mycls)) - (assert (= (. foo [(+ 1 1)] __class__ __name__ [0]) "m")) - (assert (= (. foo [(+ 1 1)] __class__ __name__ [1]) "y")) - (assert (= (. os (getcwd) (isalpha) __class__ __name__ [0]) "b")) - (assert (= (. "ab hello" (strip "ab ") (upper)) "HELLO")) - (assert (= (. "hElLO\twoRld" (expandtabs :tabsize 4) (lower)) "hello world")) - - (setv bar (mycls)) - (setv (. foo [1]) bar) - (assert (is bar (get foo 1))) - (setv (. foo [1] test) "hello") - (assert (= (getattr (. foo [1]) "test") "hello"))) - -(defn test-only-parse-lambda-list-in-defn [] - (with [(pytest.raises NameError)] - (setv x [#* spam] y 1))) - -(defn test-read-file-object [] - (import io [StringIO]) - - (setv stdin-buffer (StringIO "(+ 2 2)\n(- 2 2)")) - (assert (= (hy.eval (hy.read stdin-buffer)) 4)) - (assert (isinstance (hy.read stdin-buffer) hy.models.Expression)) - - ; Multiline test - (setv stdin-buffer (StringIO "(\n+\n41\n1\n)\n(-\n2\n1\n)")) - (assert (= (hy.eval (hy.read stdin-buffer)) 42)) - (assert (= (hy.eval (hy.read stdin-buffer)) 1)) - - ; EOF test - (setv stdin-buffer (StringIO "(+ 2 2)")) - (hy.read stdin-buffer) - (with [(pytest.raises EOFError)] - (hy.read stdin-buffer))) - -(defn test-read-str [] - (assert (= (hy.read "(print 1)") '(print 1))) - (assert (is (type (hy.read "(print 1)")) (type '(print 1)))) - - ; Watch out for false values: https://github.com/hylang/hy/issues/1243 - (assert (= (hy.read "\"\"") '"")) - (assert (is (type (hy.read "\"\"")) (type '""))) - (assert (= (hy.read "[]") '[])) - (assert (is (type (hy.read "[]")) (type '[]))) - (assert (= (hy.read "0") '0)) - (assert (is (type (hy.read "0")) (type '0)))) - -(defn test-keyword-creation [] - (assert (= (hy.models.Keyword "foo") :foo)) - (assert (= (hy.models.Keyword "foo_bar") :foo_bar)) - (assert (= (hy.models.Keyword "foo-bar") :foo-bar)) - (assert (!= :foo_bar :foo-bar)) - (assert (= (hy.models.Keyword "") :))) - -(defn test-keywords-in-fn-calls [] - (assert (= (kwtest) {})) - (assert (= (kwtest :key "value") {"key" "value"})) - (assert (= (kwtest :key-with-dashes "value") {"key_with_dashes" "value"})) - (assert (= (kwtest :result (+ 1 1)) {"result" 2})) - (assert (= (kwtest :key (kwtest :key2 "value")) {"key" {"key2" "value"}})) - (assert (= ((get (kwtest :key (fn [x] (* x 2))) "key") 3) 6))) - -(defmacro identify-keywords [#* elts] - `(list - (map - (fn [x] (if (isinstance x hy.models.Keyword) "keyword" "other")) - ~elts))) - -(defn test-keywords-and-macros [] - "Macros should still be able to handle keywords as they best see fit." - (assert - (= (identify-keywords 1 "bloo" :foo) - ["other" "other" "keyword"]))) - - -(defn test-docstrings [] - (defn f [] "docstring" 5) - (assert (= (. f __doc__) "docstring")) - - ; a single string is the return value, not a docstring - ; (https://github.com/hylang/hy/issues/1402) - (defn f3 [] "not a docstring") - (assert (is (. f3 __doc__) None)) - (assert (= (f3) "not a docstring"))) - -(defn test-module-docstring [] - (import tests.resources.module-docstring-example :as m) - (assert (= m.__doc__ "This is the module docstring.")) - (assert (= m.foo 5))) - - -(defn test-exception-cause [] - (assert (is NameError (type (. - (try - (raise ValueError :from NameError) - (except [e [ValueError]] e)) - __cause__))))) - - -(defn test-kwonly [] - ;; keyword-only with default works - (defn kwonly-foo-default-false [* [foo False]] foo) - (assert (= (kwonly-foo-default-false) False)) - (assert (= (kwonly-foo-default-false :foo True) True)) - ;; keyword-only without default ... - (defn kwonly-foo-no-default [* foo] foo) - (with [e (pytest.raises TypeError)] - (kwonly-foo-no-default)) - (assert (in "missing 1 required keyword-only argument: 'foo'" - (. e value args [0]))) - ;; works - (assert (= (kwonly-foo-no-default :foo "quux") "quux")) - ;; keyword-only with other arg types works - (defn function-of-various-args [a b #* args foo #** kwargs] - #(a b args foo kwargs)) - (assert (= (function-of-various-args 1 2 3 4 :foo 5 :bar 6 :quux 7) - #(1 2 #(3 4) 5 {"bar" 6 "quux" 7})))) - - -(defn test-extended-unpacking-1star-lvalues [] - (setv [x #*y] [1 2 3 4]) - (assert (= x 1)) - (assert (= y [2 3 4])) - (setv [a #*b c] "ghijklmno") - (assert (= a "g")) - (assert (= b (list "hijklmn"))) - (assert (= c "o"))) - - -(defn test-yield-from [] - (defn yield-from-test [] - (for [i (range 3)] - (yield i)) - (yield-from [1 2 3])) - (assert (= (list (yield-from-test)) [0 1 2 1 2 3]))) - - -(defn test-yield-from-exception-handling [] - (defn yield-from-subgenerator-test [] - (yield 1) - (yield 2) - (yield 3) - (/ 1 0)) - (defn yield-from-test [] - (for [i (range 3)] - (yield i)) - (try - (yield-from (yield-from-subgenerator-test)) - (except [e ZeroDivisionError] - (yield 4)))) - (assert (= (list (yield-from-test)) [0 1 2 1 2 3 4]))) - -(defn test-pep-3115 [] - (defclass member-table [dict] - (defn __init__ [self] - (setv self.member-names [])) - - (defn __setitem__ [self key value] - (when (not-in key self) - (.append self.member-names key)) - (dict.__setitem__ self key value))) - - (defclass OrderedClass [type] - (setv __prepare__ (classmethod (fn [metacls name bases] - (member-table)))) - - (defn __new__ [cls name bases classdict] - (setv result (type.__new__ cls name bases (dict classdict))) - (setv result.member-names classdict.member-names) - result)) - - (defclass MyClass [:metaclass OrderedClass] - (defn method1 [self] (pass)) - (defn method2 [self] (pass))) - - (assert (= (. (MyClass) member-names) - ["__module__" "__qualname__" "method1" "method2"]))) - - -(defn test-unpacking-pep448-1star [] - (setv l [1 2 3]) - (setv p [4 5]) - (assert (= ["a" #*l "b" #*p #*l] ["a" 1 2 3 "b" 4 5 1 2 3])) - (assert (= #("a" #*l "b" #*p #*l) #("a" 1 2 3 "b" 4 5 1 2 3))) - (assert (= #{"a" #*l "b" #*p #*l} #{"a" "b" 1 2 3 4 5})) - (defn f [#* args] args) - (assert (= (f "a" #*l "b" #*p #*l) #("a" 1 2 3 "b" 4 5 1 2 3))) - (assert (= (+ #*l #*p) 15)) - (assert (= (and #*l) 3))) - - -(defn test-unpacking-pep448-2star [] - (setv d1 {"a" 1 "b" 2}) - (setv d2 {"c" 3 "d" 4}) - (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"})) - (defn fun [[a None] [b None] [c None] [d None] [e None] [f None]] - [a b c d e f]) - (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None]))) - - -(defn test-fn/a [] - (assert (= (asyncio.run ((fn/a [] (await (asyncio.sleep 0)) [1 2 3]))) - [1 2 3]))) - - -(defn test-defn/a [] - (defn/a coro-test [] - (await (asyncio.sleep 0)) - [1 2 3]) - (assert (= (asyncio.run (coro-test)) [1 2 3]))) - - -(defn test-decorated-defn/a [] - (defn decorator [func] (fn/a [] (/ (await (func)) 2))) - - (defn/a [decorator] coro-test [] - (await (asyncio.sleep 0)) - 42) - (assert (= (asyncio.run (coro-test)) 21))) - - -(defn test-single-with/a [] - (asyncio.run - ((fn/a [] - (with/a [t (AsyncWithTest 1)] - (assert (= t 1))))))) - -(defn test-two-with/a [] - (asyncio.run - ((fn/a [] - (with/a [t1 (AsyncWithTest 1) - t2 (AsyncWithTest 2)] - (assert (= t1 1)) - (assert (= t2 2))))))) - -(defn test-thrice-with/a [] - (asyncio.run - ((fn/a [] - (with/a [t1 (AsyncWithTest 1) - t2 (AsyncWithTest 2) - t3 (AsyncWithTest 3)] - (assert (= t1 1)) - (assert (= t2 2)) - (assert (= t3 3))))))) - -(defn test-quince-with/a [] - (asyncio.run - ((fn/a [] - (with/a [t1 (AsyncWithTest 1) - t2 (AsyncWithTest 2) - t3 (AsyncWithTest 3) - _ (AsyncWithTest 4)] - (assert (= t1 1)) - (assert (= t2 2)) - (assert (= t3 3))))))) - -(defn test-for-async [] - (defn/a numbers [] - (for [i [1 2]] - (yield i))) - - (asyncio.run - ((fn/a [] - (setv x 0) - (for [:async a (numbers)] - (setv x (+ x a))) - (assert (= x 3)))))) - -(defn test-for-async-else [] - (defn/a numbers [] - (for [i [1 2]] - (yield i))) - - (asyncio.run - ((fn/a [] - (setv x 0) - (for [:async a (numbers)] - (setv x (+ x a)) - (else (setv x (+ x 50)))) - (assert (= x 53)))))) - -(defn test-variable-annotations [] - (defclass AnnotationContainer [] - (setv #^int x 1 y 2) - (#^bool z)) - - (setv annotations (get-type-hints AnnotationContainer)) - (assert (= (get annotations "x") int)) - (assert (= (get annotations "z") bool))) - -(defn test-pep-487 [] - (defclass QuestBase [] - (defn __init-subclass__ [cls swallow #** kwargs] - (setv cls.swallow swallow))) - - (defclass Quest [QuestBase :swallow "african"]) - (assert (= (. (Quest) swallow) "african"))) - -(defn test-eval-foo-compile-return-values [] - (eval-and-compile (setv jim 0)) - - (setv derrick (eval-and-compile (+= jim 1) 2)) - (assert (= jim 1)) - (assert (= derrick 2)) - - (setv derrick (eval-and-compile)) - (assert (is derrick None)) - - (setv derrick 3) - (setv derrick (eval-when-compile (+= jim 1) 2)) - (assert (= jim 1)) - (assert (is derrick None))) diff --git a/tests/native_tests/logic_short_circuit.hy b/tests/native_tests/logic_short_circuit.hy new file mode 100644 index 000000000..2ac651295 --- /dev/null +++ b/tests/native_tests/logic_short_circuit.hy @@ -0,0 +1,83 @@ +;; More basic tests of `and` and `or` can be found in `operators.hy`. + + +(defn test-and [] + (setv and123 (and 1 2 3) + and-false (and 1 False 3) + and-true (and) + and-single (and 1)) + (assert (= and123 3)) + (assert (= and-false False)) + (assert (= and-true True)) + (assert (= and-single 1)) + ; short circuiting + (setv a 1) + (and 0 (setv a 2)) + (assert (= a 1))) + + +(defn test-and-#1151-do [] + (setv a (and 0 (do 2 3))) + (assert (= a 0)) + (setv a (and 1 (do 2 3))) + (assert (= a 3))) + + +(defn test-and-#1151-for [] + (setv l []) + (setv x (and 0 (for [n [1 2]] (.append l n)))) + (assert (= x 0)) + (assert (= l [])) + (setv x (and 15 (for [n [1 2]] (.append l n)))) + (assert (= l [1 2]))) + + +(defn test-and-#1151-del [] + (setv l ["a" "b"]) + (setv x (and 0 (del (get l 1)))) + (assert (= x 0)) + (assert (= l ["a" "b"])) + (setv x (and 15 (del (get l 1)))) + (assert (= l ["a"]))) + + +(defn test-or [] + (setv or-all-true (or 1 2 3 True "string") + or-some-true (or False "hello") + or-none-true (or False False) + or-false (or) + or-single (or 1)) + (assert (= or-all-true 1)) + (assert (= or-some-true "hello")) + (assert (= or-none-true False)) + (assert (= or-false None)) + (assert (= or-single 1)) + ; short circuiting + (setv a 1) + (or 1 (setv a 2)) + (assert (= a 1))) + + +(defn test-or-#1151-do [] + (setv a (or 1 (do 2 3))) + (assert (= a 1)) + (setv a (or 0 (do 2 3))) + (assert (= a 3))) + + +(defn test-or-#1151-for [] + (setv l []) + (setv x (or 15 (for [n [1 2]] (.append l n)))) + (assert (= x 15)) + (assert (= l [])) + (setv x (or 0 (for [n [1 2]] (.append l n)))) + (assert (= l [1 2]))) + + +(defn test-or-#1151-del [] + (setv l ["a" "b"]) + (setv x (or 15 (del (get l 1)))) + (assert (= x 15)) + (assert (= l ["a" "b"])) + (setv x (or 0 (del (get l 1)))) + (assert (= l ["a"]))) diff --git a/tests/native_tests/macros.hy b/tests/native_tests/macros.hy index 30777494d..01bc44365 100644 --- a/tests/native_tests/macros.hy +++ b/tests/native_tests/macros.hy @@ -1,5 +1,4 @@ (import os sys - importlib pytest hy.errors [HySyntaxError HyTypeError HyMacroExpansionError]) @@ -7,6 +6,14 @@ "Execute the `body` statements in reverse" (quasiquote (do (unquote-splice (list (reversed body)))))) +(defmacro mac [x expr] + `(~@expr ~x)) + + + +(defn test-macro-call-in-called-lambda [] + (assert (= ((fn [] (mac 2 (- 10 1)))) 7))) + (defn test-stararged-native-macro [] (setv x []) @@ -97,34 +104,6 @@ (assert (= (f) 3)) (assert (= f.__doc__ "hello world"))) -(defn test-midtree-yield [] - "Test yielding with a returnable." - (defn kruft [] (yield) (+ 1 1))) - -(defn test-midtree-yield-in-for [] - "Test yielding in a for with a return." - (defn kruft-in-for [] - (for [i (range 5)] - (yield i)) - (+ 1 2))) - -(defn test-midtree-yield-in-while [] - "Test yielding in a while with a return." - (defn kruft-in-while [] - (setv i 0) - (while (< i 5) - (yield i) - (setv i (+ i 1))) - (+ 2 3))) - -(defn test-multi-yield [] - (defn multi-yield [] - (for [i (range 3)] - (yield i)) - (yield "a") - (yield "end")) - (assert (= (list (multi-yield)) [0 1 2 "a" "end"]))) - ; Macro that checks a variable defined at compile or load time (setv phase "load") @@ -168,161 +147,6 @@ (assert (not (= s1 s2)))) -(defn test-macro-namespace-resolution [] - "Confirm that local versions of macro-macro dependencies do not shadow the -versions from the macro's own module, but do resolve unbound macro references -in expansions." - - ;; `nonlocal-test-macro` is a macro used within - ;; `tests.resources.macro-with-require.test-module-macro`. - ;; Here, we introduce an equivalently named version in local scope that, when - ;; used, will expand to a different output string. - (defmacro nonlocal-test-macro [x] - (print "this is the local version of `nonlocal-test-macro`!")) - - ;; Was the above macro created properly? - (assert (in "nonlocal_test_macro" __macros__)) - - (setv nonlocal-test-macro (get __macros__ "nonlocal_test_macro")) - - (require tests.resources.macro-with-require *) - - (setv module-name-var "tests.native_tests.native_macros.test-macro-namespace-resolution") - (assert (= (+ "This macro was created in tests.resources.macros, " - "expanded in tests.native_tests.native_macros.test-macro-namespace-resolution " - "and passed the value 2.") - (test-module-macro 2))) - - ;; Now, let's use a `require`d macro that depends on another macro defined only - ;; in this scope. - (defmacro local-test-macro [x] - (.format "This is the local version of `nonlocal-test-macro` returning {}!" (int x))) - - (assert (= "This is the local version of `nonlocal-test-macro` returning 3!" - (test-module-macro-2 3)))) - -(defn test-requires-pollutes-core [] - ;; https://github.com/hylang/hy/issues/1978 - ;; Macros loaded from an external module should not pollute `__macros__` - ;; with macros from core. - - (setv pyc-file (importlib.util.cache-from-source - (os.path.realpath - (os.path.join - "tests" "resources" "macros.hy")))) - - ;; Remove any cached byte-code, so that this runs from source and - ;; gets evaluated in this module. - (when (os.path.isfile pyc-file) - (os.unlink pyc-file) - (.clear sys.path_importer_cache) - (when (in "tests.resources.macros" sys.modules) - (del (get sys.modules "tests.resources.macros")) - (__macros__.clear))) - - ;; Ensure that bytecode isn't present when we require this module. - (assert (not (os.path.isfile pyc-file))) - - (defn require-macros [] - (require tests.resources.macros :as m) - - (assert (in (hy.mangle "m.test-macro") __macros__)) - (for [macro-name __macros__] - (assert (not (and (in "with" macro-name) - (!= "with" macro-name)))))) - - (require-macros) - - ;; Now that bytecode is present, reload the module, clear the `require`d - ;; macros and tags, and rerun the tests. - (assert (os.path.isfile pyc-file)) - - ;; Reload the module and clear the local macro context. - (.clear sys.path_importer_cache) - (del (get sys.modules "tests.resources.macros")) - (.clear __macros__) - - (require-macros)) - -(defn [(pytest.mark.xfail)] test-macro-from-module [] - " - Macros loaded from an external module, which itself `require`s macros, should - work without having to `require` the module's macro dependencies (due to - [minimal] macro namespace resolution). - - In doing so we also confirm that a module's `__macros__` attribute is correctly - loaded and used. - - Additionally, we confirm that `require` statements are executed via loaded bytecode. - " - - (setv pyc-file (importlib.util.cache-from-source - (os.path.realpath - (os.path.join - "tests" "resources" "macro_with_require.hy")))) - - ;; Remove any cached byte-code, so that this runs from source and - ;; gets evaluated in this module. - (when (os.path.isfile pyc-file) - (os.unlink pyc-file) - (.clear sys.path_importer_cache) - (when (in "tests.resources.macro_with_require" sys.modules) - (del (get sys.modules "tests.resources.macro_with_require")) - (__macros__.clear))) - - ;; Ensure that bytecode isn't present when we require this module. - (assert (not (os.path.isfile pyc-file))) - - (defn test-requires-and-macros [] - (require tests.resources.macro-with-require - [test-module-macro]) - - ;; Make sure that `require` didn't add any of its `require`s - (assert (not (in (hy.mangle "nonlocal-test-macro") __macros__))) - ;; and that it didn't add its tags. - (assert (not (in (hy.mangle "#test-module-tag") __macros__))) - - ;; Now, require everything. - (require tests.resources.macro-with-require *) - - ;; Again, make sure it didn't add its required macros and/or tags. - (assert (not (in (hy.mangle "nonlocal-test-macro") __macros__))) - - ;; Its tag(s) should be here now. - (assert (in (hy.mangle "#test-module-tag") __macros__)) - - ;; The test macro expands to include this symbol. - (setv module-name-var "tests.native_tests.native_macros") - (assert (= (+ "This macro was created in tests.resources.macros, " - "expanded in tests.native_tests.native_macros " - "and passed the value 1.") - (test-module-macro 1)))) - - (test-requires-and-macros) - - ;; Now that bytecode is present, reload the module, clear the `require`d - ;; macros and tags, and rerun the tests. - (assert (os.path.isfile pyc-file)) - - ;; Reload the module and clear the local macro context. - (.clear sys.path_importer_cache) - (del (get sys.modules "tests.resources.macro_with_require")) - (.clear __macros__) - - ;; There doesn't seem to be a way--via standard import mechanisms--to - ;; ensure that an imported module used the cached bytecode. We'll simply have - ;; to trust that the .pyc loading convention was followed. - (test-requires-and-macros)) - - -(defn test-recursive-require-star [] - "(require foo *) should pull in macros required by `foo`." - (require tests.resources.macro-with-require *) - - (test-macro) - (assert (= blah 1))) - - (defn test-macro-errors [] (import traceback hy.importer [read-many]) diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy index 757e732bb..ebea9e992 100644 --- a/tests/native_tests/operators.hy +++ b/tests/native_tests/operators.hy @@ -315,6 +315,12 @@ (assert (= (f {"x" {"y" {"z" 12}}} "x" "y" "z") 12))) +(defn test-setv-get [] + (setv foo [0 1 2]) + (setv (get foo 0) 12) + (assert (= (get foo 0) 12))) + + (defn test-chained-comparison [] (assert (chainc 2 = (+ 1 1) = (- 3 1))) (assert (not (chainc 2 = (+ 1 1) = (+ 3 1)))) diff --git a/tests/native_tests/other.hy b/tests/native_tests/other.hy new file mode 100644 index 000000000..30f0f81fd --- /dev/null +++ b/tests/native_tests/other.hy @@ -0,0 +1,52 @@ +;; Tests miscellaneous features of the language not covered elsewhere + +(import + typing [get-type-hints] + pytest + hy.errors [HyLanguageError]) + + +(defn test-illegal-assignments [] + (for [form '[ + (setv (do 1 2) 1) + (setv 1 1) + (setv {1 2} 1) + (del 1 1) + ; https://github.com/hylang/hy/issues/1780 + (setv None 1) + (setv False 1) + (setv True 1) + (defn None [] (print "hello")) + (defn True [] (print "hello")) + (defn f [True] (print "hello")) + (for [True [1 2 3]] (print "hello")) + (lfor True [1 2 3] True) + (lfor :setv True 1 True) + (with [True x] (print "hello")) + (try 1 (except [True AssertionError] 2)) + (defclass True [])]] + (with [e (pytest.raises HyLanguageError)] + (hy.eval form)) + (assert (in "Can't assign" e.value.msg)))) + + +(defn test-no-str-as-sym [] + "Don't treat strings as symbols in the calling position" + (with [(pytest.raises TypeError)] ("setv" True 3)) ; A special form + (with [(pytest.raises TypeError)] ("abs" -2)) ; A function + (with [(pytest.raises TypeError)] ("when" 1 2))) ; A macro + + +(defn test-undefined-name [] + (with [(pytest.raises NameError)] + xxx)) + + +(defn test-variable-annotations [] + (defclass AnnotationContainer [] + (setv #^int x 1 y 2) + (#^bool z)) + + (setv annotations (get-type-hints AnnotationContainer)) + (assert (= (get annotations "x") int)) + (assert (= (get annotations "z") bool))) diff --git a/tests/native_tests/setv.hy b/tests/native_tests/setv.hy new file mode 100644 index 000000000..a755c46b5 --- /dev/null +++ b/tests/native_tests/setv.hy @@ -0,0 +1,91 @@ +(import + pytest) + + +(defn test-setv [] + (setv x 1) + (setv y 1) + (assert (= x y)) + (setv y 12) + (setv x y) + (assert (= x 12)) + (assert (= y 12)) + (setv y (fn [x] 9)) + (setv x y) + (assert (= (x y) 9)) + (assert (= (y x) 9)) + (try (do (setv a.b 1) (assert False)) + (except [e [NameError]] (assert (in "name 'a' is not defined" (str e))))) + (try (do (setv b.a (fn [x] x)) (assert False)) + (except [e [NameError]] (assert (in "name 'b' is not defined" (str e))))) + (import itertools) + (setv foopermutations (fn [x] (itertools.permutations x))) + (setv p (set [#(1 3 2) #(3 2 1) #(2 1 3) #(3 1 2) #(1 2 3) #(2 3 1)])) + (assert (= (set (itertools.permutations [1 2 3])) p)) + (assert (= (set (foopermutations [3 1 2])) p)) + (setv permutations- itertools.permutations) + (setv itertools.permutations (fn [x] 9)) + (assert (= (itertools.permutations p) 9)) + (assert (= (foopermutations foopermutations) 9)) + (setv itertools.permutations permutations-) + (assert (= (set (itertools.permutations [2 1 3])) p)) + (assert (= (set (foopermutations [2 3 1])) p))) + + +(defn test-setv-pairs [] + (setv a 1 b 2) + (assert (= a 1)) + (assert (= b 2)) + (setv y 0 x 1 y x) + (assert (= y 1)) + (with [(pytest.raises hy.errors.HyLanguageError)] + (hy.eval '(setv a 1 b)))) + + +(defn test-setv-returns-none [] + + (defn an [x] + (assert (is x None))) + + (an (setv)) + (an (setv x 1)) + (assert (= x 1)) + (an (setv x 2)) + (assert (= x 2)) + (an (setv y 2 z 3)) + (assert (= y 2)) + (assert (= z 3)) + (an (setv [y z] [7 8])) + (assert (= y 7)) + (assert (= z 8)) + (an (setv #(y z) [9 10])) + (assert (= y 9)) + (assert (= z 10)) + + (setv p 11) + (setv p (setv q 12)) + (assert (= q 12)) + (an p) + + (an (setv x (defn phooey [] (setv p 1) (+ p 6)))) + (an (setv x (defclass C))) + (an (setv x (for [i (range 3)] i (+ i 1)))) + (an (setv x (assert True))) + + (an (setv x (with [(open "README.md" "r")] 3))) + (assert (= x 3)) + (an (setv x (try (/ 1 2) (except [ZeroDivisionError] "E1")))) + (assert (= x .5)) + (an (setv x (try (/ 1 0) (except [ZeroDivisionError] "E2")))) + (assert (= x "E2")) + + ; https://github.com/hylang/hy/issues/1052 + (an (setv (get {} "x") 42)) + (setv l []) + (defclass Foo [object] + (defn __setattr__ [self attr val] + (.append l [attr val]))) + (setv x (Foo)) + (an (setv x.eggs "ham")) + (assert (not (hasattr x "eggs"))) + (assert (= l [["eggs" "ham"]]))) diff --git a/tests/native_tests/strings.hy b/tests/native_tests/strings.hy new file mode 100644 index 000000000..02dd0a3ec --- /dev/null +++ b/tests/native_tests/strings.hy @@ -0,0 +1,166 @@ +;; String literals (including bracket strings and docstrings), +;; plus f-strings + +(import + re + pytest) + + +(defn test-encoding-nightmares [] + (assert (= (len "ℵℵℵ♥♥♥\t♥♥\r\n") 11))) + + +(defn test-quote-bracket-string-delim [] + (assert (= (. '#[my delim[hello world]my delim] brackets) "my delim")) + (assert (= (. '#[[squid]] brackets) "")) + (assert (is (. '"squid" brackets) None))) + + +(defn test-docstrings [] + (defn f [] "docstring" 5) + (assert (= (. f __doc__) "docstring")) + + ; a single string is the return value, not a docstring + ; (https://github.com/hylang/hy/issues/1402) + (defn f3 [] "not a docstring") + (assert (is (. f3 __doc__) None)) + (assert (= (f3) "not a docstring"))) + + +(defn test-module-docstring [] + (import tests.resources.module-docstring-example :as m) + (assert (= m.__doc__ "This is the module docstring.")) + (assert (= m.foo 5))) + + +(defn test-format-strings [] + (assert (= f"hello world" "hello world")) + (assert (= f"hello {(+ 1 1)} world" "hello 2 world")) + (assert (= f"a{ (.upper (+ "g" "k")) }z" "aGKz")) + (assert (= f"a{1}{2}b" "a12b")) + + ; Referring to a variable + (setv p "xyzzy") + (assert (= f"h{p}j" "hxyzzyj")) + + ; Including a statement and setting a variable + (assert (= f"a{(do (setv floop 4) (* floop 2))}z" "a8z")) + (assert (= floop 4)) + + ; Comments + (assert (= f"a{(+ 1 + 2 ; This is a comment. + 3)}z" "a6z")) + + ; Newlines in replacement fields + (assert (= f"ey {"bee +cee"} dee" "ey bee\ncee dee")) + + ; Conversion characters and format specifiers + (setv p:9 "other") + (setv !r "bar") + (assert (= f"a{p !r}" "a'xyzzy'")) + (assert (= f"a{p :9}" "axyzzy ")) + (assert (= f"a{p:9}" "aother")) + (assert (= f"a{p !r :9}" "a'xyzzy' ")) + (assert (= f"a{p !r:9}" "a'xyzzy' ")) + (assert (= f"a{p:9 :9}" "aother ")) + (assert (= f"a{!r}" "abar")) + (assert (= f"a{!r !r}" "a'bar'")) + + ; Fun with `r` + (assert (= f"hello {r"\n"}" r"hello \n")) + (assert (= f"hello {"\n"}" "hello \n")) + + ; Braces escaped via doubling + (assert (= f"ab{{cde" "ab{cde")) + (assert (= f"ab{{cde}}}}fg{{{{{{" "ab{cde}}fg{{{")) + (assert (= f"ab{{{(+ 1 1)}}}" "ab{2}")) + + ; Nested replacement fields + (assert (= f"{2 :{(+ 2 2)}}" " 2")) + (setv value 12.34 width 10 precision 4) + (assert (= f"result: {value :{width}.{precision}}" "result: 12.34")) + + ; Nested replacement fields with ! and : + (defclass C [object] + (defn __format__ [self format-spec] + (+ "C[" format-spec "]"))) + (assert (= f"{(C) : {(str (+ 1 1)) !r :x<5}}" "C[ '2'xx]")) + + ; \N sequences + ; https://github.com/hylang/hy/issues/2321 + (setv ampersand "wich") + (assert (= f"sand{ampersand} \N{ampersand} chips" "sandwich & chips")) + + ; Format bracket strings + (assert (= #[f[a{p !r :9}]f] "a'xyzzy' ")) + (assert (= #[f-string[result: {value :{width}.{precision}}]f-string] + "result: 12.34")) + + ; Quoting shouldn't evaluate the f-string immediately + ; https://github.com/hylang/hy/issues/1844 + (setv quoted 'f"hello {world}") + (assert (isinstance quoted hy.models.FString)) + (with [(pytest.raises NameError)] + (hy.eval quoted)) + (setv world "goodbye") + (assert (= (hy.eval quoted) "hello goodbye")) + + ;; '=' debugging syntax. + (setv foo "bar") + (assert (= f"{foo =}" "foo ='bar'")) + + ;; Whitespace is preserved. + (assert (= f"xyz{ foo = }" "xyz foo = 'bar'")) + + ;; Explicit conversion is applied. + (assert (= f"{ foo = !s}" " foo = bar")) + + ;; Format spec supercedes implicit conversion. + (setv pi 3.141593 fill "_") + (assert (= f"{pi = :{fill}^8.2f}" "pi = __3.14__")) + + ;; Format spec doesn't clobber the explicit conversion. + (with [(pytest.raises + ValueError + :match r"Unknown format code '?f'? for object of type 'str'")] + f"{pi =!s:.3f}") + + ;; Nested "=" is parsed, but fails at runtime, like Python. + (setv width 7) + (with [(pytest.raises + ValueError + :match r"I|invalid format spec(?:ifier)?")] + f"{pi =:{fill =}^{width =}.2f}")) + + +(defn test-format-string-repr-roundtrip [] + (for [orig [ + 'f"hello {(+ 1 1)} world" + 'f"a{p !r:9}" + 'f"{ foo = !s}"]] + (setv new (eval (repr orig))) + (assert (= (len new) (len orig))) + (for [[n o] (zip new orig)] + (when (hasattr o "conversion") + (assert (= n.conversion o.conversion))) + (assert (= n o))))) + + +(defn test-repr-with-brackets [] + (assert (= (repr '"foo") "hy.models.String('foo')")) + (assert (= (repr '#[[foo]]) "hy.models.String('foo', brackets='')")) + (assert (= (repr '#[xx[foo]xx]) "hy.models.String('foo', brackets='xx')")) + (assert (= (repr '#[xx[]xx]) "hy.models.String('', brackets='xx')")) + + (for [g [repr str]] + (defn f [x] (re.sub r"\n\s+" "" (g x) :count 1)) + (assert (= (f 'f"foo") + "hy.models.FString([hy.models.String('foo')])")) + (assert (= (f '#[f[foo]f]) + "hy.models.FString([hy.models.String('foo')], brackets='f')")) + (assert (= (f '#[f-x[foo]f-x]) + "hy.models.FString([hy.models.String('foo')], brackets='f-x')")) + (assert (= (f '#[f-x[]f-x]) + "hy.models.FString(brackets='f-x')")))) diff --git a/tests/native_tests/try.hy b/tests/native_tests/try.hy new file mode 100644 index 000000000..623cc9813 --- /dev/null +++ b/tests/native_tests/try.hy @@ -0,0 +1,219 @@ +;; Tests of `try` and `raise` + +(defn test-try [] + + (try (do) (except [])) + + (try (do) (except [IOError]) (except [])) + + ; test that multiple statements in a try get evaluated + (setv value 0) + (try (+= value 1) (+= value 2) (except [IOError]) (except [])) + (assert (= value 3)) + + ; test that multiple expressions in a try get evaluated + ; https://github.com/hylang/hy/issues/1584 + (setv l []) + (defn f [] (.append l 1)) + (try (f) (f) (f) (except [IOError])) + (assert (= l [1 1 1])) + (setv l []) + (try (f) (f) (f) (except [IOError]) (else (f))) + (assert (= l [1 1 1 1])) + + ;; Test correct (raise) + (setv passed False) + (try + (try + (do) + (raise IndexError) + (except [IndexError] (raise))) + (except [IndexError] + (setv passed True))) + (assert passed) + + ;; Test incorrect (raise) + (setv passed False) + (try + (raise) + (except [RuntimeError] + (setv passed True))) + (assert passed) + + ;; Test (finally) + (setv passed False) + (try + (do) + (finally (setv passed True))) + (assert passed) + + ;; Test (finally) + (raise) + (setv passed False) + (try + (raise Exception) + (except []) + (finally (setv passed True))) + (assert passed) + + + ;; Test (finally) + (raise) + (else) + (setv passed False + not-elsed True) + (try + (raise Exception) + (except []) + (else (setv not-elsed False)) + (finally (setv passed True))) + (assert passed) + (assert not-elsed) + + (try + (raise (KeyError)) + (except [[IOError]] (assert False)) + (except [e [KeyError]] (assert e))) + + (try + (raise (KeyError)) + (except [[IOError]] (assert False)) + (except [e [KeyError]] (assert e))) + + (try + (get [1] 3) + (except [IndexError] (assert True)) + (except [IndexError] (do))) + + (try + (print foobar42ofthebaz) + (except [IndexError] (assert False)) + (except [NameError] (do))) + + (try + (get [1] 3) + (except [e IndexError] (assert (isinstance e IndexError)))) + + (try + (get [1] 3) + (except [e [IndexError NameError]] (assert (isinstance e IndexError)))) + + (try + (print foobar42ofthebaz) + (except [e [IndexError NameError]] (assert (isinstance e NameError)))) + + (try + (print foobar42) + (except [[IndexError NameError]] (do))) + + (try + (get [1] 3) + (except [[IndexError NameError]] (do))) + + (try + (print foobar42ofthebaz) + (except [])) + + (try + (print foobar42ofthebaz) + (except [] (do))) + + (try + (print foobar42ofthebaz) + (except [] + (setv foobar42ofthebaz 42) + (assert (= foobar42ofthebaz 42)))) + + (setv passed False) + (try + (try (do) (except []) (else (bla))) + (except [NameError] (setv passed True))) + (assert passed) + + (setv x 0) + (try + (raise IOError) + (except [IOError] + (setv x 45)) + (else (setv x 44))) + (assert (= x 45)) + + (setv x 0) + (try + (raise KeyError) + (except [] + (setv x 45)) + (else (setv x 44))) + (assert (= x 45)) + + (setv x 0) + (try + (try + (raise KeyError) + (except [IOError] + (setv x 45)) + (else (setv x 44))) + (except [])) + (assert (= x 0)) + + ; test that [except ...] and ("except" ...) aren't treated like (except ...), + ; and that the code there is evaluated normally + (setv x 0) + (try + (+= x 1) + ("except" [IOError] (+= x 1)) + (except [])) + + (assert (= x 2)) + + (setv x 0) + (try + (+= x 1) + [except [IOError] (+= x 1)] + (except [])) + + (assert (= x 2))) + + +(defn test-try-except-return [] + "Ensure we can return from an `except` form." + (assert (= ((fn [] (try xxx (except [NameError] (+ 1 1))))) 2)) + (setv foo (try xxx (except [NameError] (+ 1 1)))) + (assert (= foo 2)) + (setv foo (try (+ 2 2) (except [NameError] (+ 1 1)))) + (assert (= foo 4))) + + +(defn test-try-else-return [] + "Ensure we can return from the `else` clause of a `try`." + ; https://github.com/hylang/hy/issues/798 + + (assert (= "ef" ((fn [] + (try (+ "a" "b") + (except [NameError] (+ "c" "d")) + (else (+ "e" "f"))))))) + + (setv foo + (try (+ "A" "B") + (except [NameError] (+ "C" "D")) + (else (+ "E" "F")))) + (assert (= foo "EF")) + + ; Check that the lvalue isn't assigned in the main `try` body + ; there's an `else`. + (setv x 1) + (setv y 0) + (setv x + (try (+ "G" "H") + (except [NameError] (+ "I" "J")) + (else + (setv y 1) + (assert (= x 1)) + (+ "K" "L")))) + (assert (= x "KL")) + (assert (= y 1))) + + +(defn test-exception-cause [] + (assert (is NameError (type (. + (try + (raise ValueError :from NameError) + (except [e [ValueError]] e)) + __cause__))))) diff --git a/tests/native_tests/unpack.hy b/tests/native_tests/unpack.hy new file mode 100644 index 000000000..f8789ef79 --- /dev/null +++ b/tests/native_tests/unpack.hy @@ -0,0 +1,41 @@ +(defn test-star-unpacking [] + ; Python 3-only forms of unpacking are in py3_only_tests.hy + (setv l [1 2 3]) + (setv d {"a" "x" "b" "y"}) + (defn fun [[x1 None] [x2 None] [x3 None] [x4 None] [a None] [b None] [c None]] + [x1 x2 x3 x4 a b c]) + (assert (= (fun 5 #* l) [5 1 2 3 None None None])) + (assert (= (+ #* l) 6)) + (assert (= (fun 5 #** d) [5 None None None "x" "y" None])) + (assert (= (fun 5 #* l #** d) [5 1 2 3 "x" "y" None]))) + + +(defn test-extended-unpacking-1star-lvalues [] + (setv [x #*y] [1 2 3 4]) + (assert (= x 1)) + (assert (= y [2 3 4])) + (setv [a #*b c] "ghijklmno") + (assert (= a "g")) + (assert (= b (list "hijklmn"))) + (assert (= c "o"))) + + +(defn test-unpacking-pep448-1star [] + (setv l [1 2 3]) + (setv p [4 5]) + (assert (= ["a" #*l "b" #*p #*l] ["a" 1 2 3 "b" 4 5 1 2 3])) + (assert (= #("a" #*l "b" #*p #*l) #("a" 1 2 3 "b" 4 5 1 2 3))) + (assert (= #{"a" #*l "b" #*p #*l} #{"a" "b" 1 2 3 4 5})) + (defn f [#* args] args) + (assert (= (f "a" #*l "b" #*p #*l) #("a" 1 2 3 "b" 4 5 1 2 3))) + (assert (= (+ #*l #*p) 15)) + (assert (= (and #*l) 3))) + + +(defn test-unpacking-pep448-2star [] + (setv d1 {"a" 1 "b" 2}) + (setv d2 {"c" 3 "d" 4}) + (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"})) + (defn fun [[a None] [b None] [c None] [d None] [e None] [f None]] + [a b c d e f]) + (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None]))) diff --git a/tests/native_tests/when.hy b/tests/native_tests/when.hy deleted file mode 100644 index 3fb5b51b7..000000000 --- a/tests/native_tests/when.hy +++ /dev/null @@ -1,10 +0,0 @@ -(defn test-when [] - (assert (= (when True 1) 1)) - (assert (= (when True 1 2) 2)) - (assert (= (when True 1 3) 3)) - (assert (= (when False 2) None)) - (assert (= (when (= 1 2) 42) None)) - (assert (= (when (= 2 2) 42) 42)) - - (assert (is (when (do (setv x 3) True)) None)) - (assert (= x 3))) diff --git a/tests/native_tests/with.hy b/tests/native_tests/with.hy index 7d3cfb994..f9f9556b7 100644 --- a/tests/native_tests/with.hy +++ b/tests/native_tests/with.hy @@ -1,4 +1,16 @@ -(import pytest) +(import + asyncio + pytest + tests.resources [AsyncWithTest]) + +(defn test-context [] + (with [fd (open "README.md" "r")] (assert fd)) + (with [(open "README.md" "r")] (do))) + +(defn test-with-return [] + (defn read-file [filename] + (with [fd (open filename "r")] (.read fd))) + (assert (!= 0 (len (read-file "README.md"))))) (defclass WithTest [object] (defn __init__ [self val] @@ -38,6 +50,41 @@ (assert (= t2 2)) (assert (= t3 3)))) +(defn test-single-with/a [] + (asyncio.run + ((fn/a [] + (with/a [t (AsyncWithTest 1)] + (assert (= t 1))))))) + +(defn test-two-with/a [] + (asyncio.run + ((fn/a [] + (with/a [t1 (AsyncWithTest 1) + t2 (AsyncWithTest 2)] + (assert (= t1 1)) + (assert (= t2 2))))))) + +(defn test-thrice-with/a [] + (asyncio.run + ((fn/a [] + (with/a [t1 (AsyncWithTest 1) + t2 (AsyncWithTest 2) + t3 (AsyncWithTest 3)] + (assert (= t1 1)) + (assert (= t2 2)) + (assert (= t3 3))))))) + +(defn test-quince-with/a [] + (asyncio.run + ((fn/a [] + (with/a [t1 (AsyncWithTest 1) + t2 (AsyncWithTest 2) + t3 (AsyncWithTest 3) + _ (AsyncWithTest 4)] + (assert (= t1 1)) + (assert (= t2 2)) + (assert (= t3 3))))))) + (defn test-unnamed-context-with [] "`_` get compiled to unnamed context" (with [_ (WithTest 1) From 56fa59bfba2ffb9d23a870cadeb4f89be2bd2adc Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 8 Jan 2023 16:49:03 -0500 Subject: [PATCH 112/342] Rename some tests --- tests/native_tests/dots.hy | 4 ++-- tests/native_tests/try.hy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/native_tests/dots.hy b/tests/native_tests/dots.hy index e3fa2a29a..ef3ae3c66 100644 --- a/tests/native_tests/dots.hy +++ b/tests/native_tests/dots.hy @@ -2,7 +2,7 @@ os) -(defn dotted [] +(defn test-dotted-identifiers [] (assert (= (.join " " ["one" "two"]) "one two")) (defclass X [object] []) @@ -35,7 +35,7 @@ (assert (= (.__str__ :foo) ":foo"))) -(defn test-attribute-access [] +(defn test-dot-macro [] (defclass mycls [object]) (setv foo [(mycls) (mycls) (mycls)]) diff --git a/tests/native_tests/try.hy b/tests/native_tests/try.hy index 623cc9813..e7e3f223e 100644 --- a/tests/native_tests/try.hy +++ b/tests/native_tests/try.hy @@ -211,7 +211,7 @@ (assert (= y 1))) -(defn test-exception-cause [] +(defn test-raise-from [] (assert (is NameError (type (. (try (raise ValueError :from NameError) From 2c92e4872b62a49e3f728274cc4dc149895abb5d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 8 Jan 2023 17:03:43 -0500 Subject: [PATCH 113/342] Remove an obsolete comment --- tests/native_tests/unpack.hy | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/native_tests/unpack.hy b/tests/native_tests/unpack.hy index f8789ef79..e5e484c75 100644 --- a/tests/native_tests/unpack.hy +++ b/tests/native_tests/unpack.hy @@ -1,5 +1,4 @@ (defn test-star-unpacking [] - ; Python 3-only forms of unpacking are in py3_only_tests.hy (setv l [1 2 3]) (setv d {"a" "x" "b" "y"}) (defn fun [[x1 None] [x2 None] [x3 None] [x4 None] [a None] [b None] [c None]] From 8fe9aadb66abcf5865914bb0c424612d17df6403 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 8 Jan 2023 17:14:14 -0500 Subject: [PATCH 114/342] Remove most of `test-and` and `test-or` This stuff is redundant with the operator tests for `and` and `or`. --- tests/native_tests/logic_short_circuit.hy | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/native_tests/logic_short_circuit.hy b/tests/native_tests/logic_short_circuit.hy index 2ac651295..44dbe3085 100644 --- a/tests/native_tests/logic_short_circuit.hy +++ b/tests/native_tests/logic_short_circuit.hy @@ -2,15 +2,6 @@ (defn test-and [] - (setv and123 (and 1 2 3) - and-false (and 1 False 3) - and-true (and) - and-single (and 1)) - (assert (= and123 3)) - (assert (= and-false False)) - (assert (= and-true True)) - (assert (= and-single 1)) - ; short circuiting (setv a 1) (and 0 (setv a 2)) (assert (= a 1))) @@ -42,17 +33,6 @@ (defn test-or [] - (setv or-all-true (or 1 2 3 True "string") - or-some-true (or False "hello") - or-none-true (or False False) - or-false (or) - or-single (or 1)) - (assert (= or-all-true 1)) - (assert (= or-some-true "hello")) - (assert (= or-none-true False)) - (assert (= or-false None)) - (assert (= or-single 1)) - ; short circuiting (setv a 1) (or 1 (setv a 2)) (assert (= a 1))) From e726b67cc9f96341c5a0927e0b97596e10bf5da3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 9 Jan 2023 16:50:59 -0500 Subject: [PATCH 115/342] Expand testing of `do` --- tests/native_tests/do.hy | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/native_tests/do.hy b/tests/native_tests/do.hy index 957d87095..05cd0d4e5 100644 --- a/tests/native_tests/do.hy +++ b/tests/native_tests/do.hy @@ -1,7 +1,12 @@ -(defn test-do [] - (do)) +(defn test-empty [] + (assert (is (do) None)) + (assert (is (if True (do) (do)) None))) -(defn test-pass [] - (if True (do) (do)) - (assert (= 1 1))) +(defn test-nonempty [] + (assert (= (do 1 2 3) 3)) + (assert (= (do 3 2 1) 1)) + + (setv x "a") + (assert (= (do (setv x "b") "c") "c")) + (assert (= x "b"))) From 33bfefeba6a79be65299e27955cafa4e7be77703 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 12 Jan 2023 17:28:35 -0500 Subject: [PATCH 116/342] Autoformat --- tests/test_reader.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_reader.py b/tests/test_reader.py index 29f98649a..0465bc00c 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -322,11 +322,16 @@ def test_lists(): def test_dicts(): assert tokenize("{1 2 3 4}") == [Dict(map(Integer, (1, 2, 3, 4)))] - assert tokenize("{1 (+ 1 1) 3 (+ 2 2)}") == [Dict(( - Integer(1), - Expression((Symbol("+"), Integer(1), Integer(1))), - Integer(3), - Expression((Symbol("+"), Integer(2), Integer(2)))))] + assert tokenize("{1 (+ 1 1) 3 (+ 2 2)}") == [ + Dict( + ( + Integer(1), + Expression((Symbol("+"), Integer(1), Integer(1))), + Integer(3), + Expression((Symbol("+"), Integer(2), Integer(2))), + ) + ) + ] def test_lex_column_counting(): From c37fb38dd6168661dc188cbf82604aa295c5d6ca Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 14 Jan 2023 09:11:00 -0500 Subject: [PATCH 117/342] Add `do-mac` --- NEWS.rst | 4 ++-- docs/api.rst | 11 +++++++++ hy/core/result_macros.py | 11 +++++---- tests/native_tests/eval_foo_compile.hy | 32 +++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 9302226b5..5b63bcc97 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -18,8 +18,8 @@ Breaking Changes expressions (like `(. foo bar)` and `(. None sqrt)`) instead of symbols. Some odd cases like `foo.` and `foo..bar` are now syntactically illegal. -* `pragma` is now reserved as a core macro name, although it doesn't - do anything useful (yet). +* New macro `do-mac`. +* New macro `pragma` (although it doesn't do anything useful yet). * `hy.cmdline.HyREPL` is now `hy.REPL`. Bug Fixes diff --git a/docs/api.rst b/docs/api.rst index e05545f05..f0173fac5 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -312,6 +312,17 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (+ 1 (do (setv x (+ 1 1)) x)) ; => 3 +.. hy:function:: (do-mac [#* body]) + + ``do-mac`` evaluates its arguments (in order) at compile time, and leaves behind the value of the last argument (``None`` if no arguments were provided) as code to be run. The effect is similar to defining and then immediately calling a nullary macro, hence the name, which stands for "do macro". :: + + (do-mac `(setv ~(hy.models.Symbol (* "x" 5)) "foo")) + ; Expands to: (setv xxxxx "foo") + (print xxxxx) + ; => "foo" + + Contrast with :hy:func:`eval-and-compile`, which evaluates the same code at compile-time and run-time, instead of using the result of the compile-time run as code for run-time. ``do-mac`` is also similar to Common Lisp's SHARPSIGN DOT syntax (``#.``), from which it differs by evaluating at compile-time rather than read-time. + .. hy:function:: (for [#* args]) ``for`` compiles to one or more :py:keyword:`for` statements, which diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 31c0a758d..9d64624c6 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -54,6 +54,7 @@ String, Symbol, Tuple, + as_model, is_unpack, ) from hy.reader import mangle, unmangle @@ -88,12 +89,12 @@ def compile_do(self, expr, root, body): return self._compile_branch(body) -@pattern_macro(["eval-and-compile", "eval-when-compile"], [many(FORM)]) -def compile_eval_and_compile(compiler, expr, root, body): +@pattern_macro(["eval-and-compile", "eval-when-compile", "do-mac"], [many(FORM)]) +def compile_eval_foo_compile(compiler, expr, root, body): new_expr = Expression([Symbol("do").replace(expr[0])]).replace(expr) try: - hy_eval( + value = hy_eval( new_expr + body, compiler.module.__dict__, compiler.module, @@ -115,7 +116,9 @@ def compile_eval_and_compile(compiler, expr, root, body): raise HyEvalError(str(e), compiler.filename, body, compiler.source) return ( - compiler._compile_branch(body) + compiler.compile(as_model(value)) + if mangle(root) == "do_mac" + else compiler._compile_branch(body) if mangle(root) == "eval_and_compile" else Result() ) diff --git a/tests/native_tests/eval_foo_compile.hy b/tests/native_tests/eval_foo_compile.hy index bb14cab6e..cdd0e293f 100644 --- a/tests/native_tests/eval_foo_compile.hy +++ b/tests/native_tests/eval_foo_compile.hy @@ -1,4 +1,4 @@ -;; Tests of `eval-when-compile` and `eval-and-compile` +;; Tests of `eval-when-compile`, `eval-and-compile`, and `do-mac` (defn test-eval-foo-compile-return-values [] @@ -15,3 +15,33 @@ (setv derrick (eval-when-compile (+= jim 1) 2)) (assert (= jim 1)) (assert (is derrick None))) + + +(defn test-do-mac [] + + (assert (is (do-mac) None)) + + (setv x 2) + (setv x-compile-time (do-mac + (setv x 3) + x)) + (assert (= x 2)) + (assert (= x-compile-time 3)) + + (eval-when-compile (setv x 4)) + (assert (= x 2)) + (assert (= (do-mac x) 4)) + + (defmacro m [] + (global x) + (setv x 5)) + (m) + (assert (= x 2)) + (assert (= (do-mac x) 5)) + + (setv l []) + (do-mac `(do ~@(* ['(.append l 1)] 5))) + (assert (= l [1 1 1 1 1])) + + (do-mac `(setv ~(hy.models.Symbol (* "x" 5)) "foo")) + (assert (= xxxxx "foo"))) From 4ae89d3d59790404e688b47cbd3825c417f0e129 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 12 Jan 2023 15:12:24 -0500 Subject: [PATCH 118/342] =?UTF-8?q?Move=20around=20Python-version=E2=80=93?= =?UTF-8?q?specific=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{py3_10_only_tests.hy => match.hy} | 0 tests/native_tests/py3_11_only_tests.hy | 23 ----------------- .../{py3_8_only_tests.hy => setx.hy} | 5 ++++ tests/native_tests/sub_py3_7_only.hy | 9 ------- tests/native_tests/try.hy | 25 +++++++++++++++++++ 5 files changed, 30 insertions(+), 32 deletions(-) rename tests/native_tests/{py3_10_only_tests.hy => match.hy} (100%) delete mode 100644 tests/native_tests/py3_11_only_tests.hy rename tests/native_tests/{py3_8_only_tests.hy => setx.hy} (89%) delete mode 100644 tests/native_tests/sub_py3_7_only.hy diff --git a/tests/native_tests/py3_10_only_tests.hy b/tests/native_tests/match.hy similarity index 100% rename from tests/native_tests/py3_10_only_tests.hy rename to tests/native_tests/match.hy diff --git a/tests/native_tests/py3_11_only_tests.hy b/tests/native_tests/py3_11_only_tests.hy deleted file mode 100644 index f847b5574..000000000 --- a/tests/native_tests/py3_11_only_tests.hy +++ /dev/null @@ -1,23 +0,0 @@ -(defn test-except* [] - (setv got "") - - (setv return-value (try - (raise (ExceptionGroup "meep" [(KeyError) (ValueError)])) - (except* [KeyError] - (+= got "k") - "r1") - (except* [IndexError] - (+= got "i") - "r2") - (except* [ValueError] - (+= got "v") - "r3") - (else - (+= got "e") - "r4") - (finally - (+= got "f") - "r5"))) - - (assert (= got "kvf")) - (assert (= return-value "r3"))) diff --git a/tests/native_tests/py3_8_only_tests.hy b/tests/native_tests/setx.hy similarity index 89% rename from tests/native_tests/py3_8_only_tests.hy rename to tests/native_tests/setx.hy index 9aff621c5..941b24866 100644 --- a/tests/native_tests/py3_8_only_tests.hy +++ b/tests/native_tests/setx.hy @@ -3,6 +3,11 @@ (import pytest) +(defn test-cant-setx [] + (with [e (pytest.raises hy.errors.HySyntaxError)] + (hy.eval '(setx x 1))) + (assert (= "setx requires Python 3.8 or later"))) + (defn test-setx [] (setx y (+ (setx x (+ "a" "b")) "c")) (assert (= x "ab")) diff --git a/tests/native_tests/sub_py3_7_only.hy b/tests/native_tests/sub_py3_7_only.hy deleted file mode 100644 index 4ea19b839..000000000 --- a/tests/native_tests/sub_py3_7_only.hy +++ /dev/null @@ -1,9 +0,0 @@ -;; Tests where the emitted code relies on Python ≤3.7. -;; conftest.py skips this file when running on Python ≥3.8. - -(import pytest) - -(defn test-setx [] - (with [e (pytest.raises hy.errors.HySyntaxError)] - (hy.eval '(setx x 1))) - (assert (= "setx requires Python 3.8 or later"))) diff --git a/tests/native_tests/try.hy b/tests/native_tests/try.hy index e7e3f223e..8f2b66128 100644 --- a/tests/native_tests/try.hy +++ b/tests/native_tests/try.hy @@ -211,6 +211,31 @@ (assert (= y 1))) +(defn test-except* [] + (setv got "") + + (setv return-value (try + (raise (ExceptionGroup "meep" [(KeyError) (ValueError)])) + (except* [KeyError] + (+= got "k") + "r1") + (except* [IndexError] + (+= got "i") + "r2") + (except* [ValueError] + (+= got "v") + "r3") + (else + (+= got "e") + "r4") + (finally + (+= got "f") + "r5"))) + + (assert (= got "kvf")) + (assert (= return-value "r3"))) + + (defn test-raise-from [] (assert (is NameError (type (. (try From 1937c99388fbdb59bec53d563e2d4ef2460c6e25 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 12 Jan 2023 15:14:33 -0500 Subject: [PATCH 119/342] Remove obsolete `pytest_ignore_collect` --- conftest.py | 23 ----------------------- tests/native_tests/setx.hy | 3 --- 2 files changed, 26 deletions(-) diff --git a/conftest.py b/conftest.py index 442c3993e..a88904842 100644 --- a/conftest.py +++ b/conftest.py @@ -1,38 +1,15 @@ import importlib import os -import sys -from functools import reduce -from operator import or_ from pathlib import Path import pytest -import hy -from hy._compat import PY3_8, PY3_10, PY3_11 - NATIVE_TESTS = os.path.join("", "tests", "native_tests", "") # https://github.com/hylang/hy/issues/2029 os.environ.pop("HYSTARTUP", None) -def pytest_ignore_collect(path, config): - versions = [ - (sys.version_info < (3, 8), "sub_py3_7_only"), - (PY3_8, "py3_8_only"), - (PY3_10, "py3_10_only"), - (PY3_11, "py3_11_only"), - ] - - return ( - reduce( - or_, - (name in path.basename and not condition for condition, name in versions), - ) - or None - ) - - def pytest_collect_file(parent, path): if ( path.ext == ".hy" diff --git a/tests/native_tests/setx.hy b/tests/native_tests/setx.hy index 941b24866..5fbf78e93 100644 --- a/tests/native_tests/setx.hy +++ b/tests/native_tests/setx.hy @@ -1,6 +1,3 @@ -;; Tests where the emitted code relies on Python ≥3.8. -;; conftest.py skips this file when running on Python <3.8. - (import pytest) (defn test-cant-setx [] From 1c3613102e0218e2f72d4ed88f4fc8c7d51a2562 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 12 Jan 2023 15:16:24 -0500 Subject: [PATCH 120/342] Add Python version guards to the moved tests --- tests/native_tests/match.hy | 6 ++++++ tests/native_tests/setx.hy | 16 +++++++++------- tests/native_tests/try.hy | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/native_tests/match.hy b/tests/native_tests/match.hy index d37d70d7f..1a5408147 100644 --- a/tests/native_tests/match.hy +++ b/tests/native_tests/match.hy @@ -1,3 +1,6 @@ +(do-mac (when hy._compat.PY3_10 '(do + + (import pytest dataclasses [dataclass] hy.errors [HySyntaxError]) @@ -260,3 +263,6 @@ (match y {"b" b #**a} a))) (assert (= b 2)))) + + +))) diff --git a/tests/native_tests/setx.hy b/tests/native_tests/setx.hy index 5fbf78e93..80a4ebff9 100644 --- a/tests/native_tests/setx.hy +++ b/tests/native_tests/setx.hy @@ -1,11 +1,13 @@ (import pytest) -(defn test-cant-setx [] +(defn + [(pytest.mark.skipif hy._compat.PY3_8 :reason "Python ≥ 3.8")] + test-cant-setx [] (with [e (pytest.raises hy.errors.HySyntaxError)] (hy.eval '(setx x 1))) (assert (= "setx requires Python 3.8 or later"))) -(defn test-setx [] +(do-mac (when hy._compat.PY3_8 '(defn test-setx [] (setx y (+ (setx x (+ "a" "b")) "c")) (assert (= x "ab")) (assert (= y "abc")) @@ -24,9 +26,9 @@ (assert (= filtered ["apple" "banana"])) (assert (= v "banana")) (with [(pytest.raises NameError)] - i)) + i)))) -(defn test-setx-generator-scope [] +(do-mac (when hy._compat.PY3_8 '(defn test-setx-generator-scope [] ;; https://github.com/hylang/hy/issues/1994 (setv x 20) (lfor n (range 10) (setx x n)) @@ -49,11 +51,11 @@ (lfor n (range 0) :do x (setx z n)) (with [(pytest.raises UnboundLocalError)] - z)) + z)))) -(defn test-let-setx [] +(do-mac (when hy._compat.PY3_8 '(defn test-let-setx [] (let [x 40 y 13] (setv y (setx x 2)) (assert (= x 2)) - (assert (= y 2)))) + (assert (= y 2)))))) diff --git a/tests/native_tests/try.hy b/tests/native_tests/try.hy index 8f2b66128..bf226469c 100644 --- a/tests/native_tests/try.hy +++ b/tests/native_tests/try.hy @@ -211,7 +211,7 @@ (assert (= y 1))) -(defn test-except* [] +(do-mac (when hy._compat.PY3_11 '(defn test-except* [] (setv got "") (setv return-value (try @@ -233,7 +233,7 @@ "r5"))) (assert (= got "kvf")) - (assert (= return-value "r3"))) + (assert (= return-value "r3"))))) (defn test-raise-from [] From 88df521324c384a0629ccf2235916e3687f3d143 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 16 Jan 2023 10:51:34 -0500 Subject: [PATCH 121/342] Mark macros in `api.rst` as macros --- docs/api.rst | 98 ++++++++++++++++++++++++++-------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index f0173fac5..fdb492499 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,7 +8,7 @@ Core Macros The following macros are automatically imported into all Hy modules as their base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. -.. hy:function:: (annotate [value type]) +.. hy:macro:: (annotate [value type]) ``annotate`` and its shorthand form ``#^`` are used to denote annotations, including type hints, in three different contexts: @@ -84,7 +84,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. meaning the same thing in Hy as in Python. Also, :hy:func:`get ` is another way to subscript in Hy. -.. hy:function:: (fn [args]) +.. hy:macro:: (fn [args]) As :hy:func:`defn`, but no name for the new function is required (or allowed), and the newly created function object is returned. Decorators @@ -92,12 +92,12 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. to that of :hy:func:`defn`, without any of the restrictions of Python's :py:keyword:`lambda`. See :hy:func:`fn/a` for the asynchronous equivalent. -.. hy:function:: (fn/a [name #* args]) +.. hy:macro:: (fn/a [name #* args]) As :hy:func:`fn`, but the created function object will be a :ref:`coroutine `. -.. hy:function:: (defn [name #* args]) +.. hy:macro:: (defn [name #* args]) ``defn`` compiles to a :ref:`function definition ` (or possibly to an assignment of a :ref:`lambda expression `). It always @@ -163,12 +163,12 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print (hy.repr (f 1 2 :d 4 :e 5 :f 6))) ; => [1 2 3 4 5 {"f" 6}] -.. hy:function:: (defn/a [name lambda-list #* body]) +.. hy:macro:: (defn/a [name lambda-list #* body]) As :hy:func:`defn`, but defines a :ref:`coroutine ` like Python's ``async def``. -.. hy:function:: (defmacro [name lambda-list #* body]) +.. hy:macro:: (defmacro [name lambda-list #* body]) ``defmacro`` is used to define macros. The general format is ``(defmacro name [parameters] expr)``. @@ -205,7 +205,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. => (a-macro :b 3) [:b 3] -.. hy:function:: (if [test then else]) +.. hy:macro:: (if [test then else]) ``if`` compiles to an :py:keyword:`if` expression (or compound ``if`` statement). The form ``test`` is evaluated and categorized as true or false according to :py:class:`bool`. If the result is true, ``then`` is evaluated and returned. Othewise, ``else`` is evaluated and returned. :: @@ -220,7 +220,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. - :hy:func:`when `, for shorthand for ``(if condition (do …) None)``. - :hy:func:`cond `, for shorthand for nested ``if`` forms. -.. hy:function:: (await [obj]) +.. hy:macro:: (await [obj]) ``await`` creates an :ref:`await expression `. It takes exactly one argument: the object to wait for. :: @@ -232,7 +232,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print "world")) (asyncio.run (main)) -.. hy:function:: break +.. hy:macro:: break ``break`` compiles to a :py:keyword:`break` statement, which terminates the enclosing loop. The following example has an infinite ``while`` loop that @@ -251,7 +251,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. comprehension forms, you may also enclose it in a function and use :hy:func:`return`. -.. hy:function:: (chainc [#* args]) +.. hy:macro:: (chainc [#* args]) ``chainc`` creates a :ref:`comparison expression `. It isn't required for unchained comparisons, which have only one comparison operator, @@ -281,7 +281,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Python. -.. hy:function:: continue +.. hy:macro:: continue ``continue`` compiles to a :py:keyword:`continue` statement, which returns execution to the start of a loop. In the following example, ``(.append @@ -304,7 +304,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. in ``(for [x xs] (block (for [y ys] …)))``. You can then use ``block-ret`` in place of ``continue``. -.. hy:function:: (do [#* body]) +.. hy:macro:: (do [#* body]) ``do`` (called ``progn`` in some Lisps) takes any number of forms, evaluates them, and returns the value of the last one, or ``None`` if no @@ -312,7 +312,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (+ 1 (do (setv x (+ 1 1)) x)) ; => 3 -.. hy:function:: (do-mac [#* body]) +.. hy:macro:: (do-mac [#* body]) ``do-mac`` evaluates its arguments (in order) at compile time, and leaves behind the value of the last argument (``None`` if no arguments were provided) as code to be run. The effect is similar to defining and then immediately calling a nullary macro, hence the name, which stands for "do macro". :: @@ -323,7 +323,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Contrast with :hy:func:`eval-and-compile`, which evaluates the same code at compile-time and run-time, instead of using the result of the compile-time run as code for run-time. ``do-mac`` is also similar to Common Lisp's SHARPSIGN DOT syntax (``#.``), from which it differs by evaluating at compile-time rather than read-time. -.. hy:function:: (for [#* args]) +.. hy:macro:: (for [#* args]) ``for`` compiles to one or more :py:keyword:`for` statements, which execute code repeatedly for each element of an iterable object. @@ -378,7 +378,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. 3 loop finished -.. hy:function:: (assert [condition [label None]]) +.. hy:macro:: (assert [condition [label None]]) ``assert`` compiles to an :py:keyword:`assert` statement, which checks whether a condition is true. The first argument, specifying the condition to @@ -391,7 +391,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (assert (= 1 2) "one should equal two") ; AssertionError: one should equal two -.. hy:function:: (global [sym #* syms]) +.. hy:macro:: (global [sym #* syms]) ``global`` compiles to a :py:keyword:`global` statement, which declares one or more names as referring to global (i.e., module-level) variables. The @@ -407,7 +407,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print a b) ; => 2 10 -.. hy:function:: (import [#* forms]) +.. hy:macro:: (import [#* forms]) ``import`` compiles to an :py:keyword:`import` statement, which makes objects in a different module available in the current module. It always returns @@ -445,7 +445,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. `. The macro :hy:func:`export ` is a handy way to set ``__all__`` in a Hy program. -.. hy:function:: (eval-and-compile [#* body]) +.. hy:macro:: (eval-and-compile [#* body]) ``eval-and-compile`` takes any number of forms as arguments. The input forms are evaluated as soon as the ``eval-and-compile`` form is compiled, instead of being deferred until run-time. The input forms are also left in the program so they can be executed at run-time as usual. So, if you compile and immediately execute a program (as calling ``hy foo.hy`` does when ``foo.hy`` doesn't have an up-to-date byte-compiled version), ``eval-and-compile`` forms will be evaluated twice. The return value is the final argument, as in ``do``. @@ -463,7 +463,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Had the ``defn`` not been wrapped in ``eval-and-compile``, ``m`` wouldn't be able to call ``add``, because when the compiler was expanding ``(m 3)``, ``add`` wouldn't exist yet. -.. hy:function:: (eval-when-compile [#* body]) +.. hy:macro:: (eval-when-compile [#* body]) As ``eval-and-compile``, but the code isn't executed at run-time, and ``None`` is returned. Hence, ``eval-when-compile`` doesn't directly contribute any code to the final program, although it can still change Hy's state while compiling (e.g., by defining a function). :: @@ -477,7 +477,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print (m 3)) ; prints 5 (print (add 3 6)) ; raises NameError: name 'add' is not defined -.. hy:function:: (lfor [#* args]) +.. hy:macro:: (lfor [#* args]) The comprehension forms ``lfor``, :hy:func:`sfor`, :hy:func:`dfor`, :hy:func:`gfor`, and :hy:func:`for` are used to produce various kinds of loops, including Python-style @@ -533,7 +533,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. In ``for``, by contrast, iteration and ``:setv`` clauses share the caller's scope and are visible outside the form. -.. hy:function:: (dfor [#* args]) +.. hy:macro:: (dfor [#* args]) ``dfor`` creates a :ref:`dictionary comprehension `. Its syntax is the same as that of :hy:func:`lfor` except that it takes two trailing @@ -544,7 +544,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. {0 0 1 10 2 20 3 30 4 40} -.. hy:function:: (gfor [#* args]) +.. hy:macro:: (gfor [#* args]) ``gfor`` creates a :ref:`generator expression `. Its syntax is the same as that of :hy:func:`lfor`. The difference is that ``gfor`` returns @@ -559,12 +559,12 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. => accum [0 1 2 3 4 5] -.. hy:function:: (sfor [#* args]) +.. hy:macro:: (sfor [#* args]) ``sfor`` creates a :ref:`set comprehension `. ``(sfor CLAUSES VALUE)`` is equivalent to ``(set (lfor CLAUSES VALUE))``. See :hy:func:`lfor`. -.. hy:function:: (setv [#* args]) +.. hy:macro:: (setv [#* args]) ``setv`` compiles to an :ref:`assignment statement ` (see :hy:func:`setx` for assignment expressions), which sets the value of a variable or some other assignable expression. It requires an even number of arguments, and always returns ``None``. The most common case is two arguments, where the first is a symbol:: @@ -588,7 +588,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. See :hy:func:`let` to simulate more traditionally Lispy block-level scoping. -.. hy:function:: (setx [target value]) +.. hy:macro:: (setx [target value]) ``setx`` compiles to an assignment expression. Thus, unlike :hy:func:`setv`, it returns the assigned value. It takes exactly two arguments, and the target must be a bare symbol. Python 3.8 or later is required. :: @@ -596,7 +596,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print x "is greater than 0")) ; => 3 is greater than 0 -.. hy:function:: (let [bindings #* body]) +.. hy:macro:: (let [bindings #* body]) ``let`` creates lexically-scoped names for local variables. This form takes a list of binding pairs followed by a *body* which gets executed. A let-bound @@ -662,7 +662,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. _extended iterable unpacking: https://www.python.org/dev/peps/pep-3132/#specification -.. hy:function:: (match [subject #* cases]) +.. hy:macro:: (match [subject #* cases]) The ``match`` form creates a :ref:`match statement `. It requires Python 3.10 or later. The first argument should be the subject, @@ -711,7 +711,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. :hy:func:`case `, or simply use :hy:func:`cond `. -.. hy:function:: (defclass [arg1 #* args]) +.. hy:macro:: (defclass [arg1 #* args]) ``defclass`` compiles to a :py:keyword:`class` statement, which creates a new class. It always returns ``None``. Only one argument, specifying the @@ -734,18 +734,18 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (defn method2 [self arg1 arg2] …)) -.. hy:function:: (del [#* args]) +.. hy:macro:: (del [#* args]) ``del`` compiles to a :py:keyword:`del` statement, which deletes variables or other assignable expressions. It always returns ``None``. :: (del foo (get mydict "mykey") myobj.myattr) -.. hy:function:: (nonlocal [sym #* syms]) +.. hy:macro:: (nonlocal [sym #* syms]) As :hy:func:`global`, but the result is a :py:keyword:`nonlocal` statement. -.. hy:function:: (py [string]) +.. hy:macro:: (py [string]) ``py`` parses the given Python code at compile-time and inserts the result into the generated abstract syntax tree. Thus, you can mix Python code into a Hy @@ -768,7 +768,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. _pys-specialform: -.. hy:function:: (pys [string]) +.. hy:macro:: (pys [string]) As :hy:func:`py `, but the code can consist of zero or more statements, including compound statements such as ``for`` and ``def``. ``pys`` always @@ -782,7 +782,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Python would otherwise forbid this, but beware that significant leading whitespace in embedded string literals will be removed. -.. hy:function:: (quasiquote [form]) +.. hy:macro:: (quasiquote [form]) ``quasiquote`` allows you to quote a form, but also selectively evaluate expressions. Expressions inside a ``quasiquote`` can be selectively evaluated @@ -801,7 +801,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. ; equivalent to '(foo bar baz) -.. hy:function:: (quote [form]) +.. hy:macro:: (quote [form]) ``quote`` returns the form passed to it without evaluating it. ``quote`` can alternatively be written using the apostrophe (``'``) symbol. @@ -819,7 +819,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Hello World -.. hy:function:: (require [#* args]) +.. hy:macro:: (require [#* args]) ``require`` is used to import macros and reader macros from one or more given modules. It allows parameters in all the same formats as ``import``. @@ -945,7 +945,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. to do introspection of the current module's set of defined macros, which isn't really supported anyway. -.. hy:function:: (return [object]) +.. hy:macro:: (return [object]) ``return`` compiles to a :py:keyword:`return` statement. It exits the current function, returning its argument if provided with one, or @@ -969,7 +969,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. None) (print (f 4)) ; Prints "14" and then "None" -.. hy:function:: (cut [coll arg1 arg2 arg3]) +.. hy:macro:: (cut [coll arg1 arg2 arg3]) ``cut`` compiles to a :ref:`slicing expression `, which selects multiple elements of a sequential data structure. The first argument is the object to be sliced. The remaining arguments are optional, and understood the same way as in a Python slicing expression. :: @@ -982,7 +982,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. A ``cut`` form is a valid target for assignment (with :hy:func:`setv`, ``+=``, etc.) and for deletion (with :hy:func:`del`). -.. hy:function:: (raise [exception :from other]) +.. hy:macro:: (raise [exception :from other]) ``raise`` compiles to a :py:keyword:`raise` statement, which throws an exception. With no arguments, the current exception is reraised. With one @@ -997,7 +997,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. EXCEPTION_2)``, which compiles to a Python ``raise … from`` statement like ``raise EXCEPTION_1 from EXCEPTION_2``. -.. hy:function:: (try [#* body]) +.. hy:macro:: (try [#* body]) ``try`` compiles to a :py:keyword:`try` statement, which can catch exceptions and run cleanup actions. It begins with any number of body forms. @@ -1044,8 +1044,8 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. The return value of ``try`` is the last form evaluated among the main body, ``except`` forms, ``except*`` forms, and ``else``. -.. hy:function:: (unpack-iterable) -.. hy:function:: (unpack-mapping) +.. hy:macro:: (unpack-iterable) +.. hy:macro:: (unpack-mapping) (Also known as the splat operator, star operator, argument expansion, argument explosion, argument gathering, and varargs, among others...) @@ -1083,7 +1083,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. => (f #* [1] #* [2] #** {"c" 3} #** {"d" 4}) [1 2 3 4] -.. hy:function:: (unquote [symbol]) +.. hy:macro:: (unquote [symbol]) Within a quasiquoted form, ``unquote`` forces evaluation of a symbol. ``unquote`` is aliased to the tilde (``~``) symbol. @@ -1097,7 +1097,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. '(= nickname "Cuddles") -.. hy:function:: (unquote-splice [symbol]) +.. hy:macro:: (unquote-splice [symbol]) ``unquote-splice`` forces the evaluation of a symbol within a quasiquoted form, much like ``unquote``. ``unquote-splice`` can be used when the symbol @@ -1128,7 +1128,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. parser. So, if you want to unquote the symbol ``@foo`` with ``~``, you must use whitespace to separate ``~`` and ``@``, as in ``~ @foo``. -.. hy:function:: (while [condition #* body]) +.. hy:macro:: (while [condition #* body]) ``while`` compiles to a :py:keyword:`while` statement, which executes some code as long as a condition is met. The first argument to ``while`` is the @@ -1178,7 +1178,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. In condition At end of outer loop -.. hy:function:: (with [managers #* body]) +.. hy:macro:: (with [managers #* body]) ``with`` compiles to a :py:keyword:`with` statement, which wraps some code with one or more :ref:`context managers `. The first @@ -1217,11 +1217,11 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print (with [o (open "file.txt" "rt")] (.read o))) -.. hy:function:: (with/a [managers #* body]) +.. hy:macro:: (with/a [managers #* body]) As :hy:func:`with`, but compiles to an :py:keyword:`async with` statement. -.. hy:function:: (yield [value]) +.. hy:macro:: (yield [value]) ``yield`` compiles to a :ref:`yield expression `, which returns a value as a generator. As in Python, one argument, the value to @@ -1235,7 +1235,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. For ``yield from``, see :hy:func:`yield-from`. -.. hy:function:: (yield-from [object]) +.. hy:macro:: (yield-from [object]) ``yield-from`` compiles to a :ref:`yield-from expression `, which returns a value from a subgenerator. The syntax is the same as that of @@ -1248,7 +1248,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (hy.repr (list (zip "abc" (myrange)))) ; => [#("a" 0) #("b" 1) #("c" 2)] -.. hy:function:: (pragma) +.. hy:macro:: (pragma) ``pragma`` is reserved as a core macro name for future use, especially for allowing backwards-compatible addition of new features after the release of Hy From e8b29f3f22ec480ee0e96dbb897ce5482ce0b97b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 16 Jan 2023 10:53:11 -0500 Subject: [PATCH 122/342] Add some missing parens and args in `api.rst` --- docs/api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index fdb492499..f09e4897c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -232,7 +232,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print "world")) (asyncio.run (main)) -.. hy:macro:: break +.. hy:macro:: (break) ``break`` compiles to a :py:keyword:`break` statement, which terminates the enclosing loop. The following example has an infinite ``while`` loop that @@ -281,7 +281,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Python. -.. hy:macro:: continue +.. hy:macro:: (continue) ``continue`` compiles to a :py:keyword:`continue` statement, which returns execution to the start of a loop. In the following example, ``(.append @@ -1044,8 +1044,8 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. The return value of ``try`` is the last form evaluated among the main body, ``except`` forms, ``except*`` forms, and ``else``. -.. hy:macro:: (unpack-iterable) -.. hy:macro:: (unpack-mapping) +.. hy:macro:: (unpack-iterable [form]) +.. hy:macro:: (unpack-mapping [form]) (Also known as the splat operator, star operator, argument expansion, argument explosion, argument gathering, and varargs, among others...) From db53eba6b2c67ec1cbbe9f66c9ff75a4a6cc0628 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 16 Jan 2023 18:58:21 -0500 Subject: [PATCH 123/342] Delete a leftover debugging print --- tests/native_tests/model_patterns.hy | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/native_tests/model_patterns.hy b/tests/native_tests/model_patterns.hy index 5ca493db1..dd3ccebb9 100644 --- a/tests/native_tests/model_patterns.hy +++ b/tests/native_tests/model_patterns.hy @@ -66,5 +66,4 @@ for n from 1 to 3 for p in [k n (* 10 n)] do (.append l p) (-= k 1)) - (print l) (assert (= l [2 1 10 -1 2 20 -4 3 30]))) From 53c1ad2530992f43c6e6e729a103d5a9eae32579 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 17 Jan 2023 16:26:32 -0500 Subject: [PATCH 124/342] Use a text file in `tests/resources` for tests instead of the README. --- tests/native_tests/setv.hy | 2 +- tests/native_tests/with.hy | 8 ++++---- tests/resources/text.txt | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 tests/resources/text.txt diff --git a/tests/native_tests/setv.hy b/tests/native_tests/setv.hy index a755c46b5..467868ab1 100644 --- a/tests/native_tests/setv.hy +++ b/tests/native_tests/setv.hy @@ -72,7 +72,7 @@ (an (setv x (for [i (range 3)] i (+ i 1)))) (an (setv x (assert True))) - (an (setv x (with [(open "README.md" "r")] 3))) + (an (setv x (with [(open "tests/resources/text.txt" "r")] 3))) (assert (= x 3)) (an (setv x (try (/ 1 2) (except [ZeroDivisionError] "E1")))) (assert (= x .5)) diff --git a/tests/native_tests/with.hy b/tests/native_tests/with.hy index f9f9556b7..e01ed06bd 100644 --- a/tests/native_tests/with.hy +++ b/tests/native_tests/with.hy @@ -4,13 +4,13 @@ tests.resources [AsyncWithTest]) (defn test-context [] - (with [fd (open "README.md" "r")] (assert fd)) - (with [(open "README.md" "r")] (do))) + (with [fd (open "tests/resources/text.txt" "r")] (assert fd)) + (with [(open "tests/resources/text.txt" "r")] (do))) (defn test-with-return [] (defn read-file [filename] - (with [fd (open filename "r")] (.read fd))) - (assert (!= 0 (len (read-file "README.md"))))) + (with [fd (open filename "r" :encoding "UTF-8")] (.read fd))) + (assert (= (read-file "tests/resources/text.txt") "TAARGÜS TAARGÜS\n"))) (defclass WithTest [object] (defn __init__ [self val] diff --git a/tests/resources/text.txt b/tests/resources/text.txt new file mode 100644 index 000000000..fa80786a8 --- /dev/null +++ b/tests/resources/text.txt @@ -0,0 +1 @@ +TAARGÜS TAARGÜS From d64429a0fb66ffc87531c9b66147046dc358f805 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 16 Jan 2023 19:53:35 -0500 Subject: [PATCH 125/342] Add some Pyodide test skips --- hy/_compat.py | 1 + tests/native_tests/import.hy | 6 ++++-- tests/test_bin.py | 8 +++++++- tests/test_hy2py.py | 5 +++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/hy/_compat.py b/hy/_compat.py index ba709e092..c32cf2420 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -7,6 +7,7 @@ PY3_10 = sys.version_info >= (3, 10) PY3_11 = sys.version_info >= (3, 11) PYPY = platform.python_implementation() == "PyPy" +PYODIDE = platform.system() == 'Emscripten' if not PY3_9: diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy index 00a9b1991..8821fd77c 100644 --- a/tests/native_tests/import.hy +++ b/tests/native_tests/import.hy @@ -6,7 +6,8 @@ os.path [exists isdir isfile] sys :as systest sys - pytest) + pytest + hy._compat [PYODIDE]) (defn test-imported-bits [] @@ -202,7 +203,8 @@ in expansions." ;; Now that bytecode is present, reload the module, clear the `require`d ;; macros and tags, and rerun the tests. - (assert (os.path.isfile pyc-file)) + (when (not PYODIDE) + (assert (os.path.isfile pyc-file))) ;; Reload the module and clear the local macro context. (.clear sys.path_importer_cache) diff --git a/tests/test_bin.py b/tests/test_bin.py index bcc8dfc21..b4543407d 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -12,7 +12,13 @@ import pytest -from hy._compat import PY3_9, PYPY +from hy._compat import PY3_9, PYPY, PYODIDE + + +if PYODIDE: + pytest.skip( + '`subprocess.Popen` not implemented on Pyodide', + allow_module_level = True) def pyr(s=""): diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py index 5a993be92..2d18ed5da 100644 --- a/tests/test_hy2py.py +++ b/tests/test_hy2py.py @@ -2,8 +2,12 @@ import itertools import math import os +import platform + +import pytest import hy.importer +from hy._compat import PYODIDE from hy import mangle @@ -13,6 +17,7 @@ def test_direct_import(): assert_stuff(tests.resources.pydemo) +@pytest.mark.skipif(PYODIDE, reason = "subprocess.check_call not implemented on Pyodide") def test_hy2py_import(): import contextlib import os From 53dad182ad6a09850d919342f2a57e6689c6596a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 16 Jan 2023 19:21:43 -0500 Subject: [PATCH 126/342] Skip async tests on Pyodide --- tests/native_tests/comprehensions.hy | 7 ++++--- tests/native_tests/decorators.hy | 5 +++-- tests/native_tests/functions.hy | 7 ++++--- tests/native_tests/with.hy | 10 +++++----- tests/resources/__init__.py | 11 +++++++++++ tests/test_hy2py.py | 5 ++++- 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy index abeb10634..b92964205 100644 --- a/tests/native_tests/comprehensions.hy +++ b/tests/native_tests/comprehensions.hy @@ -1,7 +1,8 @@ (import types asyncio - pytest) + pytest + tests.resources [async-test]) (defn test-comprehension-types [] @@ -385,7 +386,7 @@ (assert (= x 3))) -(defn test-for-async [] +(defn [async-test] test-for-async [] (defn/a numbers [] (for [i [1 2]] (yield i))) @@ -398,7 +399,7 @@ (assert (= x 3)))))) -(defn test-for-async-else [] +(defn [async-test] test-for-async-else [] (defn/a numbers [] (for [i [1 2]] (yield i))) diff --git a/tests/native_tests/decorators.hy b/tests/native_tests/decorators.hy index 9997c6b6d..3fedea468 100644 --- a/tests/native_tests/decorators.hy +++ b/tests/native_tests/decorators.hy @@ -1,5 +1,6 @@ (import - asyncio) + asyncio + tests.resources [async-test]) (defn test-decorated-1line-function [] @@ -52,7 +53,7 @@ (assert (= l ["dec" "arg" "foo" "foo fn" "bar body" 1]))) -(defn test-decorated-defn/a [] +(defn [async-test] test-decorated-defn/a [] (defn decorator [func] (fn/a [] (/ (await (func)) 2))) (defn/a [decorator] coro-test [] diff --git a/tests/native_tests/functions.hy b/tests/native_tests/functions.hy index bff25a0b3..2cc69b60f 100644 --- a/tests/native_tests/functions.hy +++ b/tests/native_tests/functions.hy @@ -3,7 +3,8 @@ (import asyncio typing [List] - pytest) + pytest + tests.resources [async-test]) (defn test-fn [] @@ -24,7 +25,7 @@ (assert (= (fn-test) None))) -(defn test-fn/a [] +(defn [async-test] test-fn/a [] (assert (= (asyncio.run ((fn/a [] (await (asyncio.sleep 0)) [1 2 3]))) [1 2 3]))) @@ -132,7 +133,7 @@ (setv x [#* spam] y 1))) -(defn test-defn/a [] +(defn [async-test] test-defn/a [] (defn/a coro-test [] (await (asyncio.sleep 0)) [1 2 3]) diff --git a/tests/native_tests/with.hy b/tests/native_tests/with.hy index e01ed06bd..fb2b364fb 100644 --- a/tests/native_tests/with.hy +++ b/tests/native_tests/with.hy @@ -1,7 +1,7 @@ (import asyncio pytest - tests.resources [AsyncWithTest]) + tests.resources [async-test AsyncWithTest]) (defn test-context [] (with [fd (open "tests/resources/text.txt" "r")] (assert fd)) @@ -50,13 +50,13 @@ (assert (= t2 2)) (assert (= t3 3)))) -(defn test-single-with/a [] +(defn [async-test] test-single-with/a [] (asyncio.run ((fn/a [] (with/a [t (AsyncWithTest 1)] (assert (= t 1))))))) -(defn test-two-with/a [] +(defn [async-test] test-two-with/a [] (asyncio.run ((fn/a [] (with/a [t1 (AsyncWithTest 1) @@ -64,7 +64,7 @@ (assert (= t1 1)) (assert (= t2 2))))))) -(defn test-thrice-with/a [] +(defn [async-test] test-thrice-with/a [] (asyncio.run ((fn/a [] (with/a [t1 (AsyncWithTest 1) @@ -74,7 +74,7 @@ (assert (= t2 2)) (assert (= t3 3))))))) -(defn test-quince-with/a [] +(defn [async-test] test-quince-with/a [] (asyncio.run ((fn/a [] (with/a [t1 (AsyncWithTest 1) diff --git a/tests/resources/__init__.py b/tests/resources/__init__.py index 5fec6dbb9..5385e8aff 100644 --- a/tests/resources/__init__.py +++ b/tests/resources/__init__.py @@ -1,3 +1,8 @@ +import pytest + +from hy._compat import PYODIDE + + in_init = "chippy" @@ -9,6 +14,12 @@ def function_with_a_dash(): pass +can_test_async = not PYODIDE +async_test = pytest.mark.skipif( + not can_test_async, + reason = "`asyncio.run` not implemented") + + class AsyncWithTest: def __init__(self, val): self.val = val diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py index 2d18ed5da..1e9dfe70c 100644 --- a/tests/test_hy2py.py +++ b/tests/test_hy2py.py @@ -10,6 +10,8 @@ from hy._compat import PYODIDE from hy import mangle +from tests.resources import can_test_async + def test_direct_import(): import tests.resources.pydemo @@ -153,7 +155,8 @@ class C: assert m.pys_accum == [0, 1, 2, 3, 4] assert m.py_accum == "01234" - assert asyncio.run(m.coro()) == list("abcdef") + if can_test_async: + assert asyncio.run(m.coro()) == list("abcdef") assert m.cheese == [1, 1] assert m.mac_results == ["x", "x"] From 400dead37b3f6ab46e8df394fc737b569b34047a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 22 Jan 2023 09:27:57 -0500 Subject: [PATCH 127/342] Test on Pyodide on GitHub Actions --- .github/workflows/tests.yml | 41 ++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0277c392b..0cb5ff483 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: matrix: name-prefix: [''] os: [ubuntu-latest] - python: [3.7, 3.8, 3.9, '3.10', 3.11, pypy-3.9] + python: [3.7, 3.8, 3.9, '3.10', 3.11, pypy-3.9, pyodide] include: # To keep the overall number of runs low, we test Windows # only on the latest CPython. @@ -21,6 +21,7 @@ jobs: name: ${{ format('{0}{1}', matrix.name-prefix, matrix.python) }} runs-on: ${{ matrix.os }} env: + EMSCRIPTEN_VERSION: 3.1.27 TERM: xterm-256color # This is needed to avoid a terminfo-related crash when # testing PyPy. @@ -29,11 +30,37 @@ jobs: steps: - run: git config --global core.autocrlf false - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - if: ${{ matrix.python != 'pyodide' }} + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - - run: pip install . && rm -r hy - # We want to be sure we're testing the installed version, - # instead of running from the source tree. - - run: pip install pytest - - run: pytest + - if: ${{ matrix.python == 'pyodide' }} + uses: actions/setup-node@v3 + - if: ${{ matrix.python == 'pyodide' }} + uses: mymindstorm/setup-emsdk@v11 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + actions-cache-folder: emsdk-cache + - name: Install + shell: bash + run: | + if [[ ${{ matrix.python }} = pyodide ]] ; then + npm install pyodide + pip install 'pip >= 22.3.1' + # Older pips may fail to install `pyodide-build`. + pip install pyodide-build + pyodide venv .venv-pyodide + source .venv-pyodide/bin/activate + fi + pip install . + rm -r hy + # We want to be sure we're testing the installed version, + # instead of running from the source tree. + pip install pytest + - name: Test + shell: bash + run: | + if [[ ${{ matrix.python }} = pyodide ]] ; then + source .venv-pyodide/bin/activate + fi + python -m pytest tests From 938e5fa55f10a716a4c641ccae89319bb5c19e09 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 17 Jan 2023 16:52:07 -0500 Subject: [PATCH 128/342] List the supported Pythons in the documentation --- README.md | 3 +++ docs/index.rst | 3 +++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 8cc4d9270..5ab103a54 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,9 @@ To install the latest release of Hy, just use the command `pip3 install --user hy`. Then you can start an interactive read-eval-print loop (REPL) with the command `hy`, or run a Hy program with `hy myprogram.hy`. +Hy is tested on all released and currently maintained versions of CPython (on +Linux and Windows), and on recent versions of PyPy and Pyodide. + * [Try Hy with a web console](https://hylang.github.io/hy-interpreter) * [Why Hy?](http://docs.hylang.org/en/stable/whyhy.html) * [Tutorial](http://docs.hylang.org/en/stable/tutorial.html) diff --git a/docs/index.rst b/docs/index.rst index 2b77e6199..58efff031 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,9 @@ To install the latest release of Hy, just use the command ``pip3 install --user hy``. Then you can start an interactive read-eval-print loop (REPL) with the command ``hy``, or run a Hy program with ``hy myprogram.hy``. +Hy is tested on all released and currently maintained versions of CPython (on +Linux and Windows), and on recent versions of PyPy and Pyodide. + .. toctree:: :maxdepth: 3 From 59afccfe028ab4350e1643211a72b324a8f4542d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 22 Jan 2023 09:07:36 -0500 Subject: [PATCH 129/342] Add PyPy and Emscripten trove classifiers --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 8f427082e..175c23abb 100755 --- a/setup.py +++ b/setup.py @@ -82,6 +82,8 @@ def run(self): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: PyPy", + "Environment :: WebAssembly :: Emscripten", "Topic :: Software Development :: Code Generators", "Topic :: Software Development :: Compilers", "Topic :: Software Development :: Libraries", From 5fedec779da39f3cc6f3d4d5f80f6ecc962daf14 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 17 Jan 2023 16:34:13 -0500 Subject: [PATCH 130/342] Update NEWS --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index 5b63bcc97..4fa2722bd 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -33,6 +33,7 @@ Bug Fixes New Features ------------------------------ +* Pyodide is now officially supported. * `.`, `..`, etc. are now usable as ordinary symbols (with the remaining special rule that `...` compiles to `Ellipsis`) * On Pythons ≥ 3.7, Hy modules can now be imported from ZIP From 62b7e290ce30ae49df9db5bae70c897afecbf8e1 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 22 Jan 2023 09:32:54 -0500 Subject: [PATCH 131/342] Autoformat --- hy/_compat.py | 2 +- tests/resources/__init__.py | 5 ++--- tests/test_bin.py | 3 +-- tests/test_hy2py.py | 5 ++--- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/hy/_compat.py b/hy/_compat.py index c32cf2420..d51809308 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -7,7 +7,7 @@ PY3_10 = sys.version_info >= (3, 10) PY3_11 = sys.version_info >= (3, 11) PYPY = platform.python_implementation() == "PyPy" -PYODIDE = platform.system() == 'Emscripten' +PYODIDE = platform.system() == "Emscripten" if not PY3_9: diff --git a/tests/resources/__init__.py b/tests/resources/__init__.py index 5385e8aff..458d85fd9 100644 --- a/tests/resources/__init__.py +++ b/tests/resources/__init__.py @@ -2,7 +2,6 @@ from hy._compat import PYODIDE - in_init = "chippy" @@ -16,8 +15,8 @@ def function_with_a_dash(): can_test_async = not PYODIDE async_test = pytest.mark.skipif( - not can_test_async, - reason = "`asyncio.run` not implemented") + not can_test_async, reason="`asyncio.run` not implemented" +) class AsyncWithTest: diff --git a/tests/test_bin.py b/tests/test_bin.py index b4543407d..95899287b 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -12,8 +12,7 @@ import pytest -from hy._compat import PY3_9, PYPY, PYODIDE - +from hy._compat import PY3_9, PYODIDE, PYPY if PYODIDE: pytest.skip( diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py index 1e9dfe70c..d4c129501 100644 --- a/tests/test_hy2py.py +++ b/tests/test_hy2py.py @@ -7,9 +7,8 @@ import pytest import hy.importer -from hy._compat import PYODIDE from hy import mangle - +from hy._compat import PYODIDE from tests.resources import can_test_async @@ -19,7 +18,7 @@ def test_direct_import(): assert_stuff(tests.resources.pydemo) -@pytest.mark.skipif(PYODIDE, reason = "subprocess.check_call not implemented on Pyodide") +@pytest.mark.skipif(PYODIDE, reason="subprocess.check_call not implemented on Pyodide") def test_hy2py_import(): import contextlib import os From da1741feaf3086e875236a33a0826726ad26771f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 31 Jan 2023 14:18:38 -0500 Subject: [PATCH 132/342] Remove pre-commit hooks --- .github/workflows/static-analysis.yml | 14 -------------- .pre-commit-config.yaml | 16 ---------------- 2 files changed, 30 deletions(-) delete mode 100644 .github/workflows/static-analysis.yml delete mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml deleted file mode 100644 index 959f9f95a..000000000 --- a/.github/workflows/static-analysis.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Perform static analysis on Python files - -on: - pull_request: {} - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - uses: pre-commit/action@v2.0.3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 6e69632c8..000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -exclude: "^docs/" -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 - hooks: - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - - repo: https://github.com/pycqa/isort - rev: 5.10.1 - hooks: - - id: isort From f3d26007e5dbe0c07e72abecfeb11acea0907064 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 31 Jan 2023 14:19:46 -0500 Subject: [PATCH 133/342] Remove autoformat section from CONTRIBUTING.rst --- CONTRIBUTING.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2a1c10d78..d13b883c8 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -67,14 +67,6 @@ The first line of a commit message should describe the overall change in 50 characters or less. If you wish to add more information, separate it from the first line with a blank line. -Code formatting ---------------- - -All Python source code (``.py``) should be formatted with ``black`` and ``isort``. -This can be accomplished by running ``black hy tests`` and ``isort hy tests`` from the root of this repository. -Formatting of Python files is checked automatically via GitHub Actions for all pull requests. -No PR may be merged if it fails that check. - Testing ------- From 73ec7d8316a6d4078640b48fb5a40908af8b7f0a Mon Sep 17 00:00:00 2001 From: danieltanfh Date: Wed, 25 Jan 2023 13:34:32 +0800 Subject: [PATCH 134/342] feat: add folder output for hy2py Co-authored-by: Kodi Arfer --- NEWS.rst | 1 + docs/cli.rst | 2 +- hy/cmdline.py | 113 +++++++++++++++++++++++++++++++++------------- tests/test_bin.py | 28 +++++++++++- 4 files changed, 110 insertions(+), 34 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 4fa2722bd..850589392 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -38,6 +38,7 @@ New Features remaining special rule that `...` compiles to `Ellipsis`) * On Pythons ≥ 3.7, Hy modules can now be imported from ZIP archives in the same way as Python modules, via `zipimport`_. +* `hy2py` now supports directory input, and will recursively convert hy source code into python source code. .. _zipimport: https://docs.python.org/3.11/library/zipimport.html diff --git a/docs/cli.rst b/docs/cli.rst index 374be98c9..2390d44eb 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -43,7 +43,7 @@ for a complete list of options and :py:ref:`Python's documentation hy2py ----- -``hy2py`` is a program to convert Hy source code into Python source code. Use ``hy2py --help`` for usage instructions. It can take its input from standard input or from a filename provided as a command-line argument. The result is written to standard output. +``hy2py`` is a program to convert Hy source code into Python source code. Use ``hy2py --help`` for usage instructions. It can take its input from standard input, from a filename, or folder name provided as a command-line argument. If it is a folder, the output parameter (--output/-o) must be provided. When the output parameter is provided, the output will be written into the folder or file, otherwise the result is written to standard output. .. warning:: ``hy2py`` can execute arbitrary code. Don't give it untrusted input. diff --git a/hy/cmdline.py b/hy/cmdline.py index f6093b2e1..72d0bc224 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -7,6 +7,7 @@ import py_compile import runpy import sys +from contextlib import nullcontext from pathlib import Path import hy @@ -330,6 +331,48 @@ def hyc_main(): return rv +def hy2py_worker(source, options, filename, output_filepath=None): + if isinstance(source, Path): + source = source.read_text(encoding="UTF-8") + + if not output_filepath and options.output: + output_filepath = options.output + + set_path(filename) + with ( + open(output_filepath, "w", encoding="utf-8") + if output_filepath + else nullcontext() + ) as output_file: + + def printing_source(hst): + for node in hst: + if options.with_source: + print(node, file=output_file) + yield node + + hst = hy.models.Lazy( + printing_source(read_many(source, filename, skip_shebang=True)) + ) + hst.source = source + hst.filename = filename + + with filtered_hy_exceptions(): + _ast = hy_compile(hst, "__main__", filename=filename, source=source) + + if options.with_source: + print() + print() + + if options.with_ast: + print(ast.dump(_ast, **(dict(indent=2) if PY3_9 else {})), file=output_file) + print() + print() + + if not options.without_python: + print(ast.unparse(_ast), file=output_file) + + # entry point for cmd line script "hy2py" def hy2py_main(): options = dict( @@ -342,7 +385,8 @@ def hy2py_main(): "FILE", type=str, nargs="?", - help='Input Hy code (use STDIN if "-" or ' "not provided)", + help='Input Hy code (can be file or directory) (use STDIN if "-" or ' + "not provided)", ) parser.add_argument( "--with-source", @@ -359,46 +403,51 @@ def hy2py_main(): action="store_true", help=("Do not show the Python code generated " "from the AST"), ) + parser.add_argument( + "--output", + "-o", + type=str, + nargs="?", + help="output file / directory", + ) options = parser.parse_args(sys.argv[1:]) if options.FILE is None or options.FILE == "-": sys.path.insert(0, "") filename = "" - source = sys.stdin.read() + hy2py_worker(sys.stdin.read(), options, filename) else: filename = options.FILE - set_path(filename) - with open(options.FILE, "r", encoding="utf-8") as source_file: - source = source_file.read() - - def printing_source(hst): - for node in hst: - if options.with_source: - print(node) - yield node - - hst = hy.models.Lazy( - printing_source(read_many(source, filename, skip_shebang=True)) - ) - hst.source = source - hst.filename = filename - - with filtered_hy_exceptions(): - _ast = hy_compile(hst, "__main__", filename=filename, source=source) - - if options.with_source: - print() - print() - - if options.with_ast: - print(ast.dump(_ast, **(dict(indent=2) if PY3_9 else {}))) - print() - print() - - if not options.without_python: - print(ast.unparse(_ast)) + if os.path.isdir(filename): + # handle recursively if --output is specified + if not options.output: + raise ValueError( + f"{filename} is a directory but the output directory is not specified. Use --output or -o in command line arguments to specify the output directory." + ) + os.makedirs(options.output, exist_ok=True) + for path, subdirs, files in os.walk(filename): + for name in files: + filename_raw, filename_ext = os.path.splitext(name) + if filename_ext == ".hy": + filepath = os.path.join(path, name) + # make sure to follow original file structure + subdirectory = os.path.relpath(path, filename) + output_directory_path = os.path.join( + options.output if options.output else path, subdirectory + ) + os.makedirs(output_directory_path, exist_ok=True) + hy2py_worker( + Path(filepath), + options, + filename, + output_filepath=os.path.join( + output_directory_path, filename_raw + ".py" + ), + ) + else: + hy2py_worker(Path(options.FILE), options, filename) parser.exit(0) diff --git a/tests/test_bin.py b/tests/test_bin.py index 95899287b..4319bce92 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -26,7 +26,7 @@ def pyr(s=""): def run_cmd( cmd, stdin_data=None, expect=0, dontwritebytecode=False, - stdout=subprocess.PIPE): + cwd=None, stdout=subprocess.PIPE): env = dict(os.environ) if dontwritebytecode: env["PYTHONDONTWRITEBYTECODE"] = "1" @@ -41,6 +41,7 @@ def run_cmd( universal_newlines=True, shell=False, env=env, + cwd=cwd, ) output = p.communicate(input=stdin_data) assert p.wait() == expect @@ -717,3 +718,28 @@ def test_assert(tmp_path, monkeypatch): show_msg = has_msg and not optim and not test assert ("msging" in out) == show_msg assert ("bye" in err) == show_msg + + +def test_hy2py_recursive(tmp_path): + (tmp_path / 'hy').mkdir() + (tmp_path / "hy/first.hy").write_text(""" + (import folder.second [a b]) + (print a) + (print b)""") + (tmp_path / "hy/folder").mkdir() + (tmp_path / "hy/folder/second.hy").write_text(""" + (setv a 1) + (setv b "hello world")""") + + _, err = run_cmd(f"hy2py {(tmp_path / 'hy').as_posix()}", expect=1) + assert "ValueError" in err + + run_cmd("hy2py " + + f"{(tmp_path / 'hy').as_posix()} " + + f"--output {(tmp_path / 'py').as_posix()}") + assert set((tmp_path / 'py').rglob('*')) == { + tmp_path / 'py' / p + for p in ('first.py', 'folder', 'folder/second.py')} + + output, _ = run_cmd(f"python3 first.py", cwd = tmp_path / 'py') + assert output == "1\nhello world\n" From 1f7256b4deebd9e15ebbbf6a662e0e07ea08ea87 Mon Sep 17 00:00:00 2001 From: danieltanfh Date: Wed, 25 Jan 2023 13:23:23 +0800 Subject: [PATCH 135/342] chore: Add Daniel Tan to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 194456f09..d28bcd1b9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -109,3 +109,4 @@ * Dmitry Ivanov * Andrey Vlasovskikh * Joseph LaFreniere +* Daniel Tan From 8cae102d337300986004dbb55fd0c7802fa6402a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 4 Feb 2023 08:56:18 -0500 Subject: [PATCH 136/342] Remove some unused imports --- tests/test_hy2py.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py index d4c129501..c5743984b 100644 --- a/tests/test_hy2py.py +++ b/tests/test_hy2py.py @@ -1,8 +1,6 @@ import asyncio import itertools import math -import os -import platform import pytest From 17f62f2f01257ec6439df3ed7100df0455633ab9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 27 Jan 2023 14:00:53 -0500 Subject: [PATCH 137/342] In `Hy.REPL`, pull out `banner` --- hy/repl.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/hy/repl.py b/hy/repl.py index b34f52938..25a0ab3ea 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -431,16 +431,7 @@ def run(self): with filtered_hy_exceptions(), extend_linecache( self.cmdline_cache ), completion(Completer(namespace)): - self.interact( - "Hy {version} using " - "{py}({build}) {pyversion} on {os}".format( - version=hy.__version__, - py=platform.python_implementation(), - build=platform.python_build()[0], - pyversion=platform.python_version(), - os=platform.system(), - ) - ) + self.interact(self.banner()) finally: sys.ps1, sys.ps2, builtins.quit, builtins.exit, builtins.help = saved_values @@ -450,5 +441,14 @@ def run(self): return 0 + def banner(self): + return "Hy {version} using {py}({build}) {pyversion} on {os}".format( + version=hy.__version__, + py=platform.python_implementation(), + build=platform.python_build()[0], + pyversion=platform.python_version(), + os=platform.system(), + ) + REPL.__module__ = "hy" # Print as `hy.REPL` instead of `hy.repl.REPL`. From 2255d75a1ef1972f29d243dd236d45964e8f9722 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 28 Jan 2023 14:27:40 -0500 Subject: [PATCH 138/342] In `setup.py`, move up `requires` --- setup.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 175c23abb..981040e80 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,15 @@ #!/usr/bin/env python +# Set both `setup_requires` and `install_requires` with our +# dependencies, since we need to compile Hy files during setup. And +# put this as the first statement in the file so it's easy to parse +# out without executing the file. +requires = [ + "funcparserlib ~= 1.0", + "colorama", + 'astor>=0.8 ; python_version < "3.9"', +] + import os import fastentrypoints # Monkey-patches setuptools. @@ -31,15 +41,6 @@ def run(self): invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, ) - -# both setup_requires and install_requires -# since we need to compile .hy files during setup -requires = [ - "funcparserlib ~= 1.0", - "colorama", - 'astor>=0.8 ; python_version < "3.9"', -] - setup( name=PKG, version=__version__, From d112dc598f807eab59195e48adfbe5bd29da0cd6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 28 Jan 2023 16:32:39 -0500 Subject: [PATCH 139/342] Add an internal REPL option `allow_incomplete` --- hy/repl.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hy/repl.py b/hy/repl.py index 25a0ab3ea..abb16e92f 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -212,8 +212,9 @@ def __call__(self, source, filename="", symbol="single"): class HyCommandCompiler(codeop.CommandCompiler): - def __init__(self, *args, **kwargs): + def __init__(self, *args, allow_incomplete = True, **kwargs): self.compiler = HyCompile(*args, **kwargs) + self.allow_incomplete = allow_incomplete def __call__(self, *args, **kwargs): try: @@ -224,7 +225,8 @@ def __call__(self, *args, **kwargs): # this exception type is also a `SyntaxError`, so it will be caught # by `code.InteractiveConsole` base methods before it reaches our # `runsource`. - return None + if not self.allow_incomplete: + raise class REPL(code.InteractiveConsole): @@ -238,7 +240,7 @@ class REPL(code.InteractiveConsole): Note that as with :func:`code.interact`, changes to ``(locals)`` inside the REPL are not propagated back to the original scope.""" - def __init__(self, spy=False, output_fn=None, locals=None, filename=""): + def __init__(self, spy=False, output_fn=None, locals=None, filename="", allow_incomplete=True): # Create a proper module for this REPL so that we can obtain it easily # (e.g. using `importlib.import_module`). @@ -287,6 +289,7 @@ def __init__(self, spy=False, output_fn=None, locals=None, filename=""): ast_callback=self.ast_callback, hy_compiler=self.hy_compiler, cmdline_cache=self.cmdline_cache, + allow_incomplete=allow_incomplete, ) self.spy = spy From a15c960ea26b01691e61801af9d25c7cf42b42e8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 29 Jan 2023 15:03:59 -0500 Subject: [PATCH 140/342] Add some documentation comments --- docs/index.rst | 2 ++ docs/whyhy.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 58efff031..3255ce8dd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,8 @@ Hy is a Lisp dialect that's embedded in Python. Since Hy transforms its Lisp code into Python abstract syntax tree (AST) objects, you have the whole beautiful world of Python at your fingertips, in Lisp form. +.. + Changes to this paragraph should be mirrored on Hy's homepage. To install the latest release of Hy, just use the command ``pip3 install --user hy``. Then you can start an interactive read-eval-print loop (REPL) with the command ``hy``, or run a Hy program with ``hy myprogram.hy``. diff --git a/docs/whyhy.rst b/docs/whyhy.rst index 5fa857624..2daca7c6e 100644 --- a/docs/whyhy.rst +++ b/docs/whyhy.rst @@ -2,6 +2,8 @@ Why Hy? ======= +.. + Changes to this paragraph should be mirrored on Hy's homepage. Hy (or "Hylang" for long; named after the insect order Hymenoptera, since Paul Tagliamonte was studying swarm behavior when he created the language) is a multi-paradigm general-purpose programming language in From a3a5acce893a0d00c44ecda378d190d96c3b41bb Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 30 Jan 2023 16:28:06 -0500 Subject: [PATCH 141/342] Update some README links --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ab103a54..b1d4774cd 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ the command `hy`, or run a Hy program with `hy myprogram.hy`. Hy is tested on all released and currently maintained versions of CPython (on Linux and Windows), and on recent versions of PyPy and Pyodide. -* [Try Hy with a web console](https://hylang.github.io/hy-interpreter) +* [The Hy homepage](http://hylang.org) +* [Try Hy with a web console](http://hylang.org/try-hy) * [Why Hy?](http://docs.hylang.org/en/stable/whyhy.html) * [Tutorial](http://docs.hylang.org/en/stable/tutorial.html) From 829e9a580cc9ac58f96f58077cfefed2426637a2 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 31 Jan 2023 14:09:16 -0500 Subject: [PATCH 142/342] Don't create scripts named `hy3` etc. --- NEWS.rst | 2 ++ setup.py | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 850589392..9b9b5c847 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -21,6 +21,8 @@ Breaking Changes * New macro `do-mac`. * New macro `pragma` (although it doesn't do anything useful yet). * `hy.cmdline.HyREPL` is now `hy.REPL`. +* Redundant scripts named `hy3`, `hyc3`, and `hy2py3` are no longer + installed. Use `hy`, `hyc`, and `hy2py` instead. Bug Fixes ------------------------------ diff --git a/setup.py b/setup.py index 981040e80..ca7ffb0e3 100755 --- a/setup.py +++ b/setup.py @@ -50,11 +50,8 @@ def run(self): entry_points={ "console_scripts": [ "hy = hy.cmdline:hy_main", - "hy3 = hy.cmdline:hy_main", "hyc = hy.cmdline:hyc_main", - "hyc3 = hy.cmdline:hyc_main", - "hy2py = hy.cmdline:hy2py_main", - "hy2py3 = hy.cmdline:hy2py_main", + "hy2py = hy.cmdline:hy2py_main" ] }, packages=find_packages(exclude=["tests*"]), From 65d284dedbb07d36887b9cb12daf0aa448badfea Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 31 Jan 2023 14:54:13 -0500 Subject: [PATCH 143/342] Remove the support code for color --- hy/errors.py | 14 +--------- hy/models.py | 62 +++++++++++++------------------------------- hy/repl.py | 4 --- tests/test_models.py | 2 -- 4 files changed, 19 insertions(+), 63 deletions(-) diff --git a/hy/errors.py b/hy/errors.py index 6361550e1..6f52eeb99 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -6,13 +6,10 @@ from contextlib import contextmanager from functools import reduce -from colorama import Fore - from hy import _initialize_env_var from hy._compat import PYPY _hy_filter_internal_errors = _initialize_env_var("HY_FILTER_INTERNAL_ERRORS", True) -COLORED = _initialize_env_var("HY_COLORED_ERRORS", False) class HyError(Exception): @@ -135,20 +132,11 @@ def __str__(self): output[arrow_idx].rstrip("\n"), "-" * (self.arrow_offset - 1) ) - if COLORED: - output[msg_idx:] = [Fore.YELLOW + o + Fore.RESET for o in output[msg_idx:]] - if arrow_idx: - output[arrow_idx] = Fore.GREEN + output[arrow_idx] + Fore.RESET - for idx, line in enumerate(output[::msg_idx]): - if line.strip().startswith('File "{}", line'.format(self.filename)): - output[idx] = Fore.RED + line + Fore.RESET - # This resulting string will come after a ":" prompt, so # put it down a line. output.insert(0, "\n") - # Avoid "...expected str instance, ColoredString found" - return reduce(lambda x, y: x + y, output) + return "".join(output) class HyCompileError(HyInternalError): diff --git a/hy/models.py b/hy/models.py index ad3a93db8..abe9701d8 100644 --- a/hy/models.py +++ b/hy/models.py @@ -4,13 +4,10 @@ from itertools import groupby from math import isinf, isnan -from colorama import Fore - from hy import _initialize_env_var from hy.errors import HyWrapperError PRETTY = True -COLORED = _initialize_env_var("HY_COLORED_AST_OBJECTS", False) @contextmanager @@ -27,18 +24,6 @@ def pretty(pretty=True): PRETTY = old -class _ColoredModel: - """ - Mixin that provides a helper function for models that have color. - """ - - def _colored(self, text): - if COLORED: - return self.color + text + Fore.RESET - else: - return text - - class Object: "An abstract base class for Hy models, which represent forms." @@ -386,7 +371,7 @@ def __new__(cls, real, imag=0, *args, **kwargs): _wrappers[complex] = Complex -class Sequence(Object, tuple, _ColoredModel): +class Sequence(Object, tuple): """ An abstract base class for sequence-like forms. Sequence models can be operated on like tuples: you can iterate over them, index into them, and append them with ``+``, @@ -423,8 +408,6 @@ def __getitem__(self, item): return ret - color = None - def __repr__(self): return self._pretty_str() if PRETTY else super().__repr__() @@ -433,17 +416,12 @@ def __str__(self): def _pretty_str(self): with pretty(): - if self: - return self._colored( - "hy.models.{}{}\n {}{}".format( - self._colored(self.__class__.__name__), - self._colored("(["), - self._colored(",\n ").join(map(repr_indent, self)), - self._colored("])"), - ) - ) - else: - return self._colored(f"hy.models.{self.__class__.__name__}()") + return "hy.models.{}({})".format( + self.__class__.__name__, + "[\n {}]".format(",\n ".join(map(repr_indent, self))) + if self + else "" + ) class FComponent(Sequence): @@ -532,7 +510,7 @@ class List(Sequence): clauses. """ - color = Fore.CYAN + pass def recwrap(f): @@ -554,7 +532,7 @@ def lambda_to_return(l): _wrappers[list] = recwrap(List) -class Dict(Sequence, _ColoredModel): +class Dict(Sequence): """ Represents a literal :class:`dict`. ``keys``, ``values``, and ``items`` methods are provided, each returning a list, although this model type does none of the @@ -562,8 +540,6 @@ class Dict(Sequence, _ColoredModel): ``keys`` returns the last child whereas ``values`` and ``items`` ignores it. """ - color = Fore.GREEN - def _pretty_str(self): with pretty(): if self: @@ -571,21 +547,19 @@ def _pretty_str(self): for k, v in zip(self[::2], self[1::2]): k, v = repr_indent(k), repr_indent(v) pairs.append( - ("{0}{c}\n {1}\n " if "\n" in k + v else "{0}{c} {1}").format( - k, v, c=self._colored(",") + ("{0},\n {1}\n " if "\n" in k + v else "{0}, {1}").format( + k, v ) ) if len(self) % 2 == 1: pairs.append( - "{} {}\n".format(repr_indent(self[-1]), self._colored("# odd")) + "{} # odd\n".format(repr_indent(self[-1])) ) - return "{}\n {}{}".format( - self._colored("hy.models.Dict(["), - "{c}\n ".format(c=self._colored(",")).join(pairs), - self._colored("])"), + return "hy.models.Dict([\n {}])".format( + ",\n ".join(pairs), ) else: - return self._colored("hy.models.Dict()") + return "hy.models.Dict()" def keys(self): return list(self[0::2]) @@ -614,7 +588,7 @@ class Expression(Sequence): Represents a parenthesized Hy expression. """ - color = Fore.YELLOW + pass _wrappers[Expression] = recwrap(Expression) @@ -626,7 +600,7 @@ class Set(Sequence): and the order of elements. """ - color = Fore.RED + pass _wrappers[Set] = recwrap(Set) @@ -638,7 +612,7 @@ class Tuple(Sequence): Represents a literal :class:`tuple`. """ - color = Fore.BLUE + pass _wrappers[Tuple] = recwrap(Tuple) diff --git a/hy/repl.py b/hy/repl.py index abb16e92f..372a42fd0 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -411,8 +411,6 @@ def runsource(self, source, filename="", symbol="exec"): def run(self): "Start running the REPL. Return 0 when done." - import colorama - sentinel = [] saved_values = ( getattr(sys, "ps1", sentinel), @@ -428,8 +426,6 @@ def run(self): builtins.exit = HyQuitter("exit") builtins.help = HyHelper() - colorama.init() - namespace = self.locals with filtered_hy_exceptions(), extend_linecache( self.cmdline_cache diff --git a/tests/test_models.py b/tests/test_models.py index d1de2ce82..c000e8bd1 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -23,8 +23,6 @@ replace_hy_obj, ) -hy.models.COLORED = False - def test_symbol_or_keyword(): for x in ("foo", "foo-bar", "foo_bar", "✈é😂⁂"): From cb6c9f22c80a6f8153b5d72bba51d5694475306e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 31 Jan 2023 14:40:15 -0500 Subject: [PATCH 144/342] Remove `colorama` from `setup.py` --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index ca7ffb0e3..f68ea0647 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,6 @@ # out without executing the file. requires = [ "funcparserlib ~= 1.0", - "colorama", 'astor>=0.8 ; python_version < "3.9"', ] From e184a3a65fb45dc1747687a583154a172b3a4c4c Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 31 Jan 2023 15:01:31 -0500 Subject: [PATCH 145/342] Remove the documentation of color --- docs/api.rst | 5 ----- docs/env_var.rst | 10 ---------- docs/syntax.rst | 4 +--- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index f09e4897c..0b30f9634 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1311,11 +1311,6 @@ the following methods .. hy:autofunction:: hy.as-model -.. hy:data:: hy.errors.COLORED - - This variable is initially :py:data:`False`. If it's set to a true value, Hy - will color its error messages with ``colorama``. - .. _reader-macros: Reader Macros diff --git a/docs/env_var.rst b/docs/env_var.rst index aca679426..8053efbaf 100644 --- a/docs/env_var.rst +++ b/docs/env_var.rst @@ -11,16 +11,6 @@ set to anything else. (Default: nothing) Path to a file containing Hy source code to execute when starting the REPL. -.. envvar:: HY_COLORED_AST_OBJECTS - - (Default: false) Whether to use ANSI color when printing the Python - :func:`repr`\s of Hy :ref:`models `. - -.. envvar:: HY_COLORED_ERRORS - - (Default: false) Whether to use ANSI color when printing certain error - messages. - .. envvar:: HY_DEBUG (Default: false) Does something mysterious that's probably similar to diff --git a/docs/syntax.rst b/docs/syntax.rst index 70d9b2ced..6fd748221 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -59,9 +59,7 @@ The default representation of models (via :hy:func:`hy.repr`) uses quoting for readability, so ``(hy.models.Integer 5)`` is represented as ``'5``. Python representations (via :func:`repr`) use the constructors, and by default are pretty-printed; you can disable this globally by setting ``hy.models.PRETTY`` -to ``False``, or temporarily with the context manager ``hy.models.pretty``. You -can also color these Python representations with ``colorama`` by setting -``hy.models.COLORED`` to ``True``. +to ``False``, or temporarily with the context manager ``hy.models.pretty``. .. _hyobject: From 927a0769bf25e267c7c2bf77e87f242880c9ab73 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 31 Jan 2023 15:03:28 -0500 Subject: [PATCH 146/342] Update NEWS --- NEWS.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 9b9b5c847..9ebdd61eb 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,6 +3,11 @@ Unreleased ============================= +Removals +------------------------------ +* Coloring error messages and Python representations for models is no + longer supported. (Thus, Hy no longer depends on `colorama`.) + Breaking Changes ------------------------------ * Various warts have been smoothed over in the syntax of `'`, From 09f476345efcc32f0a53162cf62259b0f4c7bc10 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 1 Feb 2023 15:38:03 -0500 Subject: [PATCH 147/342] Get `hy.core.macros` docstrings back in the manual I probably broke this when I added support for reader macros to `sphinxcontrib-hydomain`. --- docs/api.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 0b30f9634..92878e338 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1255,9 +1255,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. 1.0. Currently, trying to use ``pragma`` is an error. .. hy:automodule:: hy.core.macros - :members: :macros: - :tags: Placeholder macros ~~~~~~~~~~~~~~~~~~ From 489ed58239b38fbe656aff8f0c6811d2c9e2760d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 1 Feb 2023 16:13:36 -0500 Subject: [PATCH 148/342] Deduplicate a test class --- tests/native_tests/match.hy | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/native_tests/match.hy b/tests/native_tests/match.hy index 1a5408147..a80c8f900 100644 --- a/tests/native_tests/match.hy +++ b/tests/native_tests/match.hy @@ -5,6 +5,10 @@ dataclasses [dataclass] hy.errors [HySyntaxError]) +(defclass [dataclass] Point [] + (#^int x) + (#^int y)) + (defn test-pattern-matching [] (assert (is (match 0 0 :if False False @@ -78,10 +82,6 @@ 0) 0)) - (defclass [dataclass] Point [] - (#^int x) - (#^int y)) - (assert (= 0 (match (Point 1 0) (Point 1 :y var) var))) (assert (is None (match (Point 0 0) (Point 1 :y var) var))) @@ -165,10 +165,6 @@ (assert (= [x y] [5 6]))) (assert (= [x y] [3 4]))) -(defclass [dataclass] Point [] - (#^int x) - (#^int y)) - (defn test-let-match-pattern [] (setv [x y] [1 2] p (Point 5 6)) From aeca8321426bdc865b75d55c39ebb2760c472f06 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 1 Feb 2023 16:14:15 -0500 Subject: [PATCH 149/342] Fix `match` patterns with a dotted constructor This bug was a regression since the last release, so it doesn't get a NEWS entry. --- hy/core/result_macros.py | 11 ++++++++--- tests/native_tests/match.hy | 6 ++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 9d64624c6..84de515b5 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1084,7 +1084,8 @@ def compile_with_expression(compiler, expr, root, args, body): | pexpr(keepsym("|"), many(_pattern)) | braces(many(LITERAL + _pattern), maybe(pvalue("unpack-mapping", SYM))) | pexpr( - notsym(".", "|", "unpack-mapping", "unpack-iterable"), + pexpr(keepsym("."), oneplus(SYM)) + | notsym(".", "|", "unpack-mapping", "unpack-iterable"), many(parse_if(lambda x: not isinstance(x, Keyword), _pattern)), many(KEYWORD + _pattern), ) @@ -1219,11 +1220,15 @@ def compile_pattern(compiler, pattern): ) ) elif isinstance(value, Expression): - root, args, kwargs = value + head, args, kwargs = value keywords, values = zip(*kwargs) if kwargs else ([], []) return asty.MatchClass( value, - cls=compiler.scope.access(asty.Name(root, id=mangle(root), ctx=ast.Load())), + cls=compiler.compile( + # `head` could be a symbol or a dotted form. + (head[:1] + head[1]).replace(head) + if type(head) is Expression + else head).expr, patterns=[compile_pattern(compiler, v) for v in args], kwd_attrs=[kwd.name for kwd in keywords], kwd_patterns=[compile_pattern(compiler, value) for value in values], diff --git a/tests/native_tests/match.hy b/tests/native_tests/match.hy index a80c8f900..cb1111066 100644 --- a/tests/native_tests/match.hy +++ b/tests/native_tests/match.hy @@ -130,6 +130,12 @@ (hy.eval '(match 1 1 :if True :as x x)))) +(defn test-dotted-constructor [] + ; https://github.com/hylang/hy/issues/2404 + (defclass C [Point] + (setv C Point)) + (assert (= (match (Point 1 2) (C.C 1 2) "ok") "ok"))) + (defn test-matching-side-effects [] (setv x 0) (defn foo [] From 8c25cb88a992b837174111c65399caee9cc91bdf Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 4 Feb 2023 10:07:13 -0500 Subject: [PATCH 150/342] Fix a README link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b1d4774cd..be86ea615 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Project * Code: https://github.com/hylang/hy * Documentation: * master, for use with the latest revision on GitHub: http://docs.hylang.org/en/master - * stable, for use with the latest release on PyPI: http://hylang.org/en/stable + * stable, for use with the latest release on PyPI: http://docs.hylang.org/en/stable * Bug reports: We have no bugs! Your bugs are your own! (https://github.com/hylang/hy/issues) * License: MIT (Expat) * [Hacking on Hy](http://docs.hylang.org/en/master/hacking.html) From aff785279ebe97ba700ac3cb9ed5553f999781bc Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 5 Feb 2023 07:53:01 -0500 Subject: [PATCH 151/342] Remove unused linters from `requirements-dev.txt` --- requirements-dev.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6aaaf00e6..caad94f2e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,10 +1,5 @@ pytest >= 6 -# autoformatting -black==22.3.0 -isort==5.10.1 -pre_commit==2.17.0 - # documentation Pygments >= 2 Sphinx == 5.0.2 From a25b1795452584c22116c288349aaf2ae64fe2e9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 5 Feb 2023 07:59:39 -0500 Subject: [PATCH 152/342] Update NEWS for release --- NEWS.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 9ebdd61eb..f0097b247 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,6 +1,6 @@ .. default-role:: code -Unreleased +0.26.0 (released 2023-02-08) ============================= Removals @@ -42,10 +42,11 @@ New Features ------------------------------ * Pyodide is now officially supported. * `.`, `..`, etc. are now usable as ordinary symbols (with the - remaining special rule that `...` compiles to `Ellipsis`) + remaining special rule that `...` compiles to `Ellipsis`). * On Pythons ≥ 3.7, Hy modules can now be imported from ZIP archives in the same way as Python modules, via `zipimport`_. -* `hy2py` now supports directory input, and will recursively convert hy source code into python source code. +* `hy2py` has a new command-line option `--output`. +* `hy2py` can now operate recursively on a directory. .. _zipimport: https://docs.python.org/3.11/library/zipimport.html From 6f1e7dbc0112fafe48ae9c33e0955cce8bae36f8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 10 Feb 2023 13:41:43 -0500 Subject: [PATCH 153/342] Don't install Emscripten on GitHub Actions This should no longer be necessary with the latest `pyodide-build`. --- .github/workflows/tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0cb5ff483..cad4cd899 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,7 +21,6 @@ jobs: name: ${{ format('{0}{1}', matrix.name-prefix, matrix.python) }} runs-on: ${{ matrix.os }} env: - EMSCRIPTEN_VERSION: 3.1.27 TERM: xterm-256color # This is needed to avoid a terminfo-related crash when # testing PyPy. @@ -36,11 +35,6 @@ jobs: python-version: ${{ matrix.python }} - if: ${{ matrix.python == 'pyodide' }} uses: actions/setup-node@v3 - - if: ${{ matrix.python == 'pyodide' }} - uses: mymindstorm/setup-emsdk@v11 - with: - version: ${{ env.EMSCRIPTEN_VERSION }} - actions-cache-folder: emsdk-cache - name: Install shell: bash run: | From 1cca21f7bc854b6d64d2f6335b8358311da110b1 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 10 Feb 2023 14:54:14 -0500 Subject: [PATCH 154/342] Don't set the package version to "unknown" This fixes a failure to install Hy on Pyodide on GitHub Actions. --- NEWS.rst | 8 ++++++++ setup.py | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index f0097b247..98a204df8 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,5 +1,13 @@ .. default-role:: code +Unreleased +============================= + +Bug Fixes +------------------------------ +* Fixed an installation failure in some situations when version lookup + fails. + 0.26.0 (released 2023-02-08) ============================= diff --git a/setup.py b/setup.py index f68ea0647..5707ce21b 100755 --- a/setup.py +++ b/setup.py @@ -42,7 +42,11 @@ def run(self): setup( name=PKG, - version=__version__, + version=( + None + if __version__ == "unknown" + else __version__ + ), setup_requires=["wheel"] + requires, install_requires=requires, python_requires=">= 3.7, < 3.12", From 3ef915e2435f4057adaf9eb8e80134c33e374994 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 12 Feb 2023 11:23:57 -0500 Subject: [PATCH 155/342] Correct the `do-while` example --- docs/tutorial.rst | 9 +++++---- docs/whyhy.rst | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index b38f0271e..b20e69b98 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -304,10 +304,11 @@ Our macro ``m`` has an especially simple return value, an integer, which at compile-time is converted to an integer literal. In general, macros can return arbitrary Hy forms to be executed as code. There are several helper macros that make it easy to construct forms programmatically, such as :hy:func:`quote` -(``'``), :hy:func:`quasiquote` (`````), :hy:func:`unquote` (``~``), and -:hy:func:`defmacro! `. The previous chapter has -:ref:`a simple example ` of using ````` and ``~`` to define a new -control construct ``do-while``. +(``'``), :hy:func:`quasiquote` (`````), :hy:func:`unquote` (``~``), +:hy:func:`unquote-splice` (``~@``), and :hy:func:`defmacro! +`. The previous chapter has :ref:`a simple example +` of using ````` and ``~@`` to define a new control construct +``do-while``. What if you want to use a macro that's defined in a different module? ``import`` won't help, because it merely translates to a Python ``import`` diff --git a/docs/whyhy.rst b/docs/whyhy.rst index 2daca7c6e..037844b76 100644 --- a/docs/whyhy.rst +++ b/docs/whyhy.rst @@ -42,9 +42,9 @@ for as long as the condition is true, but at least once. (defmacro do-while [condition #* body] `(do - ~body + ~@body (while ~condition - ~body))) + ~@body))) (setv x 0) (do-while x From b894497718d396b16c4a6471daa9b5d449e34d86 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 15 Feb 2023 15:43:29 -0500 Subject: [PATCH 156/342] Dedent the input to `py`, as for `pys` --- NEWS.rst | 4 ++++ docs/api.rst | 10 +++++----- hy/core/result_macros.py | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 98a204df8..d4ce4557e 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,6 +3,10 @@ Unreleased ============================= +Breaking Changes +------------------------------ +* The `py` macro now dedents the input like `pys`. + Bug Fixes ------------------------------ * Fixed an installation failure in some situations when version lookup diff --git a/docs/api.rst b/docs/api.rst index 92878e338..390919c6b 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -762,6 +762,11 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. code that's only defined at run-time, try the standard Python function :func:`eval`. + The code string is dedented with :func:`textwrap.dedent` before parsing, + which allows you to indent the code to match the surrounding Hy code when + Python would otherwise forbid this, but beware that significant leading + whitespace in embedded string literals will be removed. + Python code need not syntactically round-trip if you use ``hy2py`` on a Hy program that uses ``py`` or ``pys``. For example, comments will be removed. @@ -777,11 +782,6 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (pys "myvar = 5") (print "myvar is" myvar) - The code string is dedented with :func:`textwrap.dedent` before parsing, - which allows you to indent the code to match the surrounding Hy code when - Python would otherwise forbid this, but beware that significant leading - whitespace in embedded string literals will be removed. - .. hy:macro:: (quasiquote [form]) ``quasiquote`` allows you to quote a form, but also selectively evaluate diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 84de515b5..b8986f7cc 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -131,7 +131,7 @@ def compile_inline_python(compiler, expr, root, code): try: o = asty.parse( expr, - textwrap.dedent(code) if exec_mode else code, + textwrap.dedent(code), compiler.filename, "exec" if exec_mode else "eval", ).body From 7a2dc42b2542996f13661c95ef0a8d79c42cc41b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 15 Feb 2023 15:43:36 -0500 Subject: [PATCH 157/342] Expand tests of inline Python --- tests/compilers/test_ast.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 58011dde8..a7bb639b5 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -1,6 +1,7 @@ # fmt: off import ast +from textwrap import dedent import pytest @@ -618,11 +619,41 @@ def test_futures_imports(): assert hy_ast.body[0].module == "__future__" -def test_inline_python(): - can_compile('(py "1 + 1")') +def test_py(): + def py(x): assert ( + ast.dump(can_compile(f'(py "{x}")')) == + ast.dump(ast.parse(dedent(x)))) + + py("1 + 1") + # https://github.com/hylang/hy/issues/2406 + py(" 1 + 1 ") + cant_compile('(py "1 +")') - can_compile('(pys "if 1:\n 2")') + cant_compile('(py "if 1:\n 2")') + + +def test_pys(): + def pys(x): assert ( + ast.dump(can_compile(f'(pys "{x}")')) == + ast.dump(ast.parse(dedent(x)))) + + pys("") + pys("1 + 1") + pys("if 1:\n 2") + pys("if 1: 2") + pys(" if 1: 2 ") + pys(''' + if 1: + 2 + elif 3: + 4''') + cant_compile('(pys "if 1\n 2")') + cant_compile('''(pys " + if 1: + 2 + elif 3: + 4")''') def test_models_accessible(): From 74cf4c6ab9bec298c3b88f1570228989db9e45b7 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 15 Feb 2023 15:47:47 -0500 Subject: [PATCH 158/342] Remove an unneeded anchor from the manual --- docs/api.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 390919c6b..361923edb 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -770,9 +770,6 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Python code need not syntactically round-trip if you use ``hy2py`` on a Hy program that uses ``py`` or ``pys``. For example, comments will be removed. - - .. _pys-specialform: - .. hy:macro:: (pys [string]) As :hy:func:`py `, but the code can consist of zero or more statements, From a36455b7f5122d01a7c4923371d544ae92a93b32 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 15 Feb 2023 15:54:57 -0500 Subject: [PATCH 159/342] Remove some redundant docstrings for tests --- tests/compilers/test_ast.py | 48 ------------------------------------- 1 file changed, 48 deletions(-) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index a7bb639b5..eb2ff0801 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -71,30 +71,25 @@ def test_dot_unpacking(): def test_ast_bad_if(): - "Make sure AST can't compile invalid if" cant_compile("(if)") cant_compile("(if foobar)") cant_compile("(if 1 2 3 4 5)") def test_ast_valid_if(): - "Make sure AST can compile valid if" can_compile("(if foo bar baz)") def test_ast_bad_while(): - "Make sure AST can't compile invalid while" cant_compile("(while)") def test_ast_good_do(): - "Make sure AST can compile valid do" can_compile("(do)") can_compile("(do 1)") def test_ast_good_raise(): - "Make sure AST can compile valid raise" can_compile("(raise)") can_compile("(raise Exception)") can_compile("(raise e)") @@ -105,12 +100,10 @@ def test_ast_raise_from(): def test_ast_bad_raise(): - "Make sure AST can't compile invalid raise" cant_compile("(raise Exception Exception)") def test_ast_good_try(): - "Make sure AST can compile valid try" can_compile("(try 1 (except []) (else 1))") can_compile("(try 1 (finally 1))") can_compile("(try 1 (except []) (finally 1))") @@ -122,7 +115,6 @@ def test_ast_good_try(): def test_ast_bad_try(): - "Make sure AST can't compile invalid try" cant_compile("(try)") cant_compile("(try 1)") cant_compile("(try 1 bla)") @@ -138,7 +130,6 @@ def test_ast_bad_try(): def test_ast_good_except(): - "Make sure AST can compile valid except" can_compile("(try 1 (except []))") can_compile("(try 1 (except [Foobar]))") can_compile("(try 1 (except [[]]))") @@ -148,7 +139,6 @@ def test_ast_good_except(): def test_ast_bad_except(): - "Make sure AST can't compile invalid except" cant_compile("(except 1)") cant_compile("(try 1 (except))") cant_compile("(try 1 (except 1))") @@ -158,8 +148,6 @@ def test_ast_bad_except(): def test_ast_good_assert(): - """Make sure AST can compile valid asserts. Asserts may or may not - include a label.""" can_compile("(assert 1)") can_compile('(assert 1 "Assert label")') can_compile('(assert 1 (+ "spam " "eggs"))') @@ -169,38 +157,32 @@ def test_ast_good_assert(): def test_ast_bad_assert(): - "Make sure AST can't compile invalid assert" cant_compile("(assert)") cant_compile("(assert 1 2 3)") cant_compile("(assert 1 [1 2] 3)") def test_ast_good_global(): - "Make sure AST can compile valid global" can_compile("(global a)") can_compile("(global foo bar)") def test_ast_bad_global(): - "Make sure AST can't compile invalid global" cant_compile("(global)") cant_compile("(global (foo))") def test_ast_good_nonlocal(): - "Make sure AST can compile valid nonlocal" can_compile("(do (setv a 0) (nonlocal a))") can_compile("(do (setv foo 0 bar 0) (nonlocal foo bar))") def test_ast_bad_nonlocal(): - "Make sure AST can't compile invalid nonlocal" cant_compile("(nonlocal)") cant_compile("(nonlocal (foo))") def test_ast_good_defclass(): - "Make sure AST can compile valid defclass" can_compile("(defclass a)") can_compile("(defclass a [])") can_compile("(defclass a [] None 42)") @@ -209,13 +191,11 @@ def test_ast_good_defclass(): def test_ast_good_defclass_with_metaclass(): - "Make sure AST can compile valid defclass with keywords" can_compile("(defclass a [:metaclass b])") can_compile("(defclass a [:b c])") def test_ast_bad_defclass(): - "Make sure AST can't compile invalid defclass" cant_compile("(defclass)") cant_compile("(defclass a None)") cant_compile("(defclass a None None)") @@ -226,13 +206,11 @@ def test_ast_bad_defclass(): def test_ast_good_lambda(): - "Make sure AST can compile valid lambda" can_compile("(fn [])") can_compile("(fn [] 1)") def test_ast_bad_lambda(): - "Make sure AST can't compile invalid lambda" cant_compile("(fn)") cant_compile("(fn ())") cant_compile("(fn () 1)") @@ -241,12 +219,10 @@ def test_ast_bad_lambda(): def test_ast_good_yield(): - "Make sure AST can compile valid yield" can_compile("(yield 1)") def test_ast_bad_yield(): - "Make sure AST can't compile invalid yield" cant_compile("(yield 1 2)") @@ -259,12 +235,10 @@ def test_ast_import_mangle_dotted(): def test_ast_good_import_from(): - "Make sure AST can compile valid selective import" can_compile("(import x [y])") def test_ast_require(): - "Make sure AST respects (require) syntax" can_compile("(require tests.resources.tlib)") can_compile("(require tests.resources.tlib [qplah parald])") can_compile("(require tests.resources.tlib *)") @@ -296,18 +270,15 @@ def test_ast_multi_require(): def test_ast_good_get(): - "Make sure AST can compile valid get" can_compile("(get x y)") def test_ast_bad_get(): - "Make sure AST can't compile invalid get" cant_compile("(get)") cant_compile("(get 1)") def test_ast_good_cut(): - "Make sure AST can compile valid cut" can_compile("(cut x)") can_compile("(cut x y)") can_compile("(cut x y z)") @@ -315,26 +286,22 @@ def test_ast_good_cut(): def test_ast_bad_cut(): - "Make sure AST can't compile invalid cut" cant_compile("(cut)") cant_compile("(cut 1 2 3 4 5)") def test_ast_bad_with(): - "Make sure AST can't compile invalid with" cant_compile("(with)") cant_compile("(with [])") cant_compile("(with [] (pass))") def test_ast_valid_while(): - "Make sure AST can't compile invalid while" can_compile("(while foo bar)") can_compile("(while foo bar (else baz))") def test_ast_valid_for(): - "Make sure AST can compile valid for" can_compile("(for [a 2] (print a))") @@ -365,7 +332,6 @@ def test_ast_expression_basics(): def test_ast_anon_fns_basics(): - """Ensure anon fns work.""" code = can_compile("(fn [x] (* x x))").body[0].value assert type(code) == ast.Lambda code = can_compile('(fn [x] (print "multiform") (* x x))').body[0] @@ -375,7 +341,6 @@ def test_ast_anon_fns_basics(): def test_ast_lambda_lists(): - """Ensure the compiler chokes on invalid lambda-lists""" cant_compile("(fn [[a b c]] a)") cant_compile("(fn [[1 2]] (list 1 2))") @@ -387,20 +352,17 @@ def test_ast_print(): def test_ast_tuple(): - """Ensure tuples work.""" code = can_compile("#(1 2 3)").body[0].value assert type(code) == ast.Tuple def test_lambda_list_keywords_rest(): - """Ensure we can compile functions with lambda list keywords.""" can_compile("(fn [x #* xs] (print xs))") cant_compile("(fn [x #* xs #* ys] (print xs))") can_compile("(fn [[a None] #* xs] (print xs))") def test_lambda_list_keywords_kwargs(): - """Ensure we can compile functions with #** kwargs.""" can_compile("(fn [x #** kw] (list x kw))") cant_compile("(fn [x #** xs #** ys] (list x xs ys))") can_compile("(fn [[x None] #** kw] (list x kw))") @@ -416,21 +378,18 @@ def test_lambda_list_keywords_kwonly(): def test_lambda_list_keywords_mixed(): - """Ensure we can mix them up.""" can_compile("(fn [x #* xs #** kw] (list x xs kw))") cant_compile('(fn [x #* xs &fasfkey {bar "baz"}])') can_compile("(fn [x #* xs kwoxs #** kwxs]" " (list x xs kwxs kwoxs))") def test_missing_keyword_argument_value(): - """Ensure the compiler chokes on missing keyword argument values.""" with pytest.raises(HyLanguageError) as excinfo: can_compile("((fn [x] x) :x)") assert excinfo.value.msg == "Keyword argument :x needs a value." def test_ast_unicode_strings(): - """Ensure we handle unicode strings correctly""" def _compile_string(s): hy_s = hy.models.String(s) @@ -517,7 +476,6 @@ def test_for_compile_error(): def test_attribute_access(): - """Ensure attribute access compiles correctly""" can_compile("(. foo bar baz)") can_compile("(. foo [bar] baz)") can_compile("(. foo bar [baz] [0] quux [frob])") @@ -536,12 +494,10 @@ def test_misplaced_dots(): def test_bad_setv(): - """Ensure setv handles error cases""" cant_compile("(setv (a b) [1 2])") def test_defn(): - """Ensure that defn works correctly in various corner cases""" cant_compile('(defn "hy" [] 1)') cant_compile("(defn :hy [] 1)") can_compile("(defn &hy [] 1)") @@ -587,23 +543,19 @@ def test_lots_of_comment_lines(): def test_compiler_macro_tag_try(): - """Check that try forms within defmacro are compiled correctly""" # https://github.com/hylang/hy/issues/1350 can_compile("(defmacro foo [] (try None (except [] None)) `())") def test_ast_good_yield_from(): - "Make sure AST can compile valid yield-from" can_compile("(yield-from [1 2])") def test_ast_bad_yield_from(): - "Make sure AST can't compile invalid yield-from" cant_compile("(yield-from)") def test_eval_generator_with_return(): - """Ensure generators with a return statement works.""" can_eval("(fn [] (yield 1) (yield 2) (return))") From 07a1d6119a681d6b7ce7a512134c0f42ec22ec76 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 15 Feb 2023 16:02:34 -0500 Subject: [PATCH 160/342] Allow `(global)` and `(nonlocal)` --- NEWS.rst | 5 +++++ docs/api.rst | 8 ++++---- hy/core/result_macros.py | 5 ++++- tests/compilers/test_ast.py | 4 ++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index d4ce4557e..a7fcff32b 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -12,6 +12,11 @@ Bug Fixes * Fixed an installation failure in some situations when version lookup fails. +New Features +------------------------------ +* `nonlocal` and `global` can now be called with no arguments, in which + case they're no-ops. + 0.26.0 (released 2023-02-08) ============================= diff --git a/docs/api.rst b/docs/api.rst index 361923edb..5843ed1b8 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -391,12 +391,12 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (assert (= 1 2) "one should equal two") ; AssertionError: one should equal two -.. hy:macro:: (global [sym #* syms]) +.. hy:macro:: (global [#* syms]) ``global`` compiles to a :py:keyword:`global` statement, which declares one or more names as referring to global (i.e., module-level) variables. The - arguments are symbols; at least one is required. The return value is always - ``None``. :: + arguments are symbols; with no arguments, ``global`` has no effect. The + return value is always ``None``. :: (setv a 1 b 10) (print a b) ; => 1 10 @@ -741,7 +741,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (del foo (get mydict "mykey") myobj.myattr) -.. hy:macro:: (nonlocal [sym #* syms]) +.. hy:macro:: (nonlocal [#* syms]) As :hy:func:`global`, but the result is a :py:keyword:`nonlocal` statement. diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index b8986f7cc..c05dc1150 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -486,8 +486,11 @@ def compile_assign( return result -@pattern_macro(["global", "nonlocal"], [oneplus(SYM)]) +@pattern_macro(["global", "nonlocal"], [many(SYM)]) def compile_global_or_nonlocal(compiler, expr, root, syms): + if not syms: + return asty.Pass(expr) + node = asty.Global if root == "global" else asty.Nonlocal ret = node(expr, names=[mangle(s) for s in syms]) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index eb2ff0801..667c94ad1 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -163,22 +163,22 @@ def test_ast_bad_assert(): def test_ast_good_global(): + can_compile("(global)") can_compile("(global a)") can_compile("(global foo bar)") def test_ast_bad_global(): - cant_compile("(global)") cant_compile("(global (foo))") def test_ast_good_nonlocal(): + can_compile("(nonlocal)") can_compile("(do (setv a 0) (nonlocal a))") can_compile("(do (setv foo 0 bar 0) (nonlocal foo bar))") def test_ast_bad_nonlocal(): - cant_compile("(nonlocal)") cant_compile("(nonlocal (foo))") From 0bd5505b1eba0cc20ae529a4987a0899f5a1735e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 15 Feb 2023 16:08:44 -0500 Subject: [PATCH 161/342] Rename `__macros__` and `__reader_macros__` To `_hy_macros` and `_hy_reader_macros`, respectively. There's no NEWS item for this change because these variables are still (for the time being) undocumented and solely for internal use. I don't know why we mentioned them in NEWS previously. --- hy/compiler.py | 4 +-- hy/completer.py | 6 ++-- hy/core/macros.hy | 12 ++++---- hy/macros.py | 44 ++++++++++++++--------------- hy/repl.py | 4 +-- hy/reserved.hy | 4 +-- tests/native_tests/import.hy | 40 +++++++++++++------------- tests/native_tests/reader_macros.hy | 2 +- 8 files changed, 58 insertions(+), 58 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 3775051e0..70cf5b795 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -359,8 +359,8 @@ def __init__(self, module, filename=None, source=None): # Hy expects this to be present, so we prep the module for Hy # compilation. - self.module.__dict__.setdefault("__macros__", {}) - self.module.__dict__.setdefault("__reader_macros__", {}) + self.module.__dict__.setdefault("_hy_macros", {}) + self.module.__dict__.setdefault("_hy_reader_macros", {}) self.scope = ScopeGlobal(self) diff --git a/hy/completer.py b/hy/completer.py index adeebfa48..5ede05bd5 100644 --- a/hy/completer.py +++ b/hy/completer.py @@ -45,10 +45,10 @@ def __init__(self, namespace={}): self.namespace = namespace self.path = [builtins.__dict__, namespace] - namespace.setdefault("__macros__", {}) - namespace.setdefault("__reader_macros__", {}) + namespace.setdefault("_hy_macros", {}) + namespace.setdefault("_hy_reader_macros", {}) - self.path.append(namespace["__macros__"]) + self.path.append(namespace["_hy_macros"]) def attr_matches(self, text): # Borrowed from IPython's completer diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 0ab4128e0..07e06678e 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -115,7 +115,7 @@ ~@body))) (eval-when-compile (setv (get hy.&reader.reader-table ~dispatch-key) - (get __reader_macros__ ~dispatch-key))))) + (get _hy_reader_macros ~dispatch-key))))) (defmacro doc [symbol] @@ -129,10 +129,10 @@ (setv mangled (hy.mangle symbol)) (setv builtins (hy.gensym "builtins")) `(do (import builtins :as ~builtins) - (help (or (.get __macros__ ~mangled) - (.get __reader_macros__ ~mangled) - (.get (. ~builtins __macros__) ~mangled) - (.get (. ~builtins __reader_macros__) ~mangled) + (help (or (.get _hy_macros ~mangled) + (.get _hy_reader_macros ~mangled) + (.get (. ~builtins _hy_macros) ~mangled) + (.get (. ~builtins _hy_reader_macros) ~mangled) (raise (NameError f"macro {~symbol !r} is not defined")))))) @@ -192,4 +192,4 @@ (let [sym (hy.gensym)] `(eval-and-compile (for [~sym ~(lfor name names (hy.mangle name))] - (when (in ~sym __macros__) (del (get __macros__ ~sym))))))) + (when (in ~sym _hy_macros) (del (get _hy_macros ~sym))))))) diff --git a/hy/macros.py b/hy/macros.py index d04202495..c23dcbc82 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -33,7 +33,7 @@ def macro(name): def reader_macro(name, fn): fn = rename_function(fn, name) - inspect.getmodule(fn).__dict__.setdefault("__reader_macros__", {})[name] = fn + inspect.getmodule(fn).__dict__.setdefault("_hy_reader_macros", {})[name] = fn def pattern_macro(names, pattern, shadow=None): @@ -87,8 +87,8 @@ def install_macro(name, fn, module_of): name = mangle(name) fn = rename_function(fn, name) calling_module = inspect.getmodule(module_of) - macros_obj = calling_module.__dict__.setdefault("__macros__", {}) - if name in getattr(builtins, "__macros__", {}): + macros_obj = calling_module.__dict__.setdefault("_hy_macros", {}) + if name in getattr(builtins, "_hy_macros", {}): warnings.warn( ( f"{name} already refers to: `{name}` in module: `builtins`," @@ -179,15 +179,15 @@ def require_reader(source_module, target_module, assignments): if not inspect.ismodule(source_module): source_module = import_module_from_string(source_module, target_module) - source_macros = source_module.__dict__.setdefault("__reader_macros__", {}) - target_macros = target_namespace.setdefault("__reader_macros__", {}) + source_macros = source_module.__dict__.setdefault("_hy_reader_macros", {}) + target_macros = target_namespace.setdefault("_hy_reader_macros", {}) assignments = ( source_macros.keys() if assignments == "ALL" else map(mangle, assignments) ) for name in assignments: - if name in source_module.__reader_macros__: + if name in source_module._hy_reader_macros: target_macros[name] = source_macros[name] else: raise HyRequireError(f"Could not require name {name} from {source_module}") @@ -198,12 +198,12 @@ def require_reader(source_module, target_module, assignments): def enable_readers(module, reader, names): _, namespace = derive_target_module(module, inspect.stack()[1][0]) names = ( - namespace["__reader_macros__"].keys() if names == "ALL" else map(mangle, names) + namespace["_hy_reader_macros"].keys() if names == "ALL" else map(mangle, names) ) for name in names: - if name not in namespace["__reader_macros__"]: + if name not in namespace["_hy_reader_macros"]: raise NameError(f"reader {name} is not defined") - reader.reader_table[name] = namespace["__reader_macros__"][name] + reader.reader_table[name] = namespace["_hy_reader_macros"][name] def require(source_module, target_module, assignments, prefix=""): @@ -241,14 +241,14 @@ def require(source_module, target_module, assignments, prefix=""): if not inspect.ismodule(source_module): source_module = import_module_from_string(source_module, target_module) - source_macros = source_module.__dict__.setdefault("__macros__", {}) + source_macros = source_module.__dict__.setdefault("_hy_macros", {}) source_exports = getattr( source_module, "_hy_export_macros", [k for k in source_macros.keys() if not k.startswith("_")], ) - if not source_module.__macros__: + if not source_module._hy_macros: if assignments in ("ALL", "EXPORTS"): return False for name, alias in assignments: @@ -267,7 +267,7 @@ def require(source_module, target_module, assignments, prefix=""): ) return True - target_macros = target_namespace.setdefault("__macros__", {}) + target_macros = target_namespace.setdefault("_hy_macros", {}) if prefix: prefix += "." @@ -287,7 +287,7 @@ def require(source_module, target_module, assignments, prefix=""): if unmangle(alias).startswith("#") else prefix + alias ) - if _name in source_module.__macros__: + if _name in source_module._hy_macros: target_macros[alias] = source_macros[_name] else: raise HyRequireError( @@ -303,19 +303,19 @@ def load_macros(module): It is an error to call this on any module in `hy.core`. """ builtin_macros = EXTRA_MACROS - module.__macros__ = {} - module.__reader_macros__ = {} + module._hy_macros = {} + module._hy_reader_macros = {} for builtin_mod_name in builtin_macros: builtin_mod = importlib.import_module(builtin_mod_name) # This may overwrite macros in the module. - if hasattr(builtin_mod, "__macros__"): - module.__macros__.update(getattr(builtin_mod, "__macros__", {})) + if hasattr(builtin_mod, "_hy_macros"): + module._hy_macros.update(getattr(builtin_mod, "_hy_macros", {})) - if hasattr(builtin_mod, "__reader_macros__"): - module.__reader_macros__.update( - getattr(builtin_mod, "__reader_macros__", {}) + if hasattr(builtin_mod, "_hy_reader_macros"): + module._hy_reader_macros.update( + getattr(builtin_mod, "_hy_reader_macros", {}) ) @@ -406,9 +406,9 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): # Choose the first namespace with the macro. m = next( ( - mod.__macros__[fn] + mod._hy_macros[fn] for mod in expr_modules - if fn in getattr(mod, "__macros__", ()) + if fn in getattr(mod, "_hy_macros", ()) ), None, ) diff --git a/hy/repl.py b/hy/repl.py index 372a42fd0..1b5310bda 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -121,9 +121,9 @@ def __init__( super().__init__() - if hasattr(self.module, "__reader_macros__"): + if hasattr(self.module, "_hy_reader_macros"): enable_readers( - self.module, self.reader, self.module.__reader_macros__.keys() + self.module, self.reader, self.module._hy_reader_macros.keys() ) self.flags |= hy_ast_compile_flags diff --git a/hy/reserved.hy b/hy/reserved.hy index f20cae24a..6a6d4d4ff 100644 --- a/hy/reserved.hy +++ b/hy/reserved.hy @@ -7,8 +7,8 @@ (defn macros [] "Return a frozenset of Hy's core macro names." (frozenset (map hy.unmangle (+ - (list (.keys hy.core.result_macros.__macros__)) - (list (.keys hy.core.macros.__macros__)))))) + (list (.keys hy.core.result_macros._hy_macros)) + (list (.keys hy.core.macros._hy_macros)))))) (defn names [] "Return a frozenset of reserved symbol names. diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy index 8821fd77c..817c47f63 100644 --- a/tests/native_tests/import.hy +++ b/tests/native_tests/import.hy @@ -99,9 +99,9 @@ (assert (= (hyx_XairplaneX "foolish") "plane foolish")) (require tests.resources [tlib macros :as m exports-none]) - (assert (in "tlib.qplah" __macros__)) - (assert (in (hy.mangle "m.test-macro") __macros__)) - (assert (in (hy.mangle "exports-none.cinco") __macros__)) + (assert (in "tlib.qplah" _hy_macros)) + (assert (in (hy.mangle "m.test-macro") _hy_macros)) + (assert (in (hy.mangle "exports-none.cinco") _hy_macros)) (require os [path]) (with [(pytest.raises hy.errors.HyRequireError)] (hy.eval '(require tests.resources [does-not-exist]))) @@ -126,13 +126,13 @@ (defn test-relative-require [] (require ..resources.macros [test-macro]) - (assert (in "test_macro" __macros__)) + (assert (in "test_macro" _hy_macros)) (require .beside [xyzzy]) - (assert (in "xyzzy" __macros__)) + (assert (in "xyzzy" _hy_macros)) (require . [beside :as b]) - (assert (in "b.xyzzy" __macros__))) + (assert (in "b.xyzzy" _hy_macros))) (defn test-macro-namespace-resolution [] @@ -148,9 +148,9 @@ in expansions." (print "this is the local version of `nonlocal-test-macro`!")) ;; Was the above macro created properly? - (assert (in "nonlocal_test_macro" __macros__)) + (assert (in "nonlocal_test_macro" _hy_macros)) - (setv nonlocal-test-macro (get __macros__ "nonlocal_test_macro")) + (setv nonlocal-test-macro (get _hy_macros "nonlocal_test_macro")) (require tests.resources.macro-with-require *) @@ -171,7 +171,7 @@ in expansions." (defn test-requires-pollutes-core [] ;; https://github.com/hylang/hy/issues/1978 - ;; Macros loaded from an external module should not pollute `__macros__` + ;; Macros loaded from an external module should not pollute `_hy_macros` ;; with macros from core. (setv pyc-file (importlib.util.cache-from-source @@ -186,7 +186,7 @@ in expansions." (.clear sys.path_importer_cache) (when (in "tests.resources.macros" sys.modules) (del (get sys.modules "tests.resources.macros")) - (__macros__.clear))) + (_hy_macros.clear))) ;; Ensure that bytecode isn't present when we require this module. (assert (not (os.path.isfile pyc-file))) @@ -194,8 +194,8 @@ in expansions." (defn require-macros [] (require tests.resources.macros :as m) - (assert (in (hy.mangle "m.test-macro") __macros__)) - (for [macro-name __macros__] + (assert (in (hy.mangle "m.test-macro") _hy_macros)) + (for [macro-name _hy_macros] (assert (not (and (in "with" macro-name) (!= "with" macro-name)))))) @@ -209,7 +209,7 @@ in expansions." ;; Reload the module and clear the local macro context. (.clear sys.path_importer_cache) (del (get sys.modules "tests.resources.macros")) - (.clear __macros__) + (.clear _hy_macros) (require-macros)) @@ -220,7 +220,7 @@ in expansions." work without having to `require` the module's macro dependencies (due to [minimal] macro namespace resolution). - In doing so we also confirm that a module's `__macros__` attribute is correctly + In doing so we also confirm that a module's `_hy_macros` attribute is correctly loaded and used. Additionally, we confirm that `require` statements are executed via loaded bytecode. @@ -238,7 +238,7 @@ in expansions." (.clear sys.path_importer_cache) (when (in "tests.resources.macro_with_require" sys.modules) (del (get sys.modules "tests.resources.macro_with_require")) - (__macros__.clear))) + (_hy_macros.clear))) ;; Ensure that bytecode isn't present when we require this module. (assert (not (os.path.isfile pyc-file))) @@ -248,18 +248,18 @@ in expansions." [test-module-macro]) ;; Make sure that `require` didn't add any of its `require`s - (assert (not (in (hy.mangle "nonlocal-test-macro") __macros__))) + (assert (not (in (hy.mangle "nonlocal-test-macro") _hy_macros))) ;; and that it didn't add its tags. - (assert (not (in (hy.mangle "#test-module-tag") __macros__))) + (assert (not (in (hy.mangle "#test-module-tag") _hy_macros))) ;; Now, require everything. (require tests.resources.macro-with-require *) ;; Again, make sure it didn't add its required macros and/or tags. - (assert (not (in (hy.mangle "nonlocal-test-macro") __macros__))) + (assert (not (in (hy.mangle "nonlocal-test-macro") _hy_macros))) ;; Its tag(s) should be here now. - (assert (in (hy.mangle "#test-module-tag") __macros__)) + (assert (in (hy.mangle "#test-module-tag") _hy_macros)) ;; The test macro expands to include this symbol. (setv module-name-var "tests.native_tests.native_macros") @@ -277,7 +277,7 @@ in expansions." ;; Reload the module and clear the local macro context. (.clear sys.path_importer_cache) (del (get sys.modules "tests.resources.macro_with_require")) - (.clear __macros__) + (.clear _hy_macros) ;; There doesn't seem to be a way--via standard import mechanisms--to ;; ensure that an imported module used the cached bytecode. We'll simply have diff --git a/tests/native_tests/reader_macros.hy b/tests/native_tests/reader_macros.hy index 5552947b8..44047d294 100644 --- a/tests/native_tests/reader_macros.hy +++ b/tests/native_tests/reader_macros.hy @@ -31,7 +31,7 @@ (defn test-reader-macros [] (assert (= (eval-module #[[(defreader foo '1) #foo]]) 1)) (assert (in (hy.mangle "#foo") - (eval-module #[[(defreader foo '1) __reader_macros__]]))) + (eval-module #[[(defreader foo '1) _hy_reader_macros]]))) ;; Assert reader macros operating exclusively at read time (with [module (temp-module "")] From d7fafc85d724518d4fb30681db46dab6f8034eb9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 18 Feb 2023 15:08:01 -0500 Subject: [PATCH 162/342] Minor simplification of `test_bin.py` --- tests/test_bin.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/tests/test_bin.py b/tests/test_bin.py index 4319bce92..d94d4b081 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -33,9 +33,9 @@ def run_cmd( else: env.pop("PYTHONDONTWRITEBYTECODE", None) - p = subprocess.Popen( - shlex.split(cmd), - stdin=subprocess.PIPE, + result = subprocess.run( + shlex.split(cmd) if isinstance(cmd, str) else cmd, + input=stdin_data, stdout=stdout, stderr=subprocess.PIPE, universal_newlines=True, @@ -43,9 +43,8 @@ def run_cmd( env=env, cwd=cwd, ) - output = p.communicate(input=stdin_data) - assert p.wait() == expect - return output + assert result.returncode == expect + return (result.stdout, result.stderr) def rm(fpath): try: @@ -351,7 +350,7 @@ def test_hyc(): assert "usage" in output path = "tests/resources/argparse_ex.hy" - _, err = run_cmd("hyc " + path) + _, err = run_cmd(["hyc", path]) assert "Compiling" in err assert os.path.exists(cache_from_source(path)) rm(cache_from_source(path)) @@ -472,7 +471,7 @@ def testc_file_sys_path(): rm(cache_from_source(test_file)) assert not os.path.exists(cache_from_source(file_relative_path)) - output, _ = run_cmd(f"{binary} {test_file}") + output, _ = run_cmd([binary, test_file]) assert repr(file_relative_path) in output @@ -500,12 +499,12 @@ def test_circular_macro_require(): test_file = "tests/resources/bin/circular_macro_require.hy" rm(cache_from_source(test_file)) assert not os.path.exists(cache_from_source(test_file)) - output, _ = run_cmd("hy {}".format(test_file)) + output, _ = run_cmd(["hy", test_file]) assert output.strip() == "WOWIE" # Now, with bytecode assert os.path.exists(cache_from_source(test_file)) - output, _ = run_cmd("hy {}".format(test_file)) + output, _ = run_cmd(["hy", test_file]) assert output.strip() == "WOWIE" @@ -519,12 +518,12 @@ def test_macro_require(): test_file = "tests/resources/bin/require_and_eval.hy" rm(cache_from_source(test_file)) assert not os.path.exists(cache_from_source(test_file)) - output, _ = run_cmd("hy {}".format(test_file)) + output, _ = run_cmd(["hy", test_file]) assert output.strip() == "abc" # Now, with bytecode assert os.path.exists(cache_from_source(test_file)) - output, _ = run_cmd("hy {}".format(test_file)) + output, _ = run_cmd(["hy", test_file]) assert output.strip() == "abc" @@ -651,11 +650,10 @@ def test_output_buffering(tmp_path): (import sys pathlib [Path]) (print :file sys.stderr (.strip (.read-text (Path #[=[{tf}]=])))) (print "line 2")''') - pf = shlex.quote(str(pf)) - for flag, expected in ("", ""), ("--unbuffered", "line 1"): + for flags, expected in ([], ""), (["--unbuffered"], "line 1"): with open(tf, "wb") as o: - _, stderr = run_cmd(f"hy {flag} {pf}", stdout=o) + _, stderr = run_cmd(["hy", *flags, pf], stdout=o) assert stderr.strip() == expected assert tf.read_text().splitlines() == ["line 1", "line 2"] @@ -671,9 +669,9 @@ def test_uufileuu(tmp_path, monkeypatch): def file_is(arg, expected_py3_9): expected = expected_py3_9 if PY3_9 and not PYPY else Path(arg) - output, _ = run_cmd("python3 " + shlex.quote(arg + "pyex.py")) + output, _ = run_cmd(["python3", arg + "pyex.py"]) assert output.rstrip() == str(expected / "pyex.py") - output, _ = run_cmd("hy " + shlex.quote(arg + "hyex.hy")) + output, _ = run_cmd(["hy", arg + "hyex.hy"]) assert output.rstrip() == str(expected / "hyex.hy") monkeypatch.chdir(tmp_path) @@ -731,15 +729,15 @@ def test_hy2py_recursive(tmp_path): (setv a 1) (setv b "hello world")""") - _, err = run_cmd(f"hy2py {(tmp_path / 'hy').as_posix()}", expect=1) + _, err = run_cmd(["hy2py", (tmp_path / 'hy')], expect=1) assert "ValueError" in err - run_cmd("hy2py " + - f"{(tmp_path / 'hy').as_posix()} " + - f"--output {(tmp_path / 'py').as_posix()}") + run_cmd(["hy2py", + (tmp_path / 'hy'), + "--output", (tmp_path / 'py')]) assert set((tmp_path / 'py').rglob('*')) == { tmp_path / 'py' / p for p in ('first.py', 'folder', 'folder/second.py')} - output, _ = run_cmd(f"python3 first.py", cwd = tmp_path / 'py') + output, _ = run_cmd("python3 first.py", cwd = tmp_path / 'py') assert output == "1\nhello world\n" From 5fb663ee5beafe0188927cf4bb23deee204bfeb4 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 18 Feb 2023 14:20:30 -0500 Subject: [PATCH 163/342] Skip shebangs in the reader, not outside it Thus the reader can properly keep track of its position in the stream. --- hy/reader/__init__.py | 9 ++------- hy/reader/hy_reader.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/hy/reader/__init__.py b/hy/reader/__init__.py index 54a856401..8744c715f 100644 --- a/hy/reader/__init__.py +++ b/hy/reader/__init__.py @@ -30,16 +30,11 @@ def read_many(stream, filename="", reader=None, skip_shebang=False): if isinstance(stream, str): stream = StringIO(stream) pos = stream.tell() - if skip_shebang: - if stream.read(2) == "#!": - stream.readline() - pos = stream.tell() - else: - stream.seek(pos) source = stream.read() stream.seek(pos) - m = hy.models.Lazy((reader or HyReader()).parse(stream, filename)) + m = hy.models.Lazy((reader or HyReader()).parse( + stream, filename, skip_shebang)) m.source = source m.filename = filename return m diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 025adab7e..3f558b2c0 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -1,5 +1,7 @@ "Character reader for parsing Hy source." +from itertools import islice + import hy from hy.models import ( Bytes, @@ -140,7 +142,7 @@ def read_default(self, key): return self.prefixed_string('"', ident) return as_identifier(ident, reader=self) - def parse(self, stream, filename=None): + def parse(self, stream, filename=None, skip_shebang=False): """Yields all `hy.models.Object`'s in `source` Additionally exposes `self` as ``hy.&reader`` during read/compile time. @@ -151,8 +153,16 @@ def parse(self, stream, filename=None): filename (str | None): Filename to use for error messages. If `None` then previously set filename is used. + skip_shebang: + Whether to detect a skip a shebang line at the start. """ self._set_source(stream, filename) + + if skip_shebang and "".join(islice(self.peeking(), len("#!"))) == "#!": + for c in self.chars(): + if c == "\n": + break + rname = mangle("&reader") old_reader = getattr(hy, rname, None) setattr(hy, rname, self) From 5f0f808ae071fde9a2426c97c6ba7e5685716418 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 18 Feb 2023 14:36:44 -0500 Subject: [PATCH 164/342] Test shebangs --- tests/test_bin.py | 9 +++++++++ tests/test_reader.py | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/test_bin.py b/tests/test_bin.py index d94d4b081..04f4ac14d 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -610,6 +610,15 @@ def req_err(x): assert error_lines[-1].startswith("TypeError") +def test_traceback_shebang(tmp_path): + # https://github.com/hylang/hy/issues/2405 + (tmp_path / 'ex.hy').write_text('#!my cool shebang\n(/ 1 0)') + _, error = run_cmd(['hy', tmp_path / 'ex.hy'], expect = 1) + assert 'ZeroDivisionError' + assert 'my cool shebang' not in error + assert '(/ 1 0)' in error + + def test_hystartup(): # spy == True and custom repl-output-fn os.environ["HYSTARTUP"] = "tests/resources/hystartup.hy" diff --git a/tests/test_reader.py b/tests/test_reader.py index 0465bc00c..89a7fc7ab 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -23,8 +23,8 @@ from hy.reader.exceptions import LexException, PrematureEndOfInput -def tokenize(s): - return list(read_many(s)) +def tokenize(*args, **kwargs): + return list(read_many(*args, **kwargs)) def peoi(): @@ -675,3 +675,13 @@ def test_read_error(): assert "".join(traceback.format_exception_only(e.type, e.value)).startswith( ' File "", line 1\n (do (defn))\n ^\n' ) + + +def test_shebang(): + from hy.errors import HySyntaxError + + with pytest.raises(HySyntaxError): + # By default, `read_many` doesn't allow a shebang. + assert tokenize('#!/usr/bin/env hy\n5') + assert (tokenize('#!/usr/bin/env hy\n5', skip_shebang = True) == + [Integer(5)]) From 2173e8a8789d5e8088694a47967f6cd63d1cf724 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 18 Feb 2023 14:23:29 -0500 Subject: [PATCH 165/342] Update NEWS --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index a7fcff32b..53ff502d2 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -11,6 +11,7 @@ Bug Fixes ------------------------------ * Fixed an installation failure in some situations when version lookup fails. +* Fixed traceback pointing in scripts with shebangs. New Features ------------------------------ From 9484899ea0509b9f9fae19da45961842c6347efb Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 15 Mar 2023 13:37:12 -0400 Subject: [PATCH 166/342] For `py`, use implicit parens instead of dedenting This effectively removes Python's indentation restrictions, which is more useful. It also prevents indents in string literals from being discarded. --- NEWS.rst | 6 ++---- docs/api.rst | 14 ++++++++++---- hy/core/result_macros.py | 2 +- tests/compilers/test_ast.py | 9 ++++++++- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 53ff502d2..27066089e 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,10 +3,6 @@ Unreleased ============================= -Breaking Changes ------------------------------- -* The `py` macro now dedents the input like `pys`. - Bug Fixes ------------------------------ * Fixed an installation failure in some situations when version lookup @@ -17,6 +13,8 @@ New Features ------------------------------ * `nonlocal` and `global` can now be called with no arguments, in which case they're no-ops. +* The `py` macro now implicitly parenthesizes the input code, so Python's + indentation restrictions don't apply. 0.26.0 (released 2023-02-08) ============================= diff --git a/docs/api.rst b/docs/api.rst index 5843ed1b8..d01a6ca44 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -762,10 +762,9 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. code that's only defined at run-time, try the standard Python function :func:`eval`. - The code string is dedented with :func:`textwrap.dedent` before parsing, - which allows you to indent the code to match the surrounding Hy code when - Python would otherwise forbid this, but beware that significant leading - whitespace in embedded string literals will be removed. + The code is implicitly wrapped in parentheses so Python won't give you grief + about indentation. After all, Python's indentation rules are only useful for + grouping statements, whereas ``py`` only allows an expression. Python code need not syntactically round-trip if you use ``hy2py`` on a Hy program that uses ``py`` or ``pys``. For example, comments will be removed. @@ -779,6 +778,13 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (pys "myvar = 5") (print "myvar is" myvar) + Unlike ``py``, no parentheses are added, because Python doesn't allow + statements to be parenthesized. Instead, the code string is dedented with + :func:`textwrap.dedent` before parsing. Thus you can indent the code to + match the surrounding Hy code when Python would otherwise forbid this, but + beware that significant leading whitespace in embedded string literals will + be removed. + .. hy:macro:: (quasiquote [form]) ``quasiquote`` allows you to quote a form, but also selectively evaluate diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index c05dc1150..cff40983f 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -131,7 +131,7 @@ def compile_inline_python(compiler, expr, root, code): try: o = asty.parse( expr, - textwrap.dedent(code), + textwrap.dedent(code) if exec_mode else "(" + code + "\n)", compiler.filename, "exec" if exec_mode else "eval", ).body diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 667c94ad1..a44607a8e 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -574,11 +574,18 @@ def test_futures_imports(): def test_py(): def py(x): assert ( ast.dump(can_compile(f'(py "{x}")')) == - ast.dump(ast.parse(dedent(x)))) + ast.dump(ast.parse('(' + x + '\n)'))) py("1 + 1") # https://github.com/hylang/hy/issues/2406 py(" 1 + 1 ") + py(""" 1 + + 1 """) + py(""" 1 + 2 + + 3 + + 4 + + 5 + # hi! + 6 # bye """) cant_compile('(py "1 +")') cant_compile('(py "if 1:\n 2")') From 33f8af64d3a50ce1a3b7f2473df6bfdc3982708d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 16 Mar 2023 14:44:47 -0400 Subject: [PATCH 167/342] Fix escaping in bracket f-strings --- NEWS.rst | 1 + hy/reader/hy_reader.py | 2 +- tests/native_tests/strings.hy | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 27066089e..a0b30a20c 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -8,6 +8,7 @@ Bug Fixes * Fixed an installation failure in some situations when version lookup fails. * Fixed traceback pointing in scripts with shebangs. +* Fixed some bugs with escaping in bracket f-strings New Features ------------------------------ diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 3f558b2c0..4483db6db 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -437,7 +437,7 @@ def delim_closing(c): index = -1 return 0 - return self.read_string_until(delim_closing, None, is_fstring, brackets=delim) + return self.read_string_until(delim_closing, "fr" if is_fstring else None, is_fstring, brackets=delim) def read_string_until(self, closing, prefix, is_fstring, **kwargs): if is_fstring: diff --git a/tests/native_tests/strings.hy b/tests/native_tests/strings.hy index 02dd0a3ec..15b7df194 100644 --- a/tests/native_tests/strings.hy +++ b/tests/native_tests/strings.hy @@ -97,6 +97,10 @@ cee"} dee" "ey bee\ncee dee")) (assert (= #[f[a{p !r :9}]f] "a'xyzzy' ")) (assert (= #[f-string[result: {value :{width}.{precision}}]f-string] "result: 12.34")) + ; https://github.com/hylang/hy/issues/2419 + (assert (= + #[f[{{escaped braces}} \n {"not escaped"}]f] + "{escaped braces} \\n not escaped")) ; Quoting shouldn't evaluate the f-string immediately ; https://github.com/hylang/hy/issues/1844 From f19ed9b2a8b15f1473c0060eb0e95dfd36d1a82b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 25 Mar 2023 12:53:53 -0400 Subject: [PATCH 168/342] Recurse in `HyReader.fill_pos` --- NEWS.rst | 2 +- hy/reader/hy_reader.py | 4 +++- tests/test_reader.py | 7 +++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index a0b30a20c..6c8cb58f5 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -7,7 +7,7 @@ Bug Fixes ------------------------------ * Fixed an installation failure in some situations when version lookup fails. -* Fixed traceback pointing in scripts with shebangs. +* Fixed some bugs with traceback pointing. * Fixed some bugs with escaping in bracket f-strings New Features diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 4483db6db..985baaa5d 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -128,7 +128,9 @@ def fill_pos(self, model, start): """ model.start_line, model.start_column = start model.end_line, model.end_column = self.pos - return model + return model.replace(model) + # `replace` will recurse into submodels and set any model + # positions that are still unset the same way. def read_default(self, key): """Default reader handler when nothing in the table matches. diff --git a/tests/test_reader.py b/tests/test_reader.py index 89a7fc7ab..3ef6e38c9 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -410,6 +410,13 @@ def test_lex_line_counting_multi_inner(): assert inner.start_column == 5 +def test_line_counting_dotted(): + # https://github.com/hylang/hy/issues/2422 + x, = tokenize(";;;;;\na.b") + for e in (x, *x): + assert e.start_line == 2 + + def test_dicts(): """Ensure that we can tokenize a dict.""" objs = tokenize("{foo bar bar baz}") From 534a8b1924981e6982c33295d1dedc934dec50d6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 1 Apr 2023 09:47:25 -0400 Subject: [PATCH 169/342] Add `hy.M` import sugar --- NEWS.rst | 1 + docs/api.rst | 2 ++ docs/conf.py | 4 +++ hy/__init__.py | 17 ++++++++++ tests/native_tests/hy_misc.hy | 58 ++++++++++++++++++++++++++++++++++- 5 files changed, 81 insertions(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 6c8cb58f5..6db42c8b4 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -16,6 +16,7 @@ New Features case they're no-ops. * The `py` macro now implicitly parenthesizes the input code, so Python's indentation restrictions don't apply. +* New built-in object `hy.M` for easy imports in macros. 0.26.0 (released 2023-02-08) ============================= diff --git a/docs/api.rst b/docs/api.rst index d01a6ca44..6d64a97cf 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1312,6 +1312,8 @@ the following methods .. hy:autofunction:: hy.as-model +.. hy:autoclass:: hy.M + .. _reader-macros: Reader Macros diff --git a/docs/conf.py b/docs/conf.py index 8c00214b3..03ad67207 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -70,6 +70,10 @@ py3_10=("https://docs.python.org/3.10/", None), hyrule=("https://hyrule.readthedocs.io/en/master/", None), ) + +import hy +hy.M = type(hy.M) # A trick to enable `hy:autoclass:: hy.M` + # ** Generate Cheatsheet import json from itertools import zip_longest diff --git a/hy/__init__.py b/hy/__init__.py index ca6adfa1c..263f8c2ef 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -15,6 +15,23 @@ def _initialize_env_var(env_var, default_val): hy.importer._inject_builtins() # we import for side-effects. + +class M: + """``hy.M`` is an object that provides syntactic sugar for imports. It allows syntax like ``(hy.M.math.sqrt 2)`` to mean ``(import math) (math.sqrt 2)``, except without bringing ``math`` or ``math.sqrt`` into scope. This is useful in macros to avoid namespace pollution. To refer to a module with dots in its name, use slashes instead: ``hy.M.os/path.basename`` gets the function ``basename`` from the module ``os.path``. + + You can also call ``hy.M`` like a function, as in ``(hy.M "math")``, which is useful when the module name isn't known until run-time. This interface just calls :py:func:`importlib.import_module`, avoiding (1) mangling due to attribute lookup, and (2) the translation of ``/`` to ``.`` in the module name. The advantage of ``(hy.M modname)`` over ``importlib.import_module(modname)`` is merely that it avoids bringing ``importlib`` itself into scope.""" + def __call__(self, module_name): + import importlib + return importlib.import_module(module_name) + def __getattr__(self, s): + import re + return self(hy.mangle(re.sub( + r'/(-*)', + lambda m: '.' + '_' * len(m.group(1)), + hy.unmangle(s)))) +M = M() + + # Import some names on demand so that the dependent modules don't have # to be loaded if they're not needed. diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index 1f283823d..a50e35d64 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -1,5 +1,5 @@ ;; Tests of `hy.gensym`, `hy.macroexpand`, `hy.macroexpand-1`, -;; `hy.disassemble`, and `hy.read` +;; `hy.disassemble`, `hy.read`, and `hy.M` (import pytest) @@ -86,3 +86,59 @@ (assert (is (type (hy.read "[]")) (type '[]))) (assert (= (hy.read "0") '0)) (assert (is (type (hy.read "0")) (type '0)))) + + +(defn test-hyM [] + (defmacro no-name [name] + `(with [(pytest.raises NameError)] ~name)) + + ; `hy.M` doesn't bring the imported stuff into scope. + (assert (= (hy.M.math.sqrt 4) 2)) + (assert (= (.sqrt (hy.M "math") 4) 2)) + (no-name math) + (no-name sqrt) + + ; It's independent of bindings to such names. + (setv math (type "Dummy" #() {"sqrt" "hello"})) + (assert (= (hy.M.math.sqrt 4) 2)) + (assert (= math.sqrt "hello")) + + ; It still works in a macro expansion. + (defmacro frac [a b] + `(hy.M.fractions.Fraction ~a ~b)) + (assert (= (* 6 (frac 1 3)) 2)) + (no-name fractions) + (no-name Fraction) + + ; You can use `/` for dotted module names. + (assert (= (hy.M.os/path.basename "foo/bar") "bar")) + (no-name os) + (no-name path) + + ; `hy.M.__getattr__` attempts to cope with mangling. + (with [e (pytest.raises ModuleNotFoundError)] + (hy.M.a-b☘c-d/e.z)) + (assert (= e.value.name (hy.mangle "a-b☘c-d"))) + ; `hy.M.__call__` doesn't. + (with [e (pytest.raises ModuleNotFoundError)] + (hy.M "a-b☘c-d/e.z")) + (assert (= e.value.name "a-b☘c-d/e"))) + + +(defn test-hyM-mangle-chain [tmp-path monkeypatch] + ; We can get an object from a submodule with various kinds of + ; mangling in the name chain. + + (setv p tmp-path) + (for [e ["foo" "foo?" "_foo" "☘foo☘"]] + (/= p (hy.mangle e)) + (.mkdir p :exist-ok True) + (.write-text (/ p "__init__.py") "")) + (.write-text (/ p "foo.hy") "(setv foo 5)") + (monkeypatch.syspath-prepend (str tmp-path)) + + ; Python will reuse any `foo` imported in an earlier test if we + ; don't reload it explicitly. + (import foo) (import importlib) (importlib.reload foo) + + (assert (= hy.M.foo/foo?/_foo/☘foo☘/foo.foo 5))) From 2585c330b4480a796db2658cdc548743eb9b02cf Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 2 Apr 2023 09:07:05 -0400 Subject: [PATCH 170/342] Downgrade Pyodide on GitHub Actions --- .github/workflows/tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cad4cd899..552800e5d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,10 +39,11 @@ jobs: shell: bash run: | if [[ ${{ matrix.python }} = pyodide ]] ; then - npm install pyodide + npm install pyodide@0.22.1 + # 0.23.0 has a regression: https://github.com/pyodide/pyodide/issues/3730 pip install 'pip >= 22.3.1' # Older pips may fail to install `pyodide-build`. - pip install pyodide-build + pip install 'pyodide-build == 0.22.1' pyodide venv .venv-pyodide source .venv-pyodide/bin/activate fi From 6f0705b2aa8d2e9a7043380b9f1d0a4cc0ccd50e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 5 Apr 2023 11:42:59 -0400 Subject: [PATCH 171/342] Declare Python 3.12 compatibility For the first time I can remember, no code changes appear to be necessary to support the new Python. --- .github/workflows/tests.yml | 2 +- NEWS.rst | 1 + setup.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 552800e5d..57e4d46ce 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: matrix: name-prefix: [''] os: [ubuntu-latest] - python: [3.7, 3.8, 3.9, '3.10', 3.11, pypy-3.9, pyodide] + python: [3.7, 3.8, 3.9, '3.10', 3.11, 3.12-dev, pypy-3.9, pyodide] include: # To keep the overall number of runs low, we test Windows # only on the latest CPython. diff --git a/NEWS.rst b/NEWS.rst index 6db42c8b4..6b5e20d3e 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -12,6 +12,7 @@ Bug Fixes New Features ------------------------------ +* Python 3.12 is now supported. * `nonlocal` and `global` can now be called with no arguments, in which case they're no-ops. * The `py` macro now implicitly parenthesizes the input code, so Python's diff --git a/setup.py b/setup.py index 5707ce21b..fc3db3b7f 100755 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ def run(self): ), setup_requires=["wheel"] + requires, install_requires=requires, - python_requires=">= 3.7, < 3.12", + python_requires=">= 3.7, < 3.13", entry_points={ "console_scripts": [ "hy = hy.cmdline:hy_main", @@ -83,6 +83,7 @@ def run(self): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: PyPy", "Environment :: WebAssembly :: Emscripten", "Topic :: Software Development :: Code Generators", From bf62cf271fc126c6f1f6ec37696045234830bbcb Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 5 Apr 2023 11:43:50 -0400 Subject: [PATCH 172/342] Remove Python 3.7 support The CPython guys will stop supporting it on 2023-06-27. So our next release should be after that, but still before the release of Python 3.12 on 2023-10-02. --- .github/workflows/tests.yml | 2 +- NEWS.rst | 4 ++++ hy/_compat.py | 42 --------------------------------- hy/compiler.py | 7 +----- hy/importer.py | 3 +-- hy/macros.py | 5 ++-- setup.py | 3 +-- tests/importer/test_importer.py | 5 ---- tests/native_tests/hy_misc.hy | 5 ++-- tests/native_tests/setx.hy | 19 +++++---------- 10 files changed, 18 insertions(+), 77 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 57e4d46ce..34fc3ea6d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: matrix: name-prefix: [''] os: [ubuntu-latest] - python: [3.7, 3.8, 3.9, '3.10', 3.11, 3.12-dev, pypy-3.9, pyodide] + python: [3.8, 3.9, '3.10', 3.11, 3.12-dev, pypy-3.9, pyodide] include: # To keep the overall number of runs low, we test Windows # only on the latest CPython. diff --git a/NEWS.rst b/NEWS.rst index 6b5e20d3e..4f5073216 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,6 +3,10 @@ Unreleased ============================= +Removals +------------------------------ +* Python 3.7 is no longer supported. + Bug Fixes ------------------------------ * Fixed an installation failure in some situations when version lookup diff --git a/hy/_compat.py b/hy/_compat.py index d51809308..5c683b1ef 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -2,7 +2,6 @@ import platform import sys -PY3_8 = sys.version_info >= (3, 8) PY3_9 = sys.version_info >= (3, 9) PY3_10 = sys.version_info >= (3, 10) PY3_11 = sys.version_info >= (3, 11) @@ -41,44 +40,3 @@ def rewriting_unparse(ast_obj): return true_unparse(ast_obj) ast.unparse = rewriting_unparse - - -if not PY3_8: - # Shim `re.Pattern`. - import re - - re.Pattern = type(re.compile("")) - - -# Provide a function substitute for `CodeType.replace`. -if PY3_8: - - def code_replace(code_obj, **kwargs): - return code_obj.replace(**kwargs) - -else: - _code_args = [ - "co_" + c - for c in ( - "argcount", - "kwonlyargcount", - "nlocals", - "stacksize", - "flags", - "code", - "consts", - "names", - "varnames", - "filename", - "name", - "firstlineno", - "lnotab", - "freevars", - "cellvars", - ) - ] - - def code_replace(code_obj, **kwargs): - return type(code_obj)( - *(kwargs.get(k, getattr(code_obj, k)) for k in _code_args) - ) diff --git a/hy/compiler.py b/hy/compiler.py index 70cf5b795..efc6913b6 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -8,7 +8,6 @@ from funcparserlib.parser import NoParseError, many import hy -from hy._compat import PY3_8 from hy.errors import HyCompileError, HyLanguageError, HySyntaxError from hy.macros import macroexpand from hy.model_patterns import FORM, KEYWORD, unpack @@ -586,11 +585,7 @@ def compile_numeric_literal(self, x): @builds_model(Symbol) def compile_symbol(self, symbol): if symbol == Symbol("..."): - return ( - asty.Constant(symbol, value=Ellipsis) - if PY3_8 - else asty.Ellipsis(symbol) - ) + return asty.Constant(symbol, value=Ellipsis) # By this point, `symbol` should be either all dots or # dot-free. diff --git a/hy/importer.py b/hy/importer.py index 7f9a4d7c6..2f7ce9519 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -10,7 +10,6 @@ from functools import partial import hy -from hy._compat import PY3_8 from hy.compiler import hy_compile from hy.reader import read_many @@ -129,7 +128,7 @@ def _hy_source_to_code(self, data, path, _optimize=-1): importlib.machinery.SourceFileLoader.source_to_code = _hy_source_to_code -if PY3_8 and (".hy", False, False) not in zipimport._zip_searchorder: +if (".hy", False, False) not in zipimport._zip_searchorder: zipimport._zip_searchorder += ((".hy", False, False),) _py_compile_source = zipimport._compile_source diff --git a/hy/macros.py b/hy/macros.py index c23dcbc82..985ac2334 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -12,7 +12,7 @@ from funcparserlib.parser import NoParseError import hy.compiler -from hy._compat import PY3_11, code_replace +from hy._compat import PY3_11 from hy.errors import ( HyLanguageError, HyMacroExpansionError, @@ -443,8 +443,7 @@ def macroexpand_1(tree, module, compiler=None): def rename_function(f, new_name): """Create a copy of a function, but with a new name.""" f = type(f)( - code_replace( - f.__code__, + f.__code__.replace( co_name=new_name, **( { diff --git a/setup.py b/setup.py index fc3db3b7f..1256dcf95 100755 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ def run(self): ), setup_requires=["wheel"] + requires, install_requires=requires, - python_requires=">= 3.7, < 3.13", + python_requires=">= 3.8, < 3.13", entry_points={ "console_scripts": [ "hy = hy.cmdline:hy_main", @@ -78,7 +78,6 @@ def run(self): "Programming Language :: Lisp", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index 382a8d5fa..921fbaf7e 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -8,7 +8,6 @@ import pytest import hy -from hy._compat import PY3_8 from hy.compiler import hy_compile, hy_eval from hy.errors import HyLanguageError, hy_exc_handler from hy.importer import HyLoader @@ -276,10 +275,6 @@ def test_filtered_importlib_frames(capsys): assert "importlib._" not in captured_w_filtering -@pytest.mark.skipif( - not PY3_8, - reason="Python 3.7's `zipimport` is written in C, it can't be monkey-patched", -) def test_zipimport(tmp_path): from zipfile import ZipFile diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index a50e35d64..63c0492b1 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -44,12 +44,11 @@ (import re) (defn nos [x] (re.sub r"\s" "" x)) (assert (= (nos (hy.disassemble '(do (leaky) (leaky) (macros)))) - (nos (.format + (nos "Module( body=[Expr(value=Call(func=Name(id='leaky', ctx=Load()), args=[], keywords=[])), Expr(value=Call(func=Name(id='leaky', ctx=Load()), args=[], keywords=[])), - Expr(value=Call(func=Name(id='macros', ctx=Load()), args=[], keywords=[]))]{})" - (if hy._compat.PY3_8 ",type_ignores=[]" ""))))) + Expr(value=Call(func=Name(id='macros', ctx=Load()), args=[], keywords=[]))],type_ignores=[])"))) (assert (= (nos (hy.disassemble '(do (leaky) (leaky) (macros)) True)) "leaky()leaky()macros()")) (assert (= (re.sub r"[()\n ]" "" (hy.disassemble `(+ ~(+ 1 1) 40) True)) diff --git a/tests/native_tests/setx.hy b/tests/native_tests/setx.hy index 80a4ebff9..2b46ff451 100644 --- a/tests/native_tests/setx.hy +++ b/tests/native_tests/setx.hy @@ -1,13 +1,6 @@ (import pytest) -(defn - [(pytest.mark.skipif hy._compat.PY3_8 :reason "Python ≥ 3.8")] - test-cant-setx [] - (with [e (pytest.raises hy.errors.HySyntaxError)] - (hy.eval '(setx x 1))) - (assert (= "setx requires Python 3.8 or later"))) - -(do-mac (when hy._compat.PY3_8 '(defn test-setx [] +(defn test-setx [] (setx y (+ (setx x (+ "a" "b")) "c")) (assert (= x "ab")) (assert (= y "abc")) @@ -26,9 +19,9 @@ (assert (= filtered ["apple" "banana"])) (assert (= v "banana")) (with [(pytest.raises NameError)] - i)))) + i)) -(do-mac (when hy._compat.PY3_8 '(defn test-setx-generator-scope [] +(defn test-setx-generator-scope [] ;; https://github.com/hylang/hy/issues/1994 (setv x 20) (lfor n (range 10) (setx x n)) @@ -51,11 +44,11 @@ (lfor n (range 0) :do x (setx z n)) (with [(pytest.raises UnboundLocalError)] - z)))) + z)) -(do-mac (when hy._compat.PY3_8 '(defn test-let-setx [] +(defn test-let-setx [] (let [x 40 y 13] (setv y (setx x 2)) (assert (= x 2)) - (assert (= y 2)))))) + (assert (= y 2)))) From 3a468ee29bc6e75ecca36949f7ebe48ae392bb25 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 11 Apr 2023 14:23:17 -0400 Subject: [PATCH 173/342] Silence `PytestReturnNotNoneWarning` https://github.com/pytest-dev/pytest/issues/10465#issuecomment-1304624846 --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 4f33fc84c..630ba1208 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,4 @@ filterwarnings = once::DeprecationWarning once::PendingDeprecationWarning ignore::SyntaxWarning + ignore::pytest.PytestReturnNotNoneWarning From 38462239b388492133423e91b784f21effa9c1ff Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 11 Apr 2023 14:36:36 -0400 Subject: [PATCH 174/342] Allow EOF when peeking for a shebang --- hy/reader/hy_reader.py | 3 ++- tests/native_tests/repl.hy | 6 ++++++ tests/test_bin.py | 7 +++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 985baaa5d..b8bdb4c8f 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -160,7 +160,8 @@ def parse(self, stream, filename=None, skip_shebang=False): """ self._set_source(stream, filename) - if skip_shebang and "".join(islice(self.peeking(), len("#!"))) == "#!": + if skip_shebang and "".join( + islice(self.peeking(eof_ok = True), len("#!"))) == "#!": for c in self.chars(): if c == "\n": break diff --git a/tests/native_tests/repl.hy b/tests/native_tests/repl.hy index 2b37e0ba2..dfc36375c 100644 --- a/tests/native_tests/repl.hy +++ b/tests/native_tests/repl.hy @@ -11,3 +11,9 @@ (assert (= sys.ps1 "chippy")) (.run (hy.REPL)) (assert (= sys.ps1 "chippy"))) + +(defn test-repl-input-1char [monkeypatch capsys] + ; https://github.com/hylang/hy/issues/2430 + (monkeypatch.setattr "sys.stdin" (io.StringIO "1\n")) + (.run (hy.REPL)) + (assert (= (. capsys (readouterr) out) "=> 1\n=> " ))) diff --git a/tests/test_bin.py b/tests/test_bin.py index 04f4ac14d..a0a7666d8 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -332,6 +332,13 @@ def test_icmd_and_spy(): assert "[] + []" in output +def test_empty_file(tmp_path): + # https://github.com/hylang/hy/issues/2427 + (tmp_path / 'foo.hy').write_text('') + run_cmd(['hy', (tmp_path / 'foo.hy')]) + # This asserts that the return code is 0. + + def test_missing_file(): _, err = run_cmd("hy foobarbaz", expect=2) assert "No such file" in err From 312d55df5ec1c63f8641d550f09eb5101c90fc47 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 11 Apr 2023 14:49:41 -0400 Subject: [PATCH 175/342] Forbid shebangs in the REPL and `hy -c` --- NEWS.rst | 1 + hy/cmdline.py | 2 +- hy/repl.py | 2 +- tests/native_tests/repl.hy | 10 +++++++++- tests/test_bin.py | 4 ++++ 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 4f5073216..ed01fc8e6 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -13,6 +13,7 @@ Bug Fixes fails. * Fixed some bugs with traceback pointing. * Fixed some bugs with escaping in bracket f-strings +* The parser no longer looks for shebangs in the REPL or `hy -c`. New Features ------------------------------ diff --git a/hy/cmdline.py b/hy/cmdline.py index 72d0bc224..dca3febb0 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -35,7 +35,7 @@ def run_command(source, filename=None): with filtered_hy_exceptions(): try: hy_eval( - read_many(source, filename=filename, skip_shebang=True), + read_many(source, filename=filename), __main__.__dict__, __main__, filename=filename, diff --git a/hy/repl.py b/hy/repl.py index 1b5310bda..b4602c34a 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -167,7 +167,7 @@ def __call__(self, source, filename="", symbol="single"): self.hy_compiler.filename = name self.hy_compiler.source = source hy_ast = read_many( - source, filename=name, reader=self.reader, skip_shebang=True + source, filename=name, reader=self.reader ) exec_ast, eval_ast = hy_compile( hy_ast, diff --git a/tests/native_tests/repl.hy b/tests/native_tests/repl.hy index dfc36375c..384d0f92a 100644 --- a/tests/native_tests/repl.hy +++ b/tests/native_tests/repl.hy @@ -2,7 +2,8 @@ (import io - sys) + sys + pytest) (defn test-preserve-ps1 [monkeypatch] ; https://github.com/hylang/hy/issues/1323#issuecomment-1310837340 @@ -17,3 +18,10 @@ (monkeypatch.setattr "sys.stdin" (io.StringIO "1\n")) (.run (hy.REPL)) (assert (= (. capsys (readouterr) out) "=> 1\n=> " ))) + +(defn test-repl-no-shebangs [monkeypatch capsys] + (monkeypatch.setattr "sys.stdin" (io.StringIO "#!/usr/bin/env hy\n")) + (.run (hy.REPL)) + (assert (in + "hy.reader.exceptions.LexException" + (. capsys (readouterr) err)))) diff --git a/tests/test_bin.py b/tests/test_bin.py index a0a7666d8..8b1b17538 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -303,6 +303,10 @@ def test_cmd(): _, err = run_cmd("""hy -c '(print (.upper "hello")'""", expect=1) assert "Premature end of input" in err + # No shebang is allowed. + _, err = run_cmd("""hy -c '#!/usr/bin/env hy'""", expect = 1) + assert "LexException" in err + # https://github.com/hylang/hy/issues/1879 output, _ = run_cmd( """hy -c '(setv x "bing") (defn f [] (+ "fiz" x)) (print (f))'""" From 10b9473fec8eac8b0157ac5affb854ba184a35f6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 5 May 2023 11:28:07 -0400 Subject: [PATCH 176/342] Check for existence before calling `os.path.samefile` This fixes test failures on Windows GitHub Actions due to a module being attributed to a nonexistent file "hy.exe/__main__.py". --- hy/macros.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hy/macros.py b/hy/macros.py index 985ac2334..cc81a690a 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -133,6 +133,8 @@ def _get_filename(module): return ( source_filename and target_filename + and os.path.exists(source_filename) + and os.path.exists(target_filename) and os.path.samefile(source_filename, target_filename) ) From a7866a73e9a0baac091a0852da0419fa116bc59d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 5 May 2023 11:40:02 -0400 Subject: [PATCH 177/342] Filter out "hy.exe" when filtering tracebacks This fixes a traceback-filtering test failure on Windows GitHub Actions. --- hy/errors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hy/errors.py b/hy/errors.py index 6f52eeb99..e59a3f746 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -261,6 +261,7 @@ def hy_exc_filter(exc_type, exc_value, exc_traceback): if not ( frame[0].replace(".pyc", ".py") in _tb_hidden_modules or os.path.dirname(frame[0]) in _tb_hidden_modules + or os.path.basename(os.path.dirname(frame[0])) == "hy.exe" ): new_tb += [frame] From d841d58e5ab8425492880f54edb0600768554be1 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 5 May 2023 10:53:29 -0400 Subject: [PATCH 178/342] In the REPL, print a delimiter after `spy` output --- NEWS.rst | 2 ++ docs/cli.rst | 1 + hy/repl.py | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index ed01fc8e6..c3c314aa9 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -23,6 +23,8 @@ New Features * The `py` macro now implicitly parenthesizes the input code, so Python's indentation restrictions don't apply. * New built-in object `hy.M` for easy imports in macros. +* `hy --spy` now prints a delimiter between the Python equivalent of + your code and the result of evaluating the code for easier reading. 0.26.0 (released 2023-02-08) ============================= diff --git a/docs/cli.rst b/docs/cli.rst index 2390d44eb..f40c18f58 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -28,6 +28,7 @@ for a complete list of options and :py:ref:`Python's documentation => (+ 1 2) 1 + 2 + ------------------------------ 3 .. cmdoption:: --repl-output-fn diff --git a/hy/repl.py b/hy/repl.py index b4602c34a..5382a70f6 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -240,7 +240,7 @@ class REPL(code.InteractiveConsole): Note that as with :func:`code.interact`, changes to ``(locals)`` inside the REPL are not propagated back to the original scope.""" - def __init__(self, spy=False, output_fn=None, locals=None, filename="", allow_incomplete=True): + def __init__(self, spy=False, spy_delimiter=('-' * 30), output_fn=None, locals=None, filename="", allow_incomplete=True): # Create a proper module for this REPL so that we can obtain it easily # (e.g. using `importlib.import_module`). @@ -293,6 +293,7 @@ def __init__(self, spy=False, output_fn=None, locals=None, filename="", a ) self.spy = spy + self.spy_delimiter = spy_delimiter self.last_value = None self.print_last_value = True @@ -332,6 +333,7 @@ def ast_callback(self, exec_ast, eval_ast): type_ignores=[], ) print(ast.unparse(new_ast)) + print(self.spy_delimiter) except Exception: msg = "Exception in AST callback:\n{}\n".format(traceback.format_exc()) self.write(msg) From c5ac60f759659f9e007f02fb2456f2e0e6febd6a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 16 May 2023 12:57:00 -0400 Subject: [PATCH 179/342] Delete most of `hacking.rst` These instructions are partly out of date, partly redundant with `CONTRIBUTING.rst`, and partly beyond the call of duty for the Hy manual (like explaining how to pull from a GitHub repository). --- docs/hacking.rst | 98 ------------------------------------------------ 1 file changed, 98 deletions(-) diff --git a/docs/hacking.rst b/docs/hacking.rst index 96e2cc85c..3b5eaec38 100644 --- a/docs/hacking.rst +++ b/docs/hacking.rst @@ -4,104 +4,6 @@ Hacking on Hy =============== -.. highlight:: bash - -Join our Hyve! -============== - -Please come hack on Hy! - -Please come hang out with us on `the Github Discussions page `_! - -Please talk about it on Twitter with the ``#hy`` hashtag! - -Please blog about it! - -Please don't spraypaint it on your neighbor's fence (without asking nicely)! - - -Hack! -===== - -Do this: - -1. Create a `virtual environment - `_:: - - $ virtualenv venv - - and activate it:: - - $ . venv/bin/activate - - or use `virtualenvwrapper `_ - to create and manage your virtual environment:: - - $ mkvirtualenv hy - $ workon hy - -2. Get the source code:: - - $ git clone https://github.com/hylang/hy.git - - or use your fork:: - - $ git clone git@github.com:/hy.git - -3. Install for hacking:: - - $ cd hy/ - $ pip install -e . - -4. Install other develop-y requirements:: - - $ pip install -r requirements-dev.txt - -5. Optionally, enable the `pre-commit `_ hooks defined in ``.pre-commit-config.yaml``:: - - $ pre-commit install - - This will ensure your code adheres to the formatting conventions enforced via continuous integration (CI). - -6. Optionally, tell ``git blame`` to ignore the commits listed in ``.git-blame-ignore-revs``:: - - $ git config blame.ignoreRevsFile .git-blame-ignore-revs - - This file is intended to contains commits with large diffs but negligible semantic changes. - -7. Do awesome things; make someone shriek in delight/disgust at what - you have wrought. - - -Test! -===== - -Tests are located in ``tests/``. We use `pytest `_. - -To run the tests:: - - $ pytest - -Write tests---tests are good! - -Also, it is good to run the tests for all the platforms supported and for -PEP 8 compliant code. You can do so by running tox:: - - $ tox - -Document! -========= - -Documentation is located in ``docs/``. We use `Sphinx -`_. - -To build the docs in HTML:: - - $ cd docs - $ sphinx-build . _build -b html - -Write docs---docs are good! Even this doc! - .. include:: ../CONTRIBUTING.rst Core Team From 8ed835432982fb0ca97c4900b4382a6d0f5a32c3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 16 May 2023 13:16:27 -0400 Subject: [PATCH 180/342] Rewording in `hacking.rst` --- docs/hacking.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/hacking.rst b/docs/hacking.rst index 3b5eaec38..90d0a37d9 100644 --- a/docs/hacking.rst +++ b/docs/hacking.rst @@ -1,7 +1,7 @@ .. _hacking: =============== - Hacking on Hy + Developing Hy =============== .. include:: ../CONTRIBUTING.rst @@ -9,6 +9,6 @@ Core Team ========= -The core development team of Hy consists of following developers: +Hy's core development team consists of the following people: .. include:: coreteam.rst From 0dad545136d6470158881cf7110a8219cbf11e13 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 16 May 2023 13:13:11 -0400 Subject: [PATCH 181/342] Update `coreteam.rst` to match the actual GitHub team --- docs/coreteam.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/coreteam.rst b/docs/coreteam.rst index cd9a48ecb..7b04dca87 100644 --- a/docs/coreteam.rst +++ b/docs/coreteam.rst @@ -1,10 +1,10 @@ +* `Peter Andreev `_ * `Kodi B. Arfer `_ -* `Nicolas Dandrimont `_ +* `Allie Jo Casey `_ +* `Sunjay Cauligi `_ * `Julien Danjou `_ -* `Rob Day `_ * `Simon Gomizelj `_ * `Ryan Gonzalez `_ -* `Abhishek Lekshmanan `_ * `Morten Linderud `_ * `Matthew Odendahl `_ * `Paul Tagliamonte `_ From 64bbff26766fb3f0d3249fcb900d14b4e120ce2b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 16 May 2023 13:32:01 -0400 Subject: [PATCH 182/342] Add a bit about documentation to `CONTRIBUTING.rst` --- CONTRIBUTING.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d13b883c8..8709b0f5d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -81,6 +81,11 @@ No PR may be merged if it causes any tests to fail. The byte-compiled versions of the test files can be purged using ``git clean -dfx tests/``. If you want to run the tests while skipping the slow ones in ``test_bin.py``, use ``pytest --ignore=tests/test_bin.py``. +Documentation +------------- + +Generally, new features deserve coverage in the manual, either by editing the manual files directly or by changing docstrings that get included in the manual. To render the manual, install its dependencies with ``pip install -r requirements-dev.txt`` and then use the command ``cd docs; sphinx-build . _build -b html``. + NEWS and AUTHORS ---------------- From 46206da0c4bcd370a290aa2d8dc18bc36fb24d0a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 16 May 2023 13:25:21 -0400 Subject: [PATCH 183/342] Fix some syntax in `CONTRIBUTING.rst` --- CONTRIBUTING.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8709b0f5d..d3658dd06 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,7 +8,9 @@ helps in making Hy better. Potential contributions include: - Requesting features. - Adding features. - Writing tests for outstanding bugs or untested features. + - You can mark tests that Hy can't pass yet as xfail_. + - Cleaning up the code. - Improving the documentation. - Answering questions on `the Github Discussions page`_ or @@ -70,7 +72,7 @@ first line with a blank line. Testing ------- -Tests can be run by executing `pytest` in the root of this repository. +Tests can be run by executing ``pytest`` in the root of this repository. New features and bug fixes should be tested. If you've caused an xfail_ test to start passing, remove the xfail mark. If you're From f005203ff567f2eda8c6b1082d33299bf237f8cd Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Tue, 23 May 2023 01:57:55 +0200 Subject: [PATCH 184/342] fix tests for running locally --- tests/test_bin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_bin.py b/tests/test_bin.py index 8b1b17538..32e3c84f0 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -33,6 +33,10 @@ def run_cmd( else: env.pop("PYTHONDONTWRITEBYTECODE", None) + # ensure hy root dir is in Python's path, + # so that we can import/require modules within tests/ + env["PYTHONPATH"] = str(Path().resolve()) + os.pathsep + env.get("PYTHONPATH", "") + result = subprocess.run( shlex.split(cmd) if isinstance(cmd, str) else cmd, input=stdin_data, From aa615fcd2a3c673283cdd414b55d195cdcaf796e Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Tue, 23 May 2023 01:57:39 +0200 Subject: [PATCH 185/342] fix tests for mac osx --- .github/workflows/tests.yml | 5 ++++- tests/test_completer.py | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 34fc3ea6d..92f4c4480 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,11 +12,14 @@ jobs: os: [ubuntu-latest] python: [3.8, 3.9, '3.10', 3.11, 3.12-dev, pypy-3.9, pyodide] include: - # To keep the overall number of runs low, we test Windows + # To keep the overall number of runs low, we test Windows and MacOS # only on the latest CPython. - name-prefix: 'win-' os: windows-latest python: 3.11 + - name-prefix: 'macos-' + os: macos-latest + python: 3.11 name: ${{ format('{0}{1}', matrix.name-prefix, matrix.python) }} runs-on: ${{ matrix.os }} diff --git a/tests/test_completer.py b/tests/test_completer.py index 5ca3a7771..e58f2bc2a 100644 --- a/tests/test_completer.py +++ b/tests/test_completer.py @@ -25,6 +25,13 @@ def test_history_custom_location(tmp_path): readline.add_history(expected_entry) actual_entry = history_location.read_text() + + # yes, this is recommended way to check GNU readline vs libedit + # see https://docs.python.org/3.11/library/readline.html + if "libedit" in readline.__doc__: + # libedit saves spaces as octal escapes, so convert them back + actual_entry = actual_entry.replace("\\040", " ") + assert expected_entry in actual_entry From b2a7110d115a59e8a4be87dba6003e0fbd098b93 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 20 May 2023 11:23:56 -0400 Subject: [PATCH 186/342] Add a function version of `cut` --- docs/api.rst | 13 ------------- hy/pyops.hy | 31 +++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 6d64a97cf..13b020118 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -972,19 +972,6 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. None) (print (f 4)) ; Prints "14" and then "None" -.. hy:macro:: (cut [coll arg1 arg2 arg3]) - - ``cut`` compiles to a :ref:`slicing expression `, which selects multiple elements of a sequential data structure. The first argument is the object to be sliced. The remaining arguments are optional, and understood the same way as in a Python slicing expression. :: - - (setv x "abcdef") - (cut x) ; => "abcdef" - (cut x 3) ; => "abc" - (cut x 3 5) ; => "de" - (cut x -3 None) ; => "def" - (cut x 0 None 2) ; => "ace" - - A ``cut`` form is a valid target for assignment (with :hy:func:`setv`, ``+=``, etc.) and for deletion (with :hy:func:`del`). - .. hy:macro:: (raise [exception :from other]) ``raise`` compiles to a :py:keyword:`raise` statement, which throws an diff --git a/hy/pyops.hy b/hy/pyops.hy index 1869d1060..bb45c38cc 100644 --- a/hy/pyops.hy +++ b/hy/pyops.hy @@ -232,7 +232,7 @@ is equivalent to ``(+= count (+ n1 n2 n3)).``" If you're looking for the Hy equivalent of Python list slicing, as in ``foo[1:3]``, note that this is just Python's syntactic sugar for ``foo[slice(1, 3)]``, and Hy - provides its own syntactic sugar for this with a different macro, :hy:func:`cut`. + provides its own syntactic sugar for this with a different macro, :hy:func:`cut `. Note that ``.`` (:ref:`dot `) forms can also subscript. See also Hyrule's :hy:func:`assoc ` to easily assign multiple elements of a @@ -243,6 +243,33 @@ is equivalent to ``(+= count (+ n1 n2 n3)).``" (setv coll (get coll k))) coll) +(defn cut [coll / [arg1 'sentinel] [arg2 'sentinel] [arg3 'sentinel]] + #[[``cut`` compiles to a :ref:`slicing expression `, which selects multiple + elements of a sequential data structure. The first argument is the object to be + sliced. The remaining arguments are optional, and understood the same way as in a + Python slicing expression. :: + + (setv x "abcdef") + (cut x) ; => "abcdef" + (cut x 3) ; => "abc" + (cut x 3 5) ; => "de" + (cut x -3 None) ; => "def" + (cut x 0 None 2) ; => "ace" + + A call to the ``cut`` macro (but not its function version in ``hy.pyops``) is a valid + target for assignment (with :hy:func:`setv`, ``+=``, etc.) and for deletion (with + :hy:func:`del`).]] + + (cond + (= arg1 'sentinel) + (cut coll) + (= arg2 'sentinel) + (cut coll arg1) + (= arg3 'sentinel) + (cut coll arg1 arg2) + True + (cut coll arg1 arg2 arg3))) + (setv __all__ (list (map hy.mangle [ '+ '- '* '** '/ '// '% '@ @@ -250,4 +277,4 @@ is equivalent to ``(+= count (+ n1 n2 n3)).``" '< '> '<= '>= '= '!= 'and 'or 'not 'is 'is-not 'in 'not-in - 'get]))) + 'get 'cut]))) From 9eb7246a486816ee675973d2d04629312bc7cfee Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 20 May 2023 11:24:16 -0400 Subject: [PATCH 187/342] Expand testing of `cut` --- tests/native_tests/cut.hy | 5 ----- tests/native_tests/operators.hy | 21 ++++++++++++++++++++- 2 files changed, 20 insertions(+), 6 deletions(-) delete mode 100644 tests/native_tests/cut.hy diff --git a/tests/native_tests/cut.hy b/tests/native_tests/cut.hy deleted file mode 100644 index 999a5246c..000000000 --- a/tests/native_tests/cut.hy +++ /dev/null @@ -1,5 +0,0 @@ -(defn test-cut [] - (assert (= (cut [1 2 3 4 5] 3) [1 2 3])) - (assert (= (cut [1 2 3 4 5] 1 None) [2 3 4 5])) - (assert (= (cut [1 2 3 4 5] 1 3) [2 3])) - (assert (= (cut [1 2 3 4 5]) [1 2 3 4 5]))) diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy index ebea9e992..6e88ab027 100644 --- a/tests/native_tests/operators.hy +++ b/tests/native_tests/operators.hy @@ -314,13 +314,32 @@ (assert (= (f [[1 2 3] [4 5 6] [7 8 9]] 1 2) 6)) (assert (= (f {"x" {"y" {"z" 12}}} "x" "y" "z") 12))) - (defn test-setv-get [] (setv foo [0 1 2]) (setv (get foo 0) 12) (assert (= (get foo 0) 12))) +(op-and-shadow-test [cut] + (forbid (f)) + (setv x "abcdef") + (assert (= (f x) "abcdef")) + (assert (= (f x 3) "abc")) + (assert (= (f x -2) "abcd")) + (assert (= (f x 3 None) "def")) + (assert (= (f x -2 None) "ef")) + (assert (= (f x 3 5) "de")) + (assert (= (f x 0 None 2) "ace")) + (assert (= (list (f (range 100) 20 80 13)) [20 33 46 59 72]))) + +(defn test-setv-cut [] + (setv foo (list (range 20))) + (setv (cut foo 2 18 3) (* [0] 6)) + (assert (= + foo + [0 1 0 3 4 0 6 7 0 9 10 0 12 13 0 15 16 0 18 19]))) + + (defn test-chained-comparison [] (assert (chainc 2 = (+ 1 1) = (- 3 1))) (assert (not (chainc 2 = (+ 1 1) = (+ 3 1)))) From 0bcce5f531cf6d1d06faa050a531df143504a376 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 20 May 2023 11:25:05 -0400 Subject: [PATCH 188/342] Expand the module docstring for `hy.pyops` --- hy/pyops.hy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hy/pyops.hy b/hy/pyops.hy index bb45c38cc..34342d79c 100644 --- a/hy/pyops.hy +++ b/hy/pyops.hy @@ -1,11 +1,16 @@ "Python provides various :ref:`binary and unary operators `. These are usually invoked in Hy using core macros of the same name: for example, ``(+ 1 2)`` calls the core macro named -``+``, which uses Python's addition operator. There are two exceptions +``+``, which uses Python's addition operator. There are a few exceptions to the names being the same: -- ``==`` in Python is ``=`` in Hy. -- ``~`` in Python is ``bnot`` in Hy. +- ``==`` in Python is :hy:func:`= ` in Hy. +- ``~`` in Python is :hy:func:`bnot ` in Hy. +- ``is not`` in Python is :hy:func:`is-not ` in Hy. +- ``not in`` in Python is :hy:func:`not-in ` in Hy. + +For Python's subscription expressions (like ``x[2]``), Hy has two named +macros, :hy:func:`get ` and :hy:func:`cut `. By importing from the module ``hy.pyops`` (typically with a star import, as in ``(import hy.pyops *)``), you can also use these operators as From 1abd62c77d9e8af32b68ee87a3dbc21479f2bbcd Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 20 May 2023 11:25:50 -0400 Subject: [PATCH 189/342] Update NEWS --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index c3c314aa9..7e9634f08 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -22,6 +22,7 @@ New Features case they're no-ops. * The `py` macro now implicitly parenthesizes the input code, so Python's indentation restrictions don't apply. +* `cut` now has a function version in `hy.pyops`. * New built-in object `hy.M` for easy imports in macros. * `hy --spy` now prints a delimiter between the Python equivalent of your code and the result of evaluating the code for easier reading. From 7f20bfcd2d1c6162adf5a7c375547fcd097f38eb Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Fri, 26 May 2023 09:57:16 +0200 Subject: [PATCH 190/342] add new argument `type_params` to FunctionDef and ClassDef nodes --- hy/_compat.py | 1 + hy/core/result_macros.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/hy/_compat.py b/hy/_compat.py index 5c683b1ef..a752bfec7 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -5,6 +5,7 @@ PY3_9 = sys.version_info >= (3, 9) PY3_10 = sys.version_info >= (3, 10) PY3_11 = sys.version_info >= (3, 11) +PY3_12 = sys.version_info >= (3, 12) PYPY = platform.python_implementation() == "PyPy" PYODIDE = platform.system() == "Emscripten" diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index cff40983f..0a8ce99e2 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -15,7 +15,7 @@ from funcparserlib.parser import finished, forward_decl, many, maybe, oneplus, some -from hy._compat import PY3_11 +from hy._compat import PY3_11, PY3_12 from hy.compiler import Result, asty, hy_eval, mkexpr from hy.errors import HyEvalError, HyInternalError, HyTypeError from hy.macros import pattern_macro, require, require_reader @@ -896,6 +896,7 @@ def f(parts): ), body=stmts + f(parts).stmts, decorator_list=[], + **({"type_params": []} if PY3_12 else {}), ) # Immediately call the new function. Unless the user asked # for a generator, wrap the call in `[].__class__(...)` or @@ -1134,6 +1135,7 @@ def compile_match_expression(compiler, expr, root, subject, clauses): ), body=guard.stmts + [asty.Return(guard.expr, value=guard.expr)], decorator_list=[], + **({"type_params": []} if PY3_12 else {}), ) lifted_if_defs.append(guardret) guard = Result(expr=asty.parse(guard, f"{fname}()").body[0].value) @@ -1466,6 +1468,7 @@ def compile_function_node(compiler, expr, node, decorators, name, args, returns, body=body.stmts or [asty.Pass(expr)], decorator_list=decorators, returns=compiler.compile(returns).force_expr if returns is not None else None, + **({"type_params": []} if PY3_12 else {}), ) ast_name = asty.Name(expr, id=name, ctx=ast.Load()) @@ -1679,6 +1682,7 @@ def compile_class_expression(compiler, expr, root, decorators, name, rest): kwargs=None, bases=bases_expr, body=bodyr.stmts or [asty.Pass(expr)], + **({"type_params": []} if PY3_12 else {}), ) From 2fe418ab8f72362fd892920e00f20c2e2efadc64 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Tue, 23 May 2023 12:03:07 +0200 Subject: [PATCH 191/342] move #^ into a reader macro function --- hy/core/macros.hy | 8 ++------ hy/reader/hy_reader.py | 12 +++++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 07e06678e..6ffdb979a 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -60,9 +60,8 @@ and its base class :py:class:`Reader ` for details regarding the available processing methods. - Reader macro names can be any symbol that does not start with a ``^`` and are - callable by prefixing the name with a ``#``. i.e. ``(defreader upper ...)`` is - called with ``#upper``. + Reader macro names can be any valid identifier and are callable by prefixing + the name with a ``#``. i.e. ``(defreader upper ...)`` is called with ``#upper``. Examples: @@ -99,9 +98,6 @@ (when (not (isinstance key hy.models.Symbol)) (raise (ValueError f"expected a name, but got {key}"))) - (when (.startswith key "^") - (raise (ValueError "reader macro cannot start with a ^"))) - (if (and body (isinstance (get body 0) hy.models.String)) (setv [docstr #* body] body) (setv docstr None)) diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index b8bdb4c8f..31b0e557a 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -353,11 +353,6 @@ def tag_dispatch(self, key): "Premature end of input while attempting dispatch", self ) - if self.peek_and_getc("^"): - typ = self.parse_one_form() - target = self.parse_one_form() - return mkexpr("annotate", target, typ) - tag = None # try dispatching tagged ident ident = self.read_ident(just_peeking=True) @@ -394,6 +389,13 @@ def hash_star(self, _): self.parse_one_form(), ) + @reader_for("#^") + def annotate(self, _): + """Annotate a symbol, usually with a type.""" + typ = self.parse_one_form() + target = self.parse_one_form() + return mkexpr("annotate", target, typ) + ### # Strings # (these are more complicated because f-strings From 1903f9b6f26f18c7bf2de835124e7c67fe87127f Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Tue, 23 May 2023 03:50:07 +0200 Subject: [PATCH 192/342] reader macros now read a whole ident always --- hy/core/macros.hy | 15 ++++++++------ hy/core/result_macros.py | 2 +- hy/macros.py | 2 +- hy/reader/hy_reader.py | 42 +++++++++++++++++++--------------------- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 6ffdb979a..47d3bcb89 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -102,7 +102,7 @@ (setv [docstr #* body] body) (setv docstr None)) - (setv dispatch-key (hy.mangle (+ "#" (str key)))) + (setv dispatch-key (hy.mangle (str key))) `(do (eval-and-compile (hy.macros.reader-macro ~dispatch-key @@ -110,7 +110,7 @@ ~@(if docstr [docstr] []) ~@body))) (eval-when-compile - (setv (get hy.&reader.reader-table ~dispatch-key) + (setv (get hy.&reader.reader-macros ~dispatch-key) (get _hy_reader_macros ~dispatch-key))))) @@ -122,13 +122,16 @@ Use ``(help foo)`` instead for help with runtime objects." (setv symbol (str symbol)) + (setv namespace + (if (= (cut symbol 1) "#") + (do (setv symbol (cut symbol 1 None)) + '_hy_reader_macros) + '_hy_macros)) (setv mangled (hy.mangle symbol)) (setv builtins (hy.gensym "builtins")) `(do (import builtins :as ~builtins) - (help (or (.get _hy_macros ~mangled) - (.get _hy_reader_macros ~mangled) - (.get (. ~builtins _hy_macros) ~mangled) - (.get (. ~builtins _hy_reader_macros) ~mangled) + (help (or (.get ~namespace ~mangled) + (.get (. ~builtins ~namespace) ~mangled) (raise (NameError f"macro {~symbol !r} is not defined")))))) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 0a8ce99e2..61e929826 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1800,7 +1800,7 @@ def compile_require(compiler, expr, root, entries): reader_assignments = ( "ALL" if readers == Symbol("*") - else ["#" + reader for reader in readers[0]] + else [str(reader) for reader in readers[0]] ) if require_reader(module_name, compiler.module, reader_assignments): ret += compiler.compile( diff --git a/hy/macros.py b/hy/macros.py index cc81a690a..c323d706a 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -205,7 +205,7 @@ def enable_readers(module, reader, names): for name in names: if name not in namespace["_hy_reader_macros"]: raise NameError(f"reader {name} is not defined") - reader.reader_table[name] = namespace["_hy_reader_macros"][name] + reader.reader_macros[name] = namespace["_hy_reader_macros"][name] def require(source_module, target_module, assignments, prefix=""): diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 31b0e557a..b65abca3d 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -22,7 +22,7 @@ ) from .exceptions import LexException, PrematureEndOfInput -from .mangling import mangle +from .mangling import mangle, unmangle from .reader import Reader, isnormalizedspace @@ -116,6 +116,17 @@ class HyReader(Reader): NON_IDENT = set("()[]{};\"'`~") + def __init__(self): + super().__init__() + + # move any reader macros declared using + # `reader_for("#...")` to the macro table + self.reader_macros = {} + for tag in list(self.reader_table.keys()): + if tag[0] == '#' and tag[1:]: + self.reader_macros[tag[1:]] = self.reader_table.pop(tag) + + def fill_pos(self, model, start): """Attach line/col information to a model. @@ -342,10 +353,6 @@ def tag_dispatch(self, key): Reads a full identifier after the `#` and calls the corresponding handler (this allows, e.g., `#reads-multiple-forms foo bar baz`). - - Failing that, reads a single character after the `#` and immediately - calls the corresponding handler (this allows, e.g., `#*args` to parse - as `#*` followed by `args`). """ if not self.peekc(): @@ -355,19 +362,14 @@ def tag_dispatch(self, key): tag = None # try dispatching tagged ident - ident = self.read_ident(just_peeking=True) - if ident and mangle(key + ident) in self.reader_table: - self.getn(len(ident)) - tag = mangle(key + ident) - # failing that, dispatch tag + single character - elif key + self.peekc() in self.reader_table: - tag = key + self.getc() - if tag: - tree = self.dispatch(tag) + ident = self.read_ident() or self.getc() + if ident and mangle(ident) in self.reader_macros: + tag = mangle(ident) + tree = self.reader_macros[tag](self, tag) return as_model(tree) if tree is not None else None raise LexException.from_reader( - f"reader macro '{key + self.read_ident()}' is not defined", self + f"reader macro '{key + ident}' is not defined", self ) @reader_for("#_") @@ -377,15 +379,11 @@ def discard(self, _): return None @reader_for("#*") - def hash_star(self, _): + @reader_for("#**") + def hash_star(self, stars): """Unpacking forms `#*` and `#**`, corresponding to `*` and `**` in Python.""" - num_stars = 1 - while self.peek_and_getc("*"): - num_stars += 1 - if num_stars > 2: - raise LexException.from_reader("too many stars", self) return mkexpr( - "unpack-" + ("iterable", "mapping")[num_stars - 1], + "unpack-" + {"*": "iterable", "**": "mapping"}[stars], self.parse_one_form(), ) From 643ec12071bd4e4b42f6a1303a28ed23866c5f4a Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Thu, 25 May 2023 23:34:57 +0200 Subject: [PATCH 193/342] reader macros are no longer mangled --- hy/core/macros.hy | 10 +++++----- hy/macros.py | 4 ++-- hy/reader/hy_reader.py | 10 ++++------ 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 47d3bcb89..e5eb4a779 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -102,7 +102,7 @@ (setv [docstr #* body] body) (setv docstr None)) - (setv dispatch-key (hy.mangle (str key))) + (setv dispatch-key (str key)) `(do (eval-and-compile (hy.macros.reader-macro ~dispatch-key @@ -126,12 +126,12 @@ (if (= (cut symbol 1) "#") (do (setv symbol (cut symbol 1 None)) '_hy_reader_macros) - '_hy_macros)) - (setv mangled (hy.mangle symbol)) + (do (setv symbol (hy.mangle symbol)) + '_hy_macros))) (setv builtins (hy.gensym "builtins")) `(do (import builtins :as ~builtins) - (help (or (.get ~namespace ~mangled) - (.get (. ~builtins ~namespace) ~mangled) + (help (or (.get ~namespace ~symbol) + (.get (. ~builtins ~namespace) ~symbol) (raise (NameError f"macro {~symbol !r} is not defined")))))) diff --git a/hy/macros.py b/hy/macros.py index c323d706a..a124cee0e 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -185,7 +185,7 @@ def require_reader(source_module, target_module, assignments): target_macros = target_namespace.setdefault("_hy_reader_macros", {}) assignments = ( - source_macros.keys() if assignments == "ALL" else map(mangle, assignments) + source_macros.keys() if assignments == "ALL" else assignments ) for name in assignments: @@ -200,7 +200,7 @@ def require_reader(source_module, target_module, assignments): def enable_readers(module, reader, names): _, namespace = derive_target_module(module, inspect.stack()[1][0]) names = ( - namespace["_hy_reader_macros"].keys() if names == "ALL" else map(mangle, names) + namespace["_hy_reader_macros"].keys() if names == "ALL" else names ) for name in names: if name not in namespace["_hy_reader_macros"]: diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index b65abca3d..016b94e50 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -22,7 +22,7 @@ ) from .exceptions import LexException, PrematureEndOfInput -from .mangling import mangle, unmangle +from .mangling import mangle from .reader import Reader, isnormalizedspace @@ -355,17 +355,15 @@ def tag_dispatch(self, key): (this allows, e.g., `#reads-multiple-forms foo bar baz`). """ - if not self.peekc(): + if not self.peekc().strip(): raise PrematureEndOfInput.from_reader( "Premature end of input while attempting dispatch", self ) - tag = None # try dispatching tagged ident ident = self.read_ident() or self.getc() - if ident and mangle(ident) in self.reader_macros: - tag = mangle(ident) - tree = self.reader_macros[tag](self, tag) + if ident in self.reader_macros: + tree = self.reader_macros[ident](self, ident) return as_model(tree) if tree is not None else None raise LexException.from_reader( From 87b6334f4ae4c4955230d3e2047a70d7b5f1cef4 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Thu, 25 May 2023 23:30:55 +0200 Subject: [PATCH 194/342] update tests with spaced reader macros --- tests/native_tests/comprehensions.hy | 2 +- tests/native_tests/functions.hy | 4 +- tests/native_tests/let.hy | 2 +- tests/native_tests/match.hy | 8 +-- tests/native_tests/other.hy | 4 +- tests/native_tests/unpack.hy | 20 +++---- tests/resources/pydemo.hy | 4 +- tests/test_reader.py | 80 ++++++++++++++-------------- 8 files changed, 62 insertions(+), 62 deletions(-) diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy index b92964205..7280cafed 100644 --- a/tests/native_tests/comprehensions.hy +++ b/tests/native_tests/comprehensions.hy @@ -263,7 +263,7 @@ (assert (= out "x1-x2-y1y2-z1-z2-"))) -(defmacro eval-isolated [#*body] +(defmacro eval-isolated [#* body] `(hy.eval '(do ~@body) :module "" :locals {})) diff --git a/tests/native_tests/functions.hy b/tests/native_tests/functions.hy index 2cc69b60f..fc9dd05a7 100644 --- a/tests/native_tests/functions.hy +++ b/tests/native_tests/functions.hy @@ -86,8 +86,8 @@ (defn test-defn-annotations [] - (defn #^int f [#^(get List int) p1 p2 #^str p3 #^str [o1 None] #^int [o2 0] - #^str #* rest #^str k1 #^int [k2 0] #^bool #** kwargs]) + (defn #^ int f [#^ (get List int) p1 p2 #^ str p3 #^ str [o1 None] #^ int [o2 0] + #^ str #* rest #^ str k1 #^ int [k2 0] #^ bool #** kwargs]) (assert (is (. f __annotations__ ["return"]) int)) (for [[k v] (.items (dict diff --git a/tests/native_tests/let.hy b/tests/native_tests/let.hy index 9ff3eadaf..ffed3f376 100644 --- a/tests/native_tests/let.hy +++ b/tests/native_tests/let.hy @@ -505,7 +505,7 @@ #(10 20 30))))) -(defmacro eval-isolated [#*body] +(defmacro eval-isolated [#* body] `(hy.eval '(do ~@body) :module "" :locals {})) diff --git a/tests/native_tests/match.hy b/tests/native_tests/match.hy index cb1111066..152500638 100644 --- a/tests/native_tests/match.hy +++ b/tests/native_tests/match.hy @@ -6,8 +6,8 @@ hy.errors [HySyntaxError]) (defclass [dataclass] Point [] - (#^int x) - (#^int y)) + (#^ int x) + (#^ int y)) (defn test-pattern-matching [] (assert (is (match 0 @@ -260,10 +260,10 @@ _ _))) (assert (= {"a" 1 "c" 3} (match y - {"b" b #**e} e))) + {"b" b #** e} e))) (assert (= {"a" 1 "c" 3} (match y - {"b" b #**a} a))) + {"b" b #** a} a))) (assert (= b 2)))) diff --git a/tests/native_tests/other.hy b/tests/native_tests/other.hy index 30f0f81fd..88bdae132 100644 --- a/tests/native_tests/other.hy +++ b/tests/native_tests/other.hy @@ -44,8 +44,8 @@ (defn test-variable-annotations [] (defclass AnnotationContainer [] - (setv #^int x 1 y 2) - (#^bool z)) + (setv #^ int x 1 y 2) + (#^ bool z)) (setv annotations (get-type-hints AnnotationContainer)) (assert (= (get annotations "x") int)) diff --git a/tests/native_tests/unpack.hy b/tests/native_tests/unpack.hy index e5e484c75..181d7ecd6 100644 --- a/tests/native_tests/unpack.hy +++ b/tests/native_tests/unpack.hy @@ -10,10 +10,10 @@ (defn test-extended-unpacking-1star-lvalues [] - (setv [x #*y] [1 2 3 4]) + (setv [x #* y] [1 2 3 4]) (assert (= x 1)) (assert (= y [2 3 4])) - (setv [a #*b c] "ghijklmno") + (setv [a #* b c] "ghijklmno") (assert (= a "g")) (assert (= b (list "hijklmn"))) (assert (= c "o"))) @@ -22,19 +22,19 @@ (defn test-unpacking-pep448-1star [] (setv l [1 2 3]) (setv p [4 5]) - (assert (= ["a" #*l "b" #*p #*l] ["a" 1 2 3 "b" 4 5 1 2 3])) - (assert (= #("a" #*l "b" #*p #*l) #("a" 1 2 3 "b" 4 5 1 2 3))) - (assert (= #{"a" #*l "b" #*p #*l} #{"a" "b" 1 2 3 4 5})) + (assert (= ["a" #* l "b" #* p #* l] ["a" 1 2 3 "b" 4 5 1 2 3])) + (assert (= #("a" #* l "b" #* p #* l) #("a" 1 2 3 "b" 4 5 1 2 3))) + (assert (= #{"a" #* l "b" #* p #* l} #{"a" "b" 1 2 3 4 5})) (defn f [#* args] args) - (assert (= (f "a" #*l "b" #*p #*l) #("a" 1 2 3 "b" 4 5 1 2 3))) - (assert (= (+ #*l #*p) 15)) - (assert (= (and #*l) 3))) + (assert (= (f "a" #* l "b" #* p #* l) #("a" 1 2 3 "b" 4 5 1 2 3))) + (assert (= (+ #* l #* p) 15)) + (assert (= (and #* l) 3))) (defn test-unpacking-pep448-2star [] (setv d1 {"a" 1 "b" 2}) (setv d2 {"c" 3 "d" 4}) - (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"})) + (assert (= {1 "x" #** d1 #** d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"})) (defn fun [[a None] [b None] [c None] [d None] [e None] [f None]] [a b c d e f]) - (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None]))) + (assert (= (fun #** d1 :e "eee" #** d2) [1 2 3 4 "eee" None]))) diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy index 0a12c7347..80c4c0901 100644 --- a/tests/resources/pydemo.hy +++ b/tests/resources/pydemo.hy @@ -51,8 +51,8 @@ Call me Ishmael. Some years ago—never mind how long precisely—having little (setv boolexpr (and (or True False) (not (and True False)))) (setv condexpr (if "" "x" "y")) (setv mylambda (fn [x] (+ x "z"))) -(setv annotated-lambda-ret (fn #^int [] 1)) -(setv annotated-lambda-params (fn [#^int a * #^str [b "hello world!"]] #(a b))) +(setv annotated-lambda-ret (fn #^ int [] 1)) +(setv annotated-lambda-params (fn [#^ int a * #^ str [b "hello world!"]] #(a b))) (setv fstring1 f"hello {(+ 1 1)} world") (setv p "xyzzy") diff --git a/tests/test_reader.py b/tests/test_reader.py index 3ef6e38c9..873d5b4c8 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -559,77 +559,77 @@ def test_discard(): # empty assert tokenize("") == [] # single - assert tokenize("#_1") == [] + assert tokenize("#_ 1") == [] # multiple - assert tokenize("#_1 #_2") == [] - assert tokenize("#_1 #_2 #_3") == [] + assert tokenize("#_ 1 #_ 2") == [] + assert tokenize("#_ 1 #_ 2 #_ 3") == [] # nested discard - assert tokenize("#_ #_1 2") == [] - assert tokenize("#_ #_ #_1 2 3") == [] + assert tokenize("#_ #_ 1 2") == [] + assert tokenize("#_ #_ #_ 1 2 3") == [] # trailing assert tokenize("0") == [Integer(0)] - assert tokenize("0 #_1") == [Integer(0)] - assert tokenize("0 #_1 #_2") == [Integer(0)] + assert tokenize("0 #_ 1") == [Integer(0)] + assert tokenize("0 #_ 1 #_ 2") == [Integer(0)] # leading assert tokenize("2") == [Integer(2)] - assert tokenize("#_1 2") == [Integer(2)] - assert tokenize("#_0 #_1 2") == [Integer(2)] - assert tokenize("#_ #_0 1 2") == [Integer(2)] + assert tokenize("#_ 1 2") == [Integer(2)] + assert tokenize("#_ 0 #_ 1 2") == [Integer(2)] + assert tokenize("#_ #_ 0 1 2") == [Integer(2)] # both - assert tokenize("#_1 2 #_3") == [Integer(2)] - assert tokenize("#_0 #_1 2 #_ #_3 4") == [Integer(2)] + assert tokenize("#_ 1 2 #_ 3") == [Integer(2)] + assert tokenize("#_ 0 #_ 1 2 #_ #_ 3 4") == [Integer(2)] # inside - assert tokenize("0 #_1 2") == [Integer(0), Integer(2)] - assert tokenize("0 #_1 #_2 3") == [Integer(0), Integer(3)] - assert tokenize("0 #_ #_1 2 3") == [Integer(0), Integer(3)] + assert tokenize("0 #_ 1 2") == [Integer(0), Integer(2)] + assert tokenize("0 #_ 1 #_ 2 3") == [Integer(0), Integer(3)] + assert tokenize("0 #_ #_ 1 2 3") == [Integer(0), Integer(3)] # in List assert tokenize("[]") == [List([])] - assert tokenize("[#_1]") == [List([])] - assert tokenize("[#_1 #_2]") == [List([])] - assert tokenize("[#_ #_1 2]") == [List([])] + assert tokenize("[#_ 1]") == [List([])] + assert tokenize("[#_ 1 #_ 2]") == [List([])] + assert tokenize("[#_ #_ 1 2]") == [List([])] assert tokenize("[0]") == [List([Integer(0)])] - assert tokenize("[0 #_1]") == [List([Integer(0)])] - assert tokenize("[0 #_1 #_2]") == [List([Integer(0)])] + assert tokenize("[0 #_ 1]") == [List([Integer(0)])] + assert tokenize("[0 #_ 1 #_ 2]") == [List([Integer(0)])] assert tokenize("[2]") == [List([Integer(2)])] - assert tokenize("[#_1 2]") == [List([Integer(2)])] - assert tokenize("[#_0 #_1 2]") == [List([Integer(2)])] - assert tokenize("[#_ #_0 1 2]") == [List([Integer(2)])] + assert tokenize("[#_ 1 2]") == [List([Integer(2)])] + assert tokenize("[#_ 0 #_ 1 2]") == [List([Integer(2)])] + assert tokenize("[#_ #_ 0 1 2]") == [List([Integer(2)])] # in Set assert tokenize("#{}") == [Set()] - assert tokenize("#{#_1}") == [Set()] - assert tokenize("#{0 #_1}") == [Set([Integer(0)])] - assert tokenize("#{#_1 0}") == [Set([Integer(0)])] + assert tokenize("#{#_ 1}") == [Set()] + assert tokenize("#{0 #_ 1}") == [Set([Integer(0)])] + assert tokenize("#{#_ 1 0}") == [Set([Integer(0)])] # in Dict assert tokenize("{}") == [Dict()] - assert tokenize("{#_1}") == [Dict()] - assert tokenize("{#_0 1 2}") == [Dict([Integer(1), Integer(2)])] - assert tokenize("{1 #_0 2}") == [Dict([Integer(1), Integer(2)])] - assert tokenize("{1 2 #_0}") == [Dict([Integer(1), Integer(2)])] + assert tokenize("{#_ 1}") == [Dict()] + assert tokenize("{#_ 0 1 2}") == [Dict([Integer(1), Integer(2)])] + assert tokenize("{1 #_ 0 2}") == [Dict([Integer(1), Integer(2)])] + assert tokenize("{1 2 #_ 0}") == [Dict([Integer(1), Integer(2)])] # in Expression assert tokenize("()") == [Expression()] - assert tokenize("(#_foo)") == [Expression()] - assert tokenize("(#_foo bar)") == [Expression([Symbol("bar")])] - assert tokenize("(foo #_bar)") == [Expression([Symbol("foo")])] + assert tokenize("(#_ foo)") == [Expression()] + assert tokenize("(#_ foo bar)") == [Expression([Symbol("bar")])] + assert tokenize("(foo #_ bar)") == [Expression([Symbol("foo")])] assert tokenize("(foo :bar 1)") == [ Expression([Symbol("foo"), Keyword("bar"), Integer(1)]) ] - assert tokenize("(foo #_:bar 1)") == [Expression([Symbol("foo"), Integer(1)])] - assert tokenize("(foo :bar #_1)") == [Expression([Symbol("foo"), Keyword("bar")])] + assert tokenize("(foo #_ :bar 1)") == [Expression([Symbol("foo"), Integer(1)])] + assert tokenize("(foo :bar #_ 1)") == [Expression([Symbol("foo"), Keyword("bar")])] # discard term with nesting - assert tokenize("[1 2 #_[a b c [d e [f g] h]] 3 4]") == [ + assert tokenize("[1 2 #_ [a b c [d e [f g] h]] 3 4]") == [ List([Integer(1), Integer(2), Integer(3), Integer(4)]) ] # discard with other prefix syntax - assert tokenize("a #_'b c") == [Symbol("a"), Symbol("c")] - assert tokenize("a '#_b c") == [ + assert tokenize("a #_ 'b c") == [Symbol("a"), Symbol("c")] + assert tokenize("a '#_ b c") == [ Symbol("a"), Expression([Symbol("quote"), Symbol("c")]), ] - assert tokenize("a '#_b #_c d") == [ + assert tokenize("a '#_ b #_ c d") == [ Symbol("a"), Expression([Symbol("quote"), Symbol("d")]), ] - assert tokenize("a '#_ #_b c d") == [ + assert tokenize("a '#_ #_ b c d") == [ Symbol("a"), Expression([Symbol("quote"), Symbol("d")]), ] From a1f40662ca306488b97a6c50f2f21d7283b37471 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Tue, 23 May 2023 12:07:06 +0200 Subject: [PATCH 195/342] update tests for new reader macro name restrictions --- tests/native_tests/reader_macros.hy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/native_tests/reader_macros.hy b/tests/native_tests/reader_macros.hy index 44047d294..4809084aa 100644 --- a/tests/native_tests/reader_macros.hy +++ b/tests/native_tests/reader_macros.hy @@ -4,6 +4,7 @@ types contextlib [contextmanager] hy.errors [HyMacroExpansionError] + hy.reader.exceptions [PrematureEndOfInput] pytest) @@ -32,6 +33,7 @@ (assert (= (eval-module #[[(defreader foo '1) #foo]]) 1)) (assert (in (hy.mangle "#foo") (eval-module #[[(defreader foo '1) _hy_reader_macros]]))) + (assert (= (eval-module #[[(defreader ^foo '1) #^foo]]) 1)) ;; Assert reader macros operating exclusively at read time (with [module (temp-module "")] @@ -47,9 +49,8 @@ (defn test-bad-reader-macro-name [] (with [(pytest.raises HyMacroExpansionError)] (eval-module "(defreader :a-key '1)")) - - (with [(pytest.raises HyMacroExpansionError)] - (eval-module "(defreader ^foo '1)"))) + (with [(pytest.raises PrematureEndOfInput)] + (eval-module "# _ 3"))) (defn test-require-readers [] (with [module (temp-module "")] From 75b363b88fd6e5204b0263e867517bb28599945f Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Thu, 25 May 2023 23:30:19 +0200 Subject: [PATCH 196/342] update tests for no longer mangling reader macro names --- tests/native_tests/mangling.hy | 14 +++++++++----- tests/native_tests/reader_macros.hy | 27 ++++++++++++++++----------- tests/resources/tlib.hy | 2 +- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/tests/native_tests/mangling.hy b/tests/native_tests/mangling.hy index db359483c..64c896841 100644 --- a/tests/native_tests/mangling.hy +++ b/tests/native_tests/mangling.hy @@ -109,14 +109,18 @@ (assert (= x "aabb"))) -(defreader rm---x +(defreader rm--- (setv form (.parse-one-form &reader)) - [form form]) + `(do (+= ~form "a") + ~form)) +(defreader rm___ + (setv form (.parse-one-form &reader)) + `(do (+= ~form "b") + ~form)) (defn test-reader-macro [] (setv x "") - (assert (= #rm---x (do (+= x "a") 1) [1 1])) - (assert (= #rm___x (do (+= x "b") 2) [2 2])) - (assert (= x "aabb"))) + (assert (= #rm--- x "a")) + (assert (= #rm___ x "ab"))) (defn test-special-form [] diff --git a/tests/native_tests/reader_macros.hy b/tests/native_tests/reader_macros.hy index 4809084aa..9d62e7b22 100644 --- a/tests/native_tests/reader_macros.hy +++ b/tests/native_tests/reader_macros.hy @@ -31,10 +31,15 @@ (defn test-reader-macros [] (assert (= (eval-module #[[(defreader foo '1) #foo]]) 1)) - (assert (in (hy.mangle "#foo") + (assert (in "foo" (eval-module #[[(defreader foo '1) _hy_reader_macros]]))) (assert (= (eval-module #[[(defreader ^foo '1) #^foo]]) 1)) + (assert (not-in "rm___x" + (eval-module + #[[(defreader rm---x '1) + _hy_reader_macros]]))) + ;; Assert reader macros operating exclusively at read time (with [module (temp-module "")] (setv it (hy.read-many #[reader[ @@ -54,33 +59,33 @@ (defn test-require-readers [] (with [module (temp-module "")] - (setv it (hy.read-many #[[(require tests.resources.tlib :readers [upper]) - #upper hello]])) + (setv it (hy.read-many #[[(require tests.resources.tlib :readers [upper!]) + #upper! hello]])) (eval-isolated (next it) module) (assert (= (next it) 'HELLO))) ;; test require :readers & :macros is order independent - (for [s ["[qplah] :readers [upper]" - ":readers [upper] [qplah]" - ":macros [qplah] :readers [upper]" - ":readers [upper] :macros [qplah]"]] + (for [s ["[qplah] :readers [upper!]" + ":readers [upper!] [qplah]" + ":macros [qplah] :readers [upper!]" + ":readers [upper!] :macros [qplah]"]] (assert (= (eval-module #[f[ (require tests.resources.tlib {s}) - [(qplah 1) #upper "hello"]]f]) + [(qplah 1) #upper! "hello"]]f]) [[8 1] "HELLO"]))) ;; test require :readers * (assert (= (eval-module #[=[ (require tests.resources.tlib :readers *) - [#upper "eVeRy" #lower "ReAdEr"]]=]) + [#upper! "eVeRy" #lower "ReAdEr"]]=]) ["EVERY" "reader"])) ;; test can't redefine :macros or :readers assignment brackets (with [(pytest.raises hy.errors.HySyntaxError)] - (eval-module #[[(require tests.resources.tlib [taggart] [upper])]])) + (eval-module #[[(require tests.resources.tlib [taggart] [upper!])]])) (with [(pytest.raises hy.errors.HySyntaxError)] - (eval-module #[[(require tests.resources.tlib :readers [taggart] :readers [upper])]])) + (eval-module #[[(require tests.resources.tlib :readers [taggart] :readers [upper!])]])) (with [(pytest.raises hy.errors.HyRequireError)] (eval-module #[[(require tests.resources.tlib :readers [not-a-real-reader])]]))) diff --git a/tests/resources/tlib.hy b/tests/resources/tlib.hy index 2fb4d28ca..8d18233c4 100644 --- a/tests/resources/tlib.hy +++ b/tests/resources/tlib.hy @@ -9,7 +9,7 @@ (defmacro ✈ [arg] `(+ "plane " ~arg)) -(defreader upper +(defreader upper! (let [node (&reader.parse-one-form)] (if (isinstance node #(hy.models.Symbol hy.models.String)) (.__class__ node (.upper node)) From 7415d704a10600d803f51d990cdca223baafea73 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Fri, 26 May 2023 20:13:01 +0200 Subject: [PATCH 197/342] update NEWS --- NEWS.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 7e9634f08..eb5c3b3fd 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -7,6 +7,14 @@ Removals ------------------------------ * Python 3.7 is no longer supported. +Breaking Changes +------------------------------ +* Reader macros now always read a full identifier after the initial `#`, + allowing for reader macros that start with characters such as `*`, `^`, `_`. + Forms like `#*word` will attempt to dispatch a macro named `*word`; + to unpack a symbol named `word`, write `#* word` (note the space). +* Reader macro names are no longer mangled. + Bug Fixes ------------------------------ * Fixed an installation failure in some situations when version lookup From 47ea0a1ed5f50601cc3430eb4c3b8f8e049c6433 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 27 May 2023 09:46:32 -0400 Subject: [PATCH 198/342] Don't use deprecated AST elements --- hy/_compat.py | 2 +- hy/compiler.py | 10 +++++----- hy/core/result_macros.py | 2 +- tests/compilers/test_ast.py | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/hy/_compat.py b/hy/_compat.py index a752bfec7..4b83686fa 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -27,7 +27,7 @@ def rewriting_unparse(ast_obj): ast_obj = copy.deepcopy(ast_obj) for node in ast.walk(ast_obj): - if type(node) in (ast.Constant, ast.Str): + if type(node) is ast.Constant: # Don't touch string literals. continue for field in node._fields: diff --git a/hy/compiler.py b/hy/compiler.py index efc6913b6..862e4427c 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -580,7 +580,7 @@ def compile_expression(self, expr, *, allow_annotation_expression=False): @builds_model(Integer, Float, Complex) def compile_numeric_literal(self, x): f = {Integer: int, Float: float, Complex: complex}[type(x)] - return asty.Num(x, n=f(x)) + return asty.Constant(x, value=f(x)) @builds_model(Symbol) def compile_symbol(self, symbol): @@ -612,16 +612,15 @@ def compile_keyword(self, obj): attr="Keyword", ctx=ast.Load(), ), - args=[asty.Str(obj, s=obj.name)], + args=[asty.Constant(obj, value=obj.name)], keywords=[], ) return ret @builds_model(String, Bytes) def compile_string(self, string): - node = asty.Bytes if type(string) is Bytes else asty.Str f = bytes if type(string) is Bytes else str - return node(string, s=f(string)) + return asty.Constant(string, value=f(string)) @builds_model(FComponent) def compile_fcomponent(self, fcomponent): @@ -856,7 +855,8 @@ def hy_compile( if ( result.stmts and isinstance(result.stmts[0], ast.Expr) - and isinstance(result.stmts[0].value, ast.Str) + and isinstance(result.stmts[0].value, ast.Constant) + and isinstance(result.stmts[0].value.value, str) ): body += [result.stmts.pop(0)] diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 61e929826..eb296ffd1 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -354,7 +354,7 @@ def compile_chained_comparison(compiler, expr, root, arg1, args): def compile_maths_expression(compiler, expr, root, args): if len(args) == 0: # Return the identity element for this operator. - return asty.Num(expr, n=({"+": 0, "|": 0, "*": 1}[root])) + return asty.Constant(expr, value=({"+": 0, "|": 0, "*": 1}[root])) if len(args) == 1: if root == "/": diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index a44607a8e..6cb4c4213 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -40,7 +40,7 @@ def cant_compile(expr): def s(x): - return can_compile('"module docstring" ' + x).body[-1].value.s + return can_compile('"module docstring" ' + x).body[-1].value.value def test_ast_bad_type(): @@ -374,7 +374,7 @@ def test_lambda_list_keywords_kwonly(): for i, kwonlyarg_name in enumerate(("a", "b")): assert kwonlyarg_name == code.body[0].args.kwonlyargs[i].arg assert code.body[0].args.kw_defaults[0] is None - assert code.body[0].args.kw_defaults[1].n == 2 + assert code.body[0].args.kw_defaults[1].value == 2 def test_lambda_list_keywords_mixed(): @@ -399,8 +399,8 @@ def _compile_string(s): ) # We put hy_s in a list so it isn't interpreted as a docstring. - # code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Str(s=xxx)]))]) - return code.body[0].value.elts[0].s + # code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Constant(value=xxx)]))]) + return code.body[0].value.elts[0].value assert _compile_string("test") == "test" assert _compile_string("\u03b1\u03b2") == "\u03b1\u03b2" From 2f6c2b63e804a1e3cccda6eb8d9f4c49b971762e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 27 May 2023 09:51:45 -0400 Subject: [PATCH 199/342] Inline some single-use variables --- hy/compiler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 862e4427c..20882092b 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -579,8 +579,8 @@ def compile_expression(self, expr, *, allow_annotation_expression=False): @builds_model(Integer, Float, Complex) def compile_numeric_literal(self, x): - f = {Integer: int, Float: float, Complex: complex}[type(x)] - return asty.Constant(x, value=f(x)) + return asty.Constant(x, value = + {Integer: int, Float: float, Complex: complex}[type(x)](x)) @builds_model(Symbol) def compile_symbol(self, symbol): @@ -619,8 +619,8 @@ def compile_keyword(self, obj): @builds_model(String, Bytes) def compile_string(self, string): - f = bytes if type(string) is Bytes else str - return asty.Constant(string, value=f(string)) + return asty.Constant(string, value = + (bytes if type(string) is Bytes else str)(string)) @builds_model(FComponent) def compile_fcomponent(self, fcomponent): From 96a085d335fd79309db7c0911a06530a3daaf157 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 27 May 2023 17:41:05 -0400 Subject: [PATCH 200/342] Don't use the deprecated `pkgutil.get_loader` --- hy/errors.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hy/errors.py b/hy/errors.py index e59a3f746..3f46a9648 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -1,8 +1,8 @@ import os -import pkgutil import re import sys import traceback +import importlib.util from contextlib import contextmanager from functools import reduce @@ -197,7 +197,11 @@ class HyWrapperError(HyError, TypeError): def _module_filter_name(module_name): try: - compiler_loader = pkgutil.get_loader(module_name) + spec = importlib.util.find_spec(module_name) + if not spec: + return None + + compiler_loader = spec.loader if not compiler_loader: return None From ebef72bd9d885cf3a5e895e001e66ea829773ced Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 27 May 2023 17:41:29 -0400 Subject: [PATCH 201/342] Rewrite `hy.macros._same_modules` To avoid use of the deprecated `pkgutil.get_loader`, and to be more concise. --- hy/macros.py | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/hy/macros.py b/hy/macros.py index a124cee0e..3f159a21b 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -2,7 +2,6 @@ import importlib import inspect import os -import pkgutil import re import sys import traceback @@ -110,33 +109,23 @@ def _same_modules(source_module, target_module): if not (source_module or target_module): return False - if target_module == source_module: + if target_module is source_module: return True - def _get_filename(module): - filename = None - try: - if not inspect.ismodule(module): - loader = pkgutil.get_loader(module) - if isinstance(loader, importlib.machinery.SourceFileLoader): - filename = loader.get_filename() - else: - filename = inspect.getfile(module) - except (TypeError, ImportError): - pass - - return filename - - source_filename = _get_filename(source_module) - target_filename = _get_filename(target_module) + def get_filename(module): + if inspect.ismodule(module): + return inspect.getfile(module) + elif ( + (spec := importlib.util.find_spec(module)) and + isinstance(spec.loader, importlib.machinery.SourceFileLoader)): + return spec.loader.get_filename() - return ( - source_filename - and target_filename - and os.path.exists(source_filename) - and os.path.exists(target_filename) - and os.path.samefile(source_filename, target_filename) - ) + try: + return os.path.samefile( + get_filename(source_module), + get_filename(target_module)) + except (ValueError, TypeError, ImportError, FileNotFoundError): + return False def derive_target_module(target_module, parent_frame): From 969b80a719fc9b498b7c5e3f48e62a6483216eee Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 27 May 2023 09:57:51 -0400 Subject: [PATCH 202/342] Shorten the name of Mac OS tests on GitHub --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 92f4c4480..4e6045594 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: - name-prefix: 'win-' os: windows-latest python: 3.11 - - name-prefix: 'macos-' + - name-prefix: 'mac-' os: macos-latest python: 3.11 From c9541eed8f3699fb9b2487494a47d63a227090e0 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 27 May 2023 17:41:55 -0400 Subject: [PATCH 203/342] Remove an unused import --- hy/errors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hy/errors.py b/hy/errors.py index 3f46a9648..440f0dc30 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -4,7 +4,6 @@ import traceback import importlib.util from contextlib import contextmanager -from functools import reduce from hy import _initialize_env_var from hy._compat import PYPY From 5f88e558b61516f404cb6904ae082beee57313f5 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 3 Jun 2023 09:09:32 -0400 Subject: [PATCH 204/342] Remove some leftover linting control comments --- tests/compilers/test_ast.py | 2 -- tests/test_bin.py | 1 - 2 files changed, 3 deletions(-) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 6cb4c4213..4a743e23f 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -1,5 +1,3 @@ -# fmt: off - import ast from textwrap import dedent diff --git a/tests/test_bin.py b/tests/test_bin.py index 32e3c84f0..56ae08792 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# fmt: off import builtins import os From 73a6732b87cedcdaa94826a3d7a4c6269568fd59 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 3 Jun 2023 10:28:44 -0400 Subject: [PATCH 205/342] Support relative `require`s in `hy -m` and `hy2py` This has necessitated changing the semantics of recursive `hy2py` a fair amount: it now expects a module name, not just any old directory. --- NEWS.rst | 5 ++++ docs/cli.rst | 2 +- hy/cmdline.py | 13 ++++++++-- hy/core/result_macros.py | 2 ++ hy/macros.py | 6 +++-- tests/test_bin.py | 53 ++++++++++++++++++++++++++++++---------- 6 files changed, 63 insertions(+), 18 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index eb5c3b3fd..1136f216d 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -14,6 +14,9 @@ Breaking Changes Forms like `#*word` will attempt to dispatch a macro named `*word`; to unpack a symbol named `word`, write `#* word` (note the space). * Reader macro names are no longer mangled. +* `hy2py`'s recursive mode now expects a module name as input, not any + old directory. You must be in the parent directory of the module + directory. Bug Fixes ------------------------------ @@ -22,6 +25,8 @@ Bug Fixes * Fixed some bugs with traceback pointing. * Fixed some bugs with escaping in bracket f-strings * The parser no longer looks for shebangs in the REPL or `hy -c`. +* `require` with relative module names should now work correctly with + `hy -m`, as well as `hy2py`'s recursive mode. New Features ------------------------------ diff --git a/docs/cli.rst b/docs/cli.rst index f40c18f58..0f8227a07 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -44,7 +44,7 @@ for a complete list of options and :py:ref:`Python's documentation hy2py ----- -``hy2py`` is a program to convert Hy source code into Python source code. Use ``hy2py --help`` for usage instructions. It can take its input from standard input, from a filename, or folder name provided as a command-line argument. If it is a folder, the output parameter (--output/-o) must be provided. When the output parameter is provided, the output will be written into the folder or file, otherwise the result is written to standard output. +``hy2py`` is a program to convert Hy source code into Python source code. Use ``hy2py --help`` for usage instructions. It can take its input from standard input, or from a file or module name provided as a command-line argument. In the case of a module name, the current working directory should be the parent directory of that module, and the output parameter (``--output/-o``) is required. When the output parameter is provided, the output will be written into the folder or file. Otherwise, the result is written to standard output. .. warning:: ``hy2py`` can execute arbitrary code. Don't give it untrusted input. diff --git a/hy/cmdline.py b/hy/cmdline.py index dca3febb0..73e02eac7 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -5,6 +5,7 @@ import os import platform import py_compile +import re import runpy import sys from contextlib import nullcontext @@ -332,7 +333,9 @@ def hyc_main(): def hy2py_worker(source, options, filename, output_filepath=None): + source_path = None if isinstance(source, Path): + source_path = source source = source.read_text(encoding="UTF-8") if not output_filepath and options.output: @@ -358,7 +361,13 @@ def printing_source(hst): hst.filename = filename with filtered_hy_exceptions(): - _ast = hy_compile(hst, "__main__", filename=filename, source=source) + _ast = hy_compile( + hst, + re.sub(r'\.hy$', '', '.'.join(source_path.parts)) + if source_path + else '__main__', + filename=filename, + source=source) if options.with_source: print() @@ -385,7 +394,7 @@ def hy2py_main(): "FILE", type=str, nargs="?", - help='Input Hy code (can be file or directory) (use STDIN if "-" or ' + help='Input Hy code (can be file or module) (use STDIN if "-" or ' "not provided)", ) parser.add_argument( diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index eb296ffd1..4d42a0034 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1783,6 +1783,8 @@ def compile_require(compiler, expr, root, entries): dotted("hy.macros.require"), String(module_name), Symbol("None"), + Keyword("target_module_name"), + String(compiler.module.__name__), Keyword("assignments"), ( String("EXPORTS") diff --git a/hy/macros.py b/hy/macros.py index 3f159a21b..d684310fc 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -197,7 +197,7 @@ def enable_readers(module, reader, names): reader.reader_macros[name] = namespace["_hy_reader_macros"][name] -def require(source_module, target_module, assignments, prefix=""): +def require(source_module, target_module, assignments, prefix="", target_module_name=None): """Load macros from one module into the namespace of another. This function is called from the macro also named `require`. @@ -213,6 +213,7 @@ def require(source_module, target_module, assignments, prefix=""): prefix (str): If nonempty, its value is prepended to the name of each imported macro. This allows one to emulate namespaced macros, like "mymacromodule.mymacro", which looks like an attribute of a module. Defaults to "" + target_module_name: If true, overrides the apparent name of `target_module`. Returns: bool: Whether or not macros were actually transferred. @@ -230,7 +231,8 @@ def require(source_module, target_module, assignments, prefix=""): return False if not inspect.ismodule(source_module): - source_module = import_module_from_string(source_module, target_module) + source_module = import_module_from_string(source_module, + target_module_name or target_module) source_macros = source_module.__dict__.setdefault("_hy_macros", {}) source_exports = getattr( diff --git a/tests/test_bin.py b/tests/test_bin.py index 56ae08792..b31682434 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -741,26 +741,53 @@ def test_assert(tmp_path, monkeypatch): assert ("bye" in err) == show_msg -def test_hy2py_recursive(tmp_path): - (tmp_path / 'hy').mkdir() - (tmp_path / "hy/first.hy").write_text(""" - (import folder.second [a b]) +def test_hy2py_recursive(monkeypatch, tmp_path): + (tmp_path / 'foo').mkdir() + (tmp_path / 'foo/__init__.py').touch() + (tmp_path / "foo/first.hy").write_text(""" + (import foo.folder.second [a b]) (print a) (print b)""") - (tmp_path / "hy/folder").mkdir() - (tmp_path / "hy/folder/second.hy").write_text(""" + (tmp_path / "foo/folder").mkdir() + (tmp_path / "foo/folder/__init__.py").touch() + (tmp_path / "foo/folder/second.hy").write_text(""" (setv a 1) (setv b "hello world")""") - _, err = run_cmd(["hy2py", (tmp_path / 'hy')], expect=1) + monkeypatch.chdir(tmp_path) + + _, err = run_cmd("hy2py foo", expect=1) assert "ValueError" in err - run_cmd(["hy2py", - (tmp_path / 'hy'), - "--output", (tmp_path / 'py')]) - assert set((tmp_path / 'py').rglob('*')) == { - tmp_path / 'py' / p + run_cmd("hy2py foo --output bar") + assert set((tmp_path / 'bar').rglob('*')) == { + tmp_path / 'bar' / p for p in ('first.py', 'folder', 'folder/second.py')} - output, _ = run_cmd("python3 first.py", cwd = tmp_path / 'py') + output, _ = run_cmd("python3 first.py", cwd = tmp_path / 'bar') assert output == "1\nhello world\n" + + +@pytest.mark.parametrize('case', ['hy -m', 'hy2py']) +def test_relative_require(case, monkeypatch, tmp_path): + # https://github.com/hylang/hy/issues/2204 + + (tmp_path / 'pkg').mkdir() + (tmp_path / 'pkg' / '__init__.py').touch() + (tmp_path / 'pkg' / 'a.hy').write_text(''' + (defmacro m [] + '(setv x (.upper "hello")))''') + (tmp_path / 'pkg' / 'b.hy').write_text(''' + (require .a [m]) + (m) + (print x)''') + monkeypatch.chdir(tmp_path) + + if case == 'hy -m': + output, _ = run_cmd('hy -m pkg.b') + elif case == 'hy2py': + run_cmd('hy2py pkg -o out') + (tmp_path / 'out' / '__init__.py').touch() + output, _ = run_cmd('python3 -m out.b') + + assert 'HELLO' in output From 3aab8903097b85dcdbfd4e2638fa629882ed4136 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 5 Jun 2023 15:52:18 -0400 Subject: [PATCH 206/342] Don't mangle question marks specially --- hy/reader/mangling.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/hy/reader/mangling.py b/hy/reader/mangling.py index ce9f85822..e71e6d4cb 100644 --- a/hy/reader/mangling.py +++ b/hy/reader/mangling.py @@ -35,19 +35,15 @@ def mangle(s): if "." in s and s.strip("."): return ".".join(mangle(x) if x else "" for x in s.split(".")) - # Step 1: Remove and save leading underscores + # Remove and save leading underscores s2 = s.lstrip(normalizes_to_underscore) leading_underscores = "_" * (len(s) - len(s2)) s = s2 - # Step 2: Convert hyphens without introducing a new leading underscore + # Convert hyphens without introducing a new leading underscore s = s[0] + s[1:].replace("-", "_") if s else s - # Step 3: Convert trailing `?` to leading `is_` - if s.endswith("?"): - s = "is_" + s[:-1] - - # Step 4: Convert invalid characters or reserved words + # Convert invalid characters or reserved words if not (leading_underscores + s).isidentifier(): # Replace illegal characters with their Unicode character # names, or hexadecimal if they don't have one. @@ -126,8 +122,6 @@ def unmangle(s): ), s[len("hyx_") :], ) - if s.startswith("is_"): - s = s[len("is_") :] + "?" s = s.replace("_", "-") return prefix + s + suffix From 8539d956b29883966ae91f923e6e8e327b879e87 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 5 Jun 2023 15:53:23 -0400 Subject: [PATCH 207/342] Update the tests for the mangling change --- tests/native_tests/mangling.hy | 42 ++++++++++++++-------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/tests/native_tests/mangling.hy b/tests/native_tests/mangling.hy index 64c896841..5f361c8e7 100644 --- a/tests/native_tests/mangling.hy +++ b/tests/native_tests/mangling.hy @@ -28,8 +28,6 @@ (assert (= (hy.mangle "--__") "hyx_XhyphenHminusX___")) (assert (= (hy.mangle "__--") "__hyx_XhyphenHminusX_")) (assert (= (hy.mangle "__--__") "__hyx_XhyphenHminusX___")) - (assert (= (hy.mangle "--?") "hyx_is_XhyphenHminusX_")) - (assert (= (hy.mangle "__--?") "__hyx_is_XhyphenHminusX_")) ;; test unmangling choices (assert (= (hy.unmangle "hyx_XhyphenHminusX") "-")) @@ -49,12 +47,10 @@ (defn test-question-mark [] + ; Once treated specially, but no more. (setv foo? "nachos") (assert (= foo? "nachos")) - (assert (= is_foo "nachos")) - (setv ___ab_cd? "tacos") - (assert (= ___ab_cd? "tacos")) - (assert (= ___is_ab_cd "tacos"))) + (assert (= hyx_fooXquestion_markX "nachos"))) (defn test-py-forbidden-ascii [] @@ -71,10 +67,7 @@ (assert (= (+ hyx_XflowerXab hyx_Xblack_heart_suitX) "flowerlove")) (setv ⚘-⚘ "doubleflower") (assert (= ⚘-⚘ "doubleflower")) - (assert (= hyx_XflowerX_XflowerX "doubleflower")) - (setv ⚘? "mystery") - (assert (= ⚘? "mystery")) - (assert (= hyx_is_XflowerX "mystery"))) + (assert (= hyx_XflowerX_XflowerX "doubleflower"))) (defn test-higher-unicode [] @@ -156,34 +149,33 @@ (defn test-keyword-args [] - (defn f [a a-b foo? ☘] - [a a-b foo? ☘]) - (assert (= (f :foo? 3 :☘ 4 :a 1 :a-b 2) [1 2 3 4])) - (assert (= (f :is_foo 3 :hyx_XshamrockX 4 :a 1 :a_b 2) [1 2 3 4])) + (defn f [a a-b ☘] + [a a-b ☘]) + (assert (= (f :☘ 3 :a 1 :a-b 2) [1 2 3])) + (assert (= (f :hyx_XshamrockX 3 :a 1 :a_b 2) [1 2 3])) (defn g [#** x] x) - (assert (= (g :foo? 3 :☘ 4 :a 1 :a-b 2) - {"a" 1 "a_b" 2 "is_foo" 3 "hyx_XshamrockX" 4})) - (assert (= (g :is_foo 3 :hyx_XshamrockX 4 :a 1 :a_b 2) - {"a" 1 "a_b" 2 "is_foo" 3 "hyx_XshamrockX" 4}))) + (assert (= (g :☘ 3 :a 1 :a-b 2) + {"a" 1 "a_b" 2 "hyx_XshamrockX" 3})) + (assert (= (g :hyx_XshamrockX 3 :a 1 :a_b 2) + {"a" 1 "a_b" 2 "hyx_XshamrockX" 3}))) (defn test-late-mangling [] ; Mangling should only happen during compilation. - (assert (!= 'foo? 'is_foo)) - (setv sym 'foo?) - (assert (= sym (hy.models.Symbol "foo?"))) - (assert (!= sym (hy.models.Symbol "is_foo"))) + (assert (!= 'foo-bar 'foo_bar)) + (setv sym 'foo-bar) + (assert (= sym (hy.models.Symbol "foo-bar"))) + (assert (!= sym (hy.models.Symbol "foo_bar"))) (setv out (hy.eval `(do (setv ~sym 10) - [foo? is_foo]))) + [foo-bar foo_bar]))) (assert (= out [10 10]))) (defn test-functions [] - (for [[a b] [["___ab-cd?" "___is_ab_cd"] - ["⚘-⚘" "hyx_XflowerX_XflowerX"]]] + (for [[a b] [["⚘-⚘" "hyx_XflowerX_XflowerX"]]] (assert (= (hy.mangle a) b)) (assert (= (hy.unmangle b) a)))) From 7aa4d467df4cd67986844eb5bda29686cd3829da Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 5 Jun 2023 16:01:08 -0400 Subject: [PATCH 208/342] Update the docs and NEWS for the mangling change --- NEWS.rst | 2 ++ docs/syntax.rst | 19 ++++++------------- hy/pyops.hy | 2 +- hy/reader/mangling.py | 10 ++-------- 4 files changed, 11 insertions(+), 22 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 1136f216d..cbd903774 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -14,6 +14,8 @@ Breaking Changes Forms like `#*word` will attempt to dispatch a macro named `*word`; to unpack a symbol named `word`, write `#* word` (note the space). * Reader macro names are no longer mangled. +* Question marks (`?`) are no longer mangled specially, so `foo?` now + mangles to `hyx_fooXquestion_markX` instead of `is_foo`. * `hy2py`'s recursive mode now expects a module name as input, not any old directory. You must be in the parent directory of the module directory. diff --git a/docs/syntax.rst b/docs/syntax.rst index 6fd748221..5773cbb73 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -266,10 +266,6 @@ Python-legal names. The steps are as follows: underscore into the name. Thus ``--has-dashes?`` becomes ``-_has_dashes?`` at this step. -#. If the name ends with ASCII ``?``, remove it and prepend ``is_``. Thus, - ``tasty?`` becomes ``is_tasty`` and ``-_has_dashes?`` becomes - ``is_-_has_dashes``. - #. If the name still isn't Python-legal, make the following changes. A name could be Python-illegal because it contains a character that's never legal in a Python name or it contains a character that's illegal in that position. @@ -282,12 +278,11 @@ Python-legal names. The steps are as follows: code point in lowercase hexadecimal. Thus, ``green☘`` becomes ``hyx_greenXshamrockX`` and - ``is_-_has_dashes`` becomes ``hyx_is_XhyphenHminusX_has_dashes``. + ``-_has_dashes`` becomes ``hyx_XhyphenHminusX_has_dashes``. #. Take any leading underscores removed in the first step, transliterate them - to ASCII, and add them back to the mangled name. Thus, ``(hy.mangle - '_tasty?)`` is ``"_is_tasty"`` instead of ``"is__tasty"`` and ``(hy.mangle - '__-_has-dashes?)`` is ``"__hyx_is_XhyphenHminusX_has_dashes"``. + to ASCII, and add them back to the mangled name. Thus, ``__green☘`` becomes + ``__hyx_greenXshamrockX``. #. Finally, normalize any leftover non-ASCII characters. The result may still not be ASCII (e.g., ``α`` is already Python-legal and normalized, so it @@ -300,11 +295,9 @@ You can invoke the mangler yourself with the function :hy:func:`hy.mangle`, and Mangling isn't something you should have to think about often, but you may see mangled names in error messages, the output of ``hy2py``, etc. A catch to be aware of is that mangling, as well as the inverse "unmangling" operation -offered by :hy:func:`hy.unmangle`, isn't one-to-one. Two different symbols -can mangle to the same string and hence compile to the same Python variable. -The chief practical consequence of this is that (non-initial) ``-`` and ``_`` are -interchangeable under mangling, so you can't use e.g. ``foo-bar`` and -``foo_bar`` as separate variables. +offered by :hy:func:`hy.unmangle`, isn't one-to-one. Two different symbols, +like ``foo-bar`` and ``foo_bar``, can mangle to the same string and hence +compile to the same Python variable. .. _string-literals: diff --git a/hy/pyops.hy b/hy/pyops.hy index 34342d79c..95941d13a 100644 --- a/hy/pyops.hy +++ b/hy/pyops.hy @@ -6,7 +6,7 @@ to the names being the same: - ``==`` in Python is :hy:func:`= ` in Hy. - ``~`` in Python is :hy:func:`bnot ` in Hy. -- ``is not`` in Python is :hy:func:`is-not ` in Hy. +- ``is not`` in Python is :hy:func:`is-not ` in Hy. - ``not in`` in Python is :hy:func:`not-in ` in Hy. For Python's subscription expressions (like ``x[2]``), Hy has two named diff --git a/hy/reader/mangling.py b/hy/reader/mangling.py index e71e6d4cb..b12100dd0 100644 --- a/hy/reader/mangling.py +++ b/hy/reader/mangling.py @@ -11,7 +11,7 @@ def mangle(s): :hy:func:`hy.repr`) and convert it to a valid Python identifier according to :ref:`Hy's mangling rules `. :: - (hy.mangle 'foo-bar?) ; => "is_foo_bar" + (hy.mangle 'foo-bar) ; => "foo_bar" (hy.mangle "🦑") ; => "hyx_squid" If the stringified argument is already both legal as a Python identifier @@ -26,7 +26,7 @@ def mangle(s): `, and ``hy.mangle`` will mangle the dot-delimited parts separately. :: - (hy.mangle "a.b?.c!.d") ; => "a.is_b.hyx_cXexclamation_markX.d" + (hy.mangle "a.c!.d") ; => "a.hyx_cXexclamation_markX.d" """ assert s @@ -84,15 +84,9 @@ def unmangle(s): => (hy.unmangle 'foo_bar) "foo-bar" - => (hy.unmangle 'is_foo_bar) - "foo-bar?" - => (hy.unmangle 'hyx_XasteriskX) "*" - => (hy.unmangle '_hyx_is_fooXsolidusXa) - "_foo/a?" - => (hy.unmangle 'hyx_XhyphenHminusX_XgreaterHthan_signX) "-->" From e48f92f0b011c153b642138b2a266f6cb2c2845c Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 7 Jun 2023 13:40:25 -0400 Subject: [PATCH 209/342] Update a test that used a removed macro --- tests/macros/test_macro_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py index 5fb317911..464dcca81 100644 --- a/tests/macros/test_macro_processor.py +++ b/tests/macros/test_macro_processor.py @@ -55,7 +55,7 @@ def test_macroexpand_nan(): def test_macroexpand_source_data(): # https://github.com/hylang/hy/issues/1944 - ast = Expression([Symbol("#@"), String("a")]) + ast = Expression([Symbol("when"), String("a")]) ast.start_line = 3 ast.start_column = 5 bad = macroexpand_1(ast, "hy.core.macros") From 9916b7d18a675506944a8bad81c1d35916a919a0 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 7 Jun 2023 13:49:15 -0400 Subject: [PATCH 210/342] Forbid `Symbol("#foo")` --- NEWS.rst | 2 ++ hy/reader/hy_reader.py | 2 +- tests/test_models.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index cbd903774..d4765e986 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -29,6 +29,8 @@ Bug Fixes * The parser no longer looks for shebangs in the REPL or `hy -c`. * `require` with relative module names should now work correctly with `hy -m`, as well as `hy2py`'s recursive mode. +* `hy.models.Symbol` no longer allows constructing a symbol beginning + with `#`. New Features ------------------------------ diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 016b94e50..923b820e2 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -98,7 +98,7 @@ def err(msg): if reader is None: if ( not ident - or ident[:1] == ":" + or ident[0] in ":#" or any(isnormalizedspace(c) for c in ident) or HyReader.NON_IDENT.intersection(ident) ): diff --git a/tests/test_models.py b/tests/test_models.py index c000e8bd1..9ad851545 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -28,7 +28,8 @@ def test_symbol_or_keyword(): for x in ("foo", "foo-bar", "foo_bar", "✈é😂⁂"): assert str(Symbol(x)) == x assert Keyword(x).name == x - for x in ("", ":foo", "5"): + for x in ("", ":foo", "5", "#foo"): + # https://github.com/hylang/hy/issues/2383 with pytest.raises(ValueError): Symbol(x) assert Keyword(x).name == x From 6dad27e2e10cc3e850a5de4b6ccbbba81cf3a8ac Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 12 Jun 2023 11:49:55 -0400 Subject: [PATCH 211/342] Add spaces in `#^` examples where required --- docs/api.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 13b020118..c0ee1d743 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -23,7 +23,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. and takes the type second:: (setv (annotate x int) 1) - (setv #^int x 1) + (setv #^ int x 1) The order difference is not merely visual: ``#^`` actually evaluates the type first. @@ -31,23 +31,23 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Here are examples with ``#^`` for all the places you can use annotations:: ; Annotate the variable `x` as an `int` (equivalent to `x: int`). - #^int x + #^ int x ; You can annotate with expressions (equivalent to `y: f(x)`). #^(f x) y ; Annotations with an assignment: each annotation `(int, str)` ; covers the term that immediately follows. ; Equivalent to `x: int = 1; y = 2; z: str = 3` - (setv #^int x 1 y 2 #^str z 3) + (setv #^ int x 1 y 2 #^ str z 3) ; Annotate `a` as an `int`, `c` as an `int`, and `b` as a `str`. ; Equivalent to `def func(a: int, b: str = None, c: int = 1): ...` - (defn func [#^int a #^str [b None] #^int [c 1]] ...) + (defn func [#^ int a #^ str [b None] #^ int [c 1]] ...) ; Function return annotations come before the function name (if ; it exists). - (defn #^int add1 [#^int x] (+ x 1)) - (fn #^int [#^int y] (+ y 2)) + (defn #^ int add1 [#^ int x] (+ x 1)) + (fn #^ int [#^ int y] (+ y 2)) For annotating items with generic types, the :hy:func:`of ` macro will likely be of use. @@ -697,8 +697,8 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (import dataclasses [dataclass]) (defclass [dataclass] Point [] - #^int x - #^int y) + #^ int x + #^ int y) (match (Point 1 2) (Point 1 x) :if (= (% x 2) 0) x) ; => 2 From ee7a1baa9279d201cf89fa1e41b88338b2ec5996 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 27 Jun 2023 08:39:38 -0400 Subject: [PATCH 212/342] Slightly expand doc. of evaluating a string of Hy --- docs/interop.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/interop.rst b/docs/interop.rst index 99f7e38c2..cf0d8a4a4 100644 --- a/docs/interop.rst +++ b/docs/interop.rst @@ -42,10 +42,16 @@ You can use :ref:`hy2py` to convert a Hy program to Python. The output will still import ``hy``, and thus require Hy to be installed in order to run; see :ref:`implicit-names` for details and workarounds. -To execute Hy code from a string, use :func:`hy.read` to convert it to -:ref:`models ` and then :func:`hy.eval` to evaluate it. There is no Hy -equivalent of :func:`exec` because :func:`hy.eval` works even when the input -isn't equivalent to a single Python expression. +To execute Hy code from a string, use :hy:func:`hy.read-many` to convert it to +:ref:`models ` and then :hy:func:`hy.eval` to evaluate it: + +.. code-block:: python + + >>> hy.eval(hy.read_many("(setv x 1) (+ x 1)")) + 2 + +There is no Hy equivalent of :func:`exec` because :hy:func:`hy.eval` works +even when the input isn't equivalent to a single Python expression. You can use :meth:`hy.REPL.run` to launch the Hy REPL from Python, as in ``hy.REPL(locals = locals()).run()``. From 2f3beb97f059b258583d05e2ed1d684a2a415a26 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 27 Jun 2023 08:40:25 -0400 Subject: [PATCH 213/342] Test PyPy for Python 3.10 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4e6045594..e60b3774f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: matrix: name-prefix: [''] os: [ubuntu-latest] - python: [3.8, 3.9, '3.10', 3.11, 3.12-dev, pypy-3.9, pyodide] + python: [3.8, 3.9, '3.10', 3.11, 3.12-dev, pypy-3.10, pyodide] include: # To keep the overall number of runs low, we test Windows and MacOS # only on the latest CPython. From 15abf28d52a736dceca4801990e730d481fb990f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 27 Jun 2023 08:42:30 -0400 Subject: [PATCH 214/342] Remove workarounds for fixed PyPy `__file__` bug --- hy/cmdline.py | 4 ++-- tests/test_bin.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hy/cmdline.py b/hy/cmdline.py index 73e02eac7..c48fc1d9e 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -12,7 +12,7 @@ from pathlib import Path import hy -from hy._compat import PY3_9, PYPY +from hy._compat import PY3_9 from hy.compiler import hy_compile, hy_eval from hy.errors import HyLanguageError, filtered_hy_exceptions, hy_exc_handler from hy.importer import runhy @@ -270,7 +270,7 @@ def proc_opt(opt, arg=None, item=None, i=None): set_path(filename) # Ensure __file__ is set correctly in the code we're about # to run. - if PY3_9 and not PYPY: + if PY3_9: if not filename.is_absolute(): filename = Path.cwd() / filename if platform.system() == "Windows": diff --git a/tests/test_bin.py b/tests/test_bin.py index b31682434..4f7e4805a 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -11,7 +11,7 @@ import pytest -from hy._compat import PY3_9, PYODIDE, PYPY +from hy._compat import PY3_9, PYODIDE if PYODIDE: pytest.skip( @@ -681,7 +681,6 @@ def test_output_buffering(tmp_path): assert tf.read_text().splitlines() == ["line 1", "line 2"] -@pytest.mark.skipif(PYPY, reason = 'https://foss.heptapod.net/pypy/pypy/-/issues/3881') def test_uufileuu(tmp_path, monkeypatch): # `__file__` should be set the same way as in Python. # https://github.com/hylang/hy/issues/2318 @@ -691,7 +690,7 @@ def test_uufileuu(tmp_path, monkeypatch): (tmp_path / "realdir" / "pyex.py").write_text('print(__file__)') def file_is(arg, expected_py3_9): - expected = expected_py3_9 if PY3_9 and not PYPY else Path(arg) + expected = expected_py3_9 if PY3_9 else Path(arg) output, _ = run_cmd(["python3", arg + "pyex.py"]) assert output.rstrip() == str(expected / "pyex.py") output, _ = run_cmd(["hy", arg + "hyex.hy"]) From b29ecf6d530ca774796cdafd61a1cdfa8d871ac0 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 29 Jun 2023 13:56:30 -0400 Subject: [PATCH 215/342] Remove obsolete test patterns in `setup.cfg` --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 630ba1208..20f7df279 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [tool:pytest] # Be sure to include Hy test functions with mangled names. -python_functions=test_* is_test_* hyx_test_* hyx_is_test_* +python_functions=test_* hyx_test_* filterwarnings = once::DeprecationWarning once::PendingDeprecationWarning From 5c8b28816e8ee9492fdb1c653106b5e018b4f935 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 29 Jun 2023 12:45:00 -0400 Subject: [PATCH 216/342] Fix `PYTHONPATH` management in `test_hy2py` This seems to be necessary for running the tests on my machine (but not on GitHub Actions) since 73a6732b87cedcdaa94826a3d7a4c6269568fd59. --- tests/test_hy2py.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py index c5743984b..b88cd08a6 100644 --- a/tests/test_hy2py.py +++ b/tests/test_hy2py.py @@ -23,13 +23,15 @@ def test_hy2py_import(): import subprocess path = "tests/resources/pydemo_as_py.py" + env = dict(os.environ) + env["PYTHONIOENCODING"] = "UTF-8" + env["PYTHONPATH"] = "." + os.pathsep + env.get("PYTHONPATH", "") try: with open(path, "wb") as o: subprocess.check_call( ["hy2py", "tests/resources/pydemo.hy"], stdout=o, - env={**os.environ, "PYTHONIOENCODING": "UTF-8"}, - ) + env=env) import tests.resources.pydemo_as_py as m finally: with contextlib.suppress(FileNotFoundError): From 0da115c090033a9193b8cfffb74a3bfff896598f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 29 Jun 2023 12:28:23 -0400 Subject: [PATCH 217/342] Clean up some `try` tests --- tests/native_tests/try.hy | 60 ++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/tests/native_tests/try.hy b/tests/native_tests/try.hy index bf226469c..3fb550f17 100644 --- a/tests/native_tests/try.hy +++ b/tests/native_tests/try.hy @@ -1,25 +1,30 @@ ;; Tests of `try` and `raise` -(defn test-try [] +(defn test-try-trivial [] (try (do) (except [])) + (try (do) (except [IOError]) (except []))) - (try (do) (except [IOError]) (except [])) - ; test that multiple statements in a try get evaluated +(defn test-try-multiple-statements [] (setv value 0) (try (+= value 1) (+= value 2) (except [IOError]) (except [])) - (assert (= value 3)) + (assert (= value 3))) + - ; test that multiple expressions in a try get evaluated +(defn test-try-multiple-expressions [] ; https://github.com/hylang/hy/issues/1584 + (setv l []) (defn f [] (.append l 1)) (try (f) (f) (f) (except [IOError])) (assert (= l [1 1 1])) (setv l []) (try (f) (f) (f) (except [IOError]) (else (f))) - (assert (= l [1 1 1 1])) + (assert (= l [1 1 1 1]))) + + +(defn test-raise-nullary [] ;; Test correct (raise) (setv passed False) @@ -38,7 +43,10 @@ (raise) (except [RuntimeError] (setv passed True))) - (assert passed) + (assert passed)) + + +(defn test-try [] ;; Test (finally) (setv passed False) @@ -151,10 +159,13 @@ (setv x 45)) (else (setv x 44))) (except [])) - (assert (= x 0)) + (assert (= x 0))) + + +(defn test-nonsyntactical-except [] + #[[Test that [except ...] and ("except" ...) aren't treated like (except ...), + and that the code there is evaluated normally.]] - ; test that [except ...] and ("except" ...) aren't treated like (except ...), - ; and that the code there is evaluated normally (setv x 0) (try (+= x 1) @@ -186,26 +197,35 @@ ; https://github.com/hylang/hy/issues/798 (assert (= "ef" ((fn [] - (try (+ "a" "b") - (except [NameError] (+ "c" "d")) - (else (+ "e" "f"))))))) + (try + (+ "a" "b") + (except [NameError] + (+ "c" "d")) + (else + (+ "e" "f"))))))) (setv foo - (try (+ "A" "B") - (except [NameError] (+ "C" "D")) - (else (+ "E" "F")))) + (try + (+ "A" "B") + (except [NameError] + (+ "C" "D")) + (else + (+ "E" "F")))) (assert (= foo "EF")) - ; Check that the lvalue isn't assigned in the main `try` body - ; there's an `else`. + ; Check that the lvalue isn't assigned by the main `try` body + ; when there's an `else`. (setv x 1) (setv y 0) (setv x - (try (+ "G" "H") - (except [NameError] (+ "I" "J")) + (try + (+ "G" "H") + (except [NameError] + (+ "I" "J")) (else (setv y 1) (assert (= x 1)) + ; `x` still has its value from before the `try`. (+ "K" "L")))) (assert (= x "KL")) (assert (= y 1))) From b9d94b679b4cb125ef64653d537317c3ab097c44 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 29 Jun 2023 13:54:07 -0400 Subject: [PATCH 218/342] Test `try` more systematically --- tests/native_tests/try.hy | 160 +++++++++++--------------------------- 1 file changed, 47 insertions(+), 113 deletions(-) diff --git a/tests/native_tests/try.hy b/tests/native_tests/try.hy index 3fb550f17..1c59ec133 100644 --- a/tests/native_tests/try.hy +++ b/tests/native_tests/try.hy @@ -1,5 +1,8 @@ ;; Tests of `try` and `raise` +(import + pytest) + (defn test-try-trivial [] (try (do) (except [])) @@ -46,120 +49,51 @@ (assert passed)) -(defn test-try [] - - ;; Test (finally) - (setv passed False) - (try - (do) - (finally (setv passed True))) - (assert passed) - - ;; Test (finally) + (raise) - (setv passed False) - (try - (raise Exception) - (except []) - (finally (setv passed True))) - (assert passed) - - - ;; Test (finally) + (raise) + (else) - (setv passed False - not-elsed True) - (try - (raise Exception) - (except []) - (else (setv not-elsed False)) - (finally (setv passed True))) - (assert passed) - (assert not-elsed) - - (try - (raise (KeyError)) - (except [[IOError]] (assert False)) - (except [e [KeyError]] (assert e))) - - (try - (raise (KeyError)) - (except [[IOError]] (assert False)) - (except [e [KeyError]] (assert e))) - - (try - (get [1] 3) - (except [IndexError] (assert True)) - (except [IndexError] (do))) - - (try - (print foobar42ofthebaz) - (except [IndexError] (assert False)) - (except [NameError] (do))) - - (try - (get [1] 3) - (except [e IndexError] (assert (isinstance e IndexError)))) - - (try - (get [1] 3) - (except [e [IndexError NameError]] (assert (isinstance e IndexError)))) - - (try - (print foobar42ofthebaz) - (except [e [IndexError NameError]] (assert (isinstance e NameError)))) +(defn test-try-clauses [] - (try - (print foobar42) - (except [[IndexError NameError]] (do))) - - (try - (get [1] 3) - (except [[IndexError NameError]] (do))) - - (try - (print foobar42ofthebaz) - (except [])) - - (try - (print foobar42ofthebaz) - (except [] (do))) - - (try - (print foobar42ofthebaz) - (except [] - (setv foobar42ofthebaz 42) - (assert (= foobar42ofthebaz 42)))) - - (setv passed False) - (try - (try (do) (except []) (else (bla))) - (except [NameError] (setv passed True))) - (assert passed) - - (setv x 0) - (try - (raise IOError) - (except [IOError] - (setv x 45)) - (else (setv x 44))) - (assert (= x 45)) - - (setv x 0) - (try - (raise KeyError) - (except [] - (setv x 45)) - (else (setv x 44))) - (assert (= x 45)) - - (setv x 0) - (try - (try - (raise KeyError) - (except [IOError] - (setv x 45)) - (else (setv x 44))) - (except [])) - (assert (= x 0))) + (defmacro try-it [body v1 v2] + `(assert (= (_try-it (fn [] ~body)) [~v1 ~v2]))) + (defn _try-it [callback] + (setv did-finally-clause? False) + (try + (callback) + (except [ZeroDivisionError] + (setv out ["aaa" None])) + (except [[IndexError NameError]] + (setv out ["bbb" None])) + (except [e TypeError] + (setv out ["ccc" (type e)])) + (except [e [KeyError AttributeError]] + (setv out ["ddd" (type e)])) + (except [] + (setv out ["eee" None])) + (else + (setv out ["zzz" None])) + (finally + (setv did-finally-clause? True))) + (assert did-finally-clause?) + out) + + (try-it (/ 1 0) "aaa" None) + (try-it (get "foo" 5) "bbb" None) + (try-it unbound "bbb" None) + (try-it (abs "hi") "ccc" TypeError) + (try-it (get {1 2} 3) "ddd" KeyError) + (try-it True.a "ddd" AttributeError) + (try-it (raise ValueError) "eee" None) + (try-it "hi" "zzz" None)) + + +(defn test-finally-executes-for-uncaught-exception [] + (setv x "") + (with [(pytest.raises ZeroDivisionError)] + (try + (+= x "a") + (/ 1 0) + (+= x "b") + (finally + (+= x "c")))) + (assert (= x "ac"))) (defn test-nonsyntactical-except [] From f656f510c23e33d43ecdb215c593d984a4fcd8b2 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 29 Jun 2023 17:10:17 -0400 Subject: [PATCH 219/342] Liberalize `try` It's clear what the semantics of `try` ought to be when `except` and `except*` are missing, and while not useful as hand-written code, such calls could arise from macros. --- NEWS.rst | 2 ++ docs/api.rst | 6 ++---- hy/core/result_macros.py | 18 +++++++++--------- tests/compilers/test_ast.py | 6 ------ tests/native_tests/try.hy | 11 ++++++++--- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index d4765e986..20775c221 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -37,6 +37,8 @@ New Features * Python 3.12 is now supported. * `nonlocal` and `global` can now be called with no arguments, in which case they're no-ops. +* `try` no longer requires `except`, `except*`, or `finally`, and it + allows `else` even without `except` or `except*`. * The `py` macro now implicitly parenthesizes the input code, so Python's indentation restrictions don't apply. * `cut` now has a function version in `hy.pyops`. diff --git a/docs/api.rst b/docs/api.rst index c0ee1d743..4ee722dd9 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -996,10 +996,8 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. list of exception types, followed by more body forms. Finally there are an optional ``else`` form and an optional ``finally`` form, which again are expressions that begin with the symbol in question and then comprise body - forms. As in Python, at least one of ``except``, ``except*``, or ``finally`` - is required; ``else`` is only allowed if at least one ``except`` or - ``except*`` is provided; ``except*`` requires Python 3.11; and ``except`` - and ``except*`` may not both be used in the same ``try``. + forms. Note that ``except*`` requires Python 3.11, and ``except*`` and + ``except`` may not both be used in the same ``try``. Here's an example of several of the allowed kinds of child forms:: diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 4d42a0034..34fb2e81e 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1292,7 +1292,16 @@ def compile_raise_expression(compiler, expr, root, exc, cause): ], ) def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbody): + if orelse is not None and not catchers: + # Python forbids `else` when there are no `except` clauses. + # But we can get the same effect by appending the `else` forms + # to the body. + body += list(orelse) + orelse = None body = compiler._compile_branch(body) + if not (catchers or finalbody): + # Python forbids this, so just return the body, per `do`. + return body return_var = asty.Name(expr, id=mangle(compiler.get_anon_var()), ctx=ast.Store()) @@ -1367,15 +1376,6 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo finalbody += finalbody.expr_as_stmt() finalbody = finalbody.stmts - # Using (else) without (except) is verboten! - if orelse and not handlers: - raise compiler._syntax_error(expr, "`try' cannot have `else' without `except'") - # Likewise a bare (try) or (try BODY). - if not (handlers or finalbody): - raise compiler._syntax_error( - expr, "`try' must have an `except' or `finally' clause" - ) - returnable = Result( expr=asty.Name(expr, id=return_var.id, ctx=ast.Load()), temp_variables=[return_var], diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 4a743e23f..d3d91a5d6 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -113,13 +113,7 @@ def test_ast_good_try(): def test_ast_bad_try(): - cant_compile("(try)") - cant_compile("(try 1)") - cant_compile("(try 1 bla)") - cant_compile("(try 1 bla bla)") - cant_compile("(try (do bla bla))") cant_compile("(try (do) (else 1) (else 2))") - cant_compile("(try 1 (else 1))") cant_compile("(try 1 (else 1) (except []))") cant_compile("(try 1 (finally 1) (except []))") cant_compile("(try 1 (except []) (finally 1) (else 1))") diff --git a/tests/native_tests/try.hy b/tests/native_tests/try.hy index 1c59ec133..54ba5bd00 100644 --- a/tests/native_tests/try.hy +++ b/tests/native_tests/try.hy @@ -4,9 +4,14 @@ pytest) -(defn test-try-trivial [] - (try (do) (except [])) - (try (do) (except [IOError]) (except []))) +(defn test-try-missing-parts [] + (assert (is (try) None)) + (assert (= (try 1) 1)) + (assert (is (try (except [])) None)) + (assert (is (try (finally)) None)) + (assert (= (try 1 (finally 2)) 1)) + (assert (is (try (else)) None)) + (assert (= (try 1 (else 2)) 2))) (defn test-try-multiple-statements [] From 922444656b42cf448e40b6eb6ec9acce54fd107e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 1 Jul 2023 14:59:05 -0400 Subject: [PATCH 220/342] Fix Pyodide testing on GitHub Actions --- .github/workflows/tests.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e60b3774f..13619bea9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,17 +36,20 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + - if: ${{ matrix.python == 'pyodide' }} + uses: actions/setup-python@v4 + with: + python-version: 3.11 - if: ${{ matrix.python == 'pyodide' }} uses: actions/setup-node@v3 - name: Install shell: bash run: | if [[ ${{ matrix.python }} = pyodide ]] ; then - npm install pyodide@0.22.1 - # 0.23.0 has a regression: https://github.com/pyodide/pyodide/issues/3730 - pip install 'pip >= 22.3.1' - # Older pips may fail to install `pyodide-build`. - pip install 'pyodide-build == 0.22.1' + pip install 'pydantic < 2' + # https://github.com/pyodide/pyodide/pull/3971 + npm install pyodide + pip install pyodide-build pyodide venv .venv-pyodide source .venv-pyodide/bin/activate fi From c28d74dc0eda78676c6cdb4540ef9d4bedcc73e8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 1 Jul 2023 14:24:07 -0400 Subject: [PATCH 221/342] In NEWS, list bug fixes after new features --- NEWS.rst | 94 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 20775c221..057746c6f 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -20,18 +20,6 @@ Breaking Changes old directory. You must be in the parent directory of the module directory. -Bug Fixes ------------------------------- -* Fixed an installation failure in some situations when version lookup - fails. -* Fixed some bugs with traceback pointing. -* Fixed some bugs with escaping in bracket f-strings -* The parser no longer looks for shebangs in the REPL or `hy -c`. -* `require` with relative module names should now work correctly with - `hy -m`, as well as `hy2py`'s recursive mode. -* `hy.models.Symbol` no longer allows constructing a symbol beginning - with `#`. - New Features ------------------------------ * Python 3.12 is now supported. @@ -46,6 +34,18 @@ New Features * `hy --spy` now prints a delimiter between the Python equivalent of your code and the result of evaluating the code for easier reading. +Bug Fixes +------------------------------ +* Fixed an installation failure in some situations when version lookup + fails. +* Fixed some bugs with traceback pointing. +* Fixed some bugs with escaping in bracket f-strings +* The parser no longer looks for shebangs in the REPL or `hy -c`. +* `require` with relative module names should now work correctly with + `hy -m`, as well as `hy2py`'s recursive mode. +* `hy.models.Symbol` no longer allows constructing a symbol beginning + with `#`. + 0.26.0 (released 2023-02-08) ============================= @@ -75,15 +75,6 @@ Breaking Changes * Redundant scripts named `hy3`, `hyc3`, and `hy2py3` are no longer installed. Use `hy`, `hyc`, and `hy2py` instead. -Bug Fixes ------------------------------- -* `hy.REPL` now restores the global values it changes (such as - `sys.ps1`) after `hy.REPL.run` terminates. -* `hy.REPL` no longer mixes up Hy's and Python's Readline histories - when run inside Python's REPL. -* Fixed `hy.repr` of non-compilable uses of sugared macros, such as - `(quote)` and `(quote 1 2)`. - New Features ------------------------------ * Pyodide is now officially supported. @@ -94,6 +85,15 @@ New Features * `hy2py` has a new command-line option `--output`. * `hy2py` can now operate recursively on a directory. +Bug Fixes +------------------------------ +* `hy.REPL` now restores the global values it changes (such as + `sys.ps1`) after `hy.REPL.run` terminates. +* `hy.REPL` no longer mixes up Hy's and Python's Readline histories + when run inside Python's REPL. +* Fixed `hy.repr` of non-compilable uses of sugared macros, such as + `(quote)` and `(quote 1 2)`. + .. _zipimport: https://docs.python.org/3.11/library/zipimport.html 0.25.0 (released 2022-11-08) @@ -180,24 +180,6 @@ Other Breaking Changes * `hy.cmdline.run_repl` has been replaced with `hy.cmdline.HyREPL.run`. -Bug Fixes ------------------------------- -* Fixed a crash when using keyword objects in `match`. -* Fixed a scoping bug in comprehensions in `let` bodies. -* Literal newlines (of all three styles) are now recognized properly - in string and bytes literals. -* `defmacro` no longer allows further arguments after `#* args`. -* `!=` with model objects is now consistent with `=`. -* Tracebacks from code parsed with `hy.read` now show source - positions. -* Elements of `builtins` such as `help` are no longer overridden until - the REPL actually starts. -* Readline is now imported only when necessary, to avoid triggering a - CPython bug regarding the standard module `curses` - (`cpython#46927`_). -* Module names supplied to `hy -m` are now mangled. -* Hy now precompiles its own Hy code during installation. - New Features ------------------------------ * Added user-defined reader macros, defined with `defreader`. @@ -220,6 +202,24 @@ New Features * Added a command-line option `-u` (or `--unbuffered`) per CPython. * Tab-completion in the REPL now attempts to unmangle names. +Bug Fixes +------------------------------ +* Fixed a crash when using keyword objects in `match`. +* Fixed a scoping bug in comprehensions in `let` bodies. +* Literal newlines (of all three styles) are now recognized properly + in string and bytes literals. +* `defmacro` no longer allows further arguments after `#* args`. +* `!=` with model objects is now consistent with `=`. +* Tracebacks from code parsed with `hy.read` now show source + positions. +* Elements of `builtins` such as `help` are no longer overridden until + the REPL actually starts. +* Readline is now imported only when necessary, to avoid triggering a + CPython bug regarding the standard module `curses` + (`cpython#46927`_). +* Module names supplied to `hy -m` are now mangled. +* Hy now precompiles its own Hy code during installation. + .. _cpython#46927: https://github.com/python/cpython/issues/46927#issuecomment-1093418916 .. _cpython#90678: https://github.com/python/cpython/issues/90678 @@ -258,6 +258,14 @@ Other Breaking Changes would be syntactically legal as a literal. * `hy.extra.reserved` has been renamed to `hy.reserved`. +New Features +------------------------------ +* `hy.repr` now supports several more standard types. +* The attribute access macro `.` now allows method calls. For example, + `(. x (f a))` is equivalent to `(x.f a)`. +* `hy.as-model` checks for self-references in its argument. +* New function `hy.model_patterns.keepsym`. + Bug Fixes ------------------------------ * In comprehension forms other than `for`, assignments (other than @@ -274,14 +282,6 @@ Bug Fixes * Fixed a bug with self-requiring files on Windows. * Improved error messages for illegal uses of `finally` and `else`. -New Features ------------------------------- -* `hy.repr` now supports several more standard types. -* The attribute access macro `.` now allows method calls. For example, - `(. x (f a))` is equivalent to `(x.f a)`. -* `hy.as-model` checks for self-references in its argument. -* New function `hy.model_patterns.keepsym`. - .. _Hyrule: https://github.com/hylang/hyrule 1.0a3 (released 2021-07-09) From 53960b5d977e82fd2b7e6e5a29dc1e71d11e02ac Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 6 Jul 2023 12:54:30 -0400 Subject: [PATCH 222/342] Clean up NEWS for release --- NEWS.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 057746c6f..e15dcbe57 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,6 +1,6 @@ .. default-role:: code -Unreleased +0.27.0 (released 2023-07-06) ============================= Removals @@ -9,11 +9,10 @@ Removals Breaking Changes ------------------------------ -* Reader macros now always read a full identifier after the initial `#`, - allowing for reader macros that start with characters such as `*`, `^`, `_`. - Forms like `#*word` will attempt to dispatch a macro named `*word`; - to unpack a symbol named `word`, write `#* word` (note the space). -* Reader macro names are no longer mangled. +* Reader macros now always read a full identifier after the initial + `#`. Thus, `#*foo` is now parsed as a call to the reader macro named + `*foo`; to unpack a variable named `foo`, say `#* foo`. +* The names of reader macros names are no longer mangled. * Question marks (`?`) are no longer mangled specially, so `foo?` now mangles to `hyx_fooXquestion_markX` instead of `is_foo`. * `hy2py`'s recursive mode now expects a module name as input, not any @@ -23,16 +22,17 @@ Breaking Changes New Features ------------------------------ * Python 3.12 is now supported. -* `nonlocal` and `global` can now be called with no arguments, in which - case they're no-ops. +* New built-in object `hy.M` for easy imports in macros. +* `cut` now has a function version in `hy.pyops`. +* The `py` macro now implicitly parenthesizes the input code, so + Python's indentation restrictions don't apply. * `try` no longer requires `except`, `except*`, or `finally`, and it allows `else` even without `except` or `except*`. -* The `py` macro now implicitly parenthesizes the input code, so Python's - indentation restrictions don't apply. -* `cut` now has a function version in `hy.pyops`. -* New built-in object `hy.M` for easy imports in macros. -* `hy --spy` now prints a delimiter between the Python equivalent of - your code and the result of evaluating the code for easier reading. +* `nonlocal` and `global` can now be called with no arguments, in + which case they're no-ops. +* For easier reading, `hy --spy` now prints a delimiter after the + Python equivalent of your code, before the result of evaluating the + code. Bug Fixes ------------------------------ From 583dbc30a6de04cbfe4d9a71656579e13949de84 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 6 Jul 2023 13:17:26 -0400 Subject: [PATCH 223/342] Adjust `requirements-dev` to ease dep. resolution --- requirements-dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index caad94f2e..05f75721e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ pytest >= 6 # documentation -Pygments >= 2 +Pygments == 2.15.1 Sphinx == 5.0.2 -sphinx_rtd_theme >= 1 +sphinx_rtd_theme == 1.2.2 git+https://github.com/hylang/sphinxcontrib-hydomain.git From 4a2712d84b2c7f38a91495bf7708de51a05bb65d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 6 Jul 2023 15:33:48 -0400 Subject: [PATCH 224/342] Add `.readthedocs.yaml` --- .readthedocs.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..ede665a04 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + builder: html + configuration: docs/conf.py + +python: + install: + - method: pip + path: . + - requirements: requirements-dev.txt From 56c0ba5b6d600cbe35585941fa61bb8e9dd7ae12 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 6 Jul 2023 18:02:56 -0400 Subject: [PATCH 225/342] Make Kodi officially the maintainer --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index be86ea615..9d1e32221 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ Project * Community: Join us on [Github Discussions](https://github.com/hylang/hy/discussions)! * [Stack Overflow: The [hy] tag](https://stackoverflow.com/questions/tagged/hy) +Hy's current maintainer is [Kodi Arfer](https://github.com/Kodiologist). He takes responsibility for answering user questions, which should primarily be asked on Stack Overflow or GitHub Discussions, but feel free to [poke him](http://arfer.net/elsewhere) if he's missed a question or you've found a serious security issue. + ![Cuddles the Hacker](https://i.imgur.com/QbPMXTN.png) (fan art from the one and only [doctormo](http://doctormo.deviantart.com/art/Cuddles-the-Hacker-372184766)) From e6b35bf995f2fbb337edf53cd14e1cbf064ecc83 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Mon, 10 Jul 2023 17:37:59 +0200 Subject: [PATCH 226/342] remove some redundant code --- hy/compiler.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 20882092b..32e10cd86 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -821,15 +821,6 @@ def hy_compile( """ module = get_compiler_module(module, compiler, False) - if isinstance(module, str): - if module.startswith("<") and module.endswith(">"): - module = types.ModuleType(module) - else: - module = importlib.import_module(mangle(module)) - - if not inspect.ismodule(module): - raise TypeError("Invalid module type: {}".format(type(module))) - filename = getattr(tree, "filename", filename) source = getattr(tree, "source", source) From a03918b81dbcd66c4ec64f1d16ea97db1190e6cc Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Mon, 10 Jul 2023 17:52:51 +0200 Subject: [PATCH 227/342] hy2py now uses '-m' flag for parsing modules; also fixes bug where hy2py was running the module to be compiled --- NEWS.rst | 9 ++++++++ hy/cmdline.py | 59 +++++++++++++++++++++++++++++++---------------- tests/test_bin.py | 24 +++++++++++++++---- 3 files changed, 67 insertions(+), 25 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index e15dcbe57..028a0f351 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,5 +1,14 @@ .. default-role:: code +Unreleased +============================= + +Breaking Changes +------------------------------ +* `hy2py` now requires `-m` to specify modules, and uses + the same `sys.path` rules as Python when parsing a module + vs a standalone script. + 0.27.0 (released 2023-07-06) ============================= diff --git a/hy/cmdline.py b/hy/cmdline.py index c48fc1d9e..2218ab1d9 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -8,6 +8,7 @@ import re import runpy import sys +import types from contextlib import nullcontext from pathlib import Path @@ -332,16 +333,18 @@ def hyc_main(): return rv -def hy2py_worker(source, options, filename, output_filepath=None): +def hy2py_worker(source, options, filename=None, parent_module=None, output_filepath=None): source_path = None if isinstance(source, Path): source_path = source source = source.read_text(encoding="UTF-8") + if parent_module is None: + set_path(source_path) if not output_filepath and options.output: output_filepath = options.output - set_path(filename) + with ( open(output_filepath, "w", encoding="utf-8") if output_filepath @@ -361,13 +364,19 @@ def printing_source(hst): hst.filename = filename with filtered_hy_exceptions(): - _ast = hy_compile( - hst, - re.sub(r'\.hy$', '', '.'.join(source_path.parts)) - if source_path - else '__main__', - filename=filename, - source=source) + module_name = source_path.stem + if parent_module: + module_name = f"{parent_module}.{module_name}" + module = types.ModuleType(module_name) + sys.modules[module_name] = module + try: + _ast = hy_compile( + hst, + module, + filename=filename, + source=source) + finally: + del sys.modules[module_name] if options.with_source: print() @@ -386,17 +395,19 @@ def printing_source(hst): def hy2py_main(): options = dict( prog="hy2py", - usage="%(prog)s [options] [FILE]", + usage="%(prog)s [options] [-m MODULE | FILE | -]", formatter_class=argparse.RawDescriptionHelpFormatter, ) parser = argparse.ArgumentParser(**options) - parser.add_argument( + gp = parser.add_argument_group().add_mutually_exclusive_group() + gp.add_argument("-m", dest="module", help="convert Hy module (or all files in module)") + gp.add_argument( "FILE", type=str, nargs="?", - help='Input Hy code (can be file or module) (use STDIN if "-" or ' - "not provided)", + help='convert Hy source file', ) + gp.add_argument("-", dest="use_stdin", action="store_true", help="read Hy from stdin") parser.add_argument( "--with-source", "-s", @@ -422,12 +433,17 @@ def hy2py_main(): options = parser.parse_args(sys.argv[1:]) - if options.FILE is None or options.FILE == "-": + if options.use_stdin or (options.FILE is None and options.module is None): sys.path.insert(0, "") filename = "" hy2py_worker(sys.stdin.read(), options, filename) - else: - filename = options.FILE + elif options.module: + if options.module[:1] == ".": + raise ValueError( + "Relative module names not supported" + ) + sys.path.insert(0, "") + filename = options.module.replace(".", os.sep) if os.path.isdir(filename): # handle recursively if --output is specified if not options.output: @@ -443,20 +459,23 @@ def hy2py_main(): # make sure to follow original file structure subdirectory = os.path.relpath(path, filename) output_directory_path = os.path.join( - options.output if options.output else path, subdirectory + options.output, subdirectory ) os.makedirs(output_directory_path, exist_ok=True) hy2py_worker( Path(filepath), options, - filename, + parent_module=path.replace(os.sep, "."), output_filepath=os.path.join( output_directory_path, filename_raw + ".py" ), ) - else: - hy2py_worker(Path(options.FILE), options, filename) + filename += ".hy" + parent_module = ".".join(options.module.split(".")[:-1]) + hy2py_worker(Path(filename), options, parent_module=parent_module) + else: + hy2py_worker(Path(options.FILE), options, options.FILE) parser.exit(0) diff --git a/tests/test_bin.py b/tests/test_bin.py index 4f7e4805a..dc7a71a38 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -740,6 +740,20 @@ def test_assert(tmp_path, monkeypatch): assert ("bye" in err) == show_msg +def test_hy2py_compile_only(monkeypatch): + def check(args): + output, _ = run_cmd(f"hy2py {args}") + assert not re.search(r"^hello world$", output, re.M) + + monkeypatch.chdir('tests/resources') + check("hello_world.hy") + check("-m hello_world") + + monkeypatch.chdir('..') + check("resources/hello_world.hy") + check("-m resources.hello_world") + + def test_hy2py_recursive(monkeypatch, tmp_path): (tmp_path / 'foo').mkdir() (tmp_path / 'foo/__init__.py').touch() @@ -755,10 +769,10 @@ def test_hy2py_recursive(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) - _, err = run_cmd("hy2py foo", expect=1) + _, err = run_cmd("hy2py -m foo", expect=1) assert "ValueError" in err - run_cmd("hy2py foo --output bar") + run_cmd("hy2py -m foo --output bar") assert set((tmp_path / 'bar').rglob('*')) == { tmp_path / 'bar' / p for p in ('first.py', 'folder', 'folder/second.py')} @@ -767,7 +781,7 @@ def test_hy2py_recursive(monkeypatch, tmp_path): assert output == "1\nhello world\n" -@pytest.mark.parametrize('case', ['hy -m', 'hy2py']) +@pytest.mark.parametrize('case', ['hy -m', 'hy2py -m']) def test_relative_require(case, monkeypatch, tmp_path): # https://github.com/hylang/hy/issues/2204 @@ -784,8 +798,8 @@ def test_relative_require(case, monkeypatch, tmp_path): if case == 'hy -m': output, _ = run_cmd('hy -m pkg.b') - elif case == 'hy2py': - run_cmd('hy2py pkg -o out') + elif case == 'hy2py -m': + run_cmd('hy2py -m pkg -o out') (tmp_path / 'out' / '__init__.py').touch() output, _ = run_cmd('python3 -m out.b') From d400d83766892bb84eb02c88d3795361d426d046 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 19 Jul 2023 12:18:56 -0400 Subject: [PATCH 228/342] Test executing a directory or ZIP file --- tests/test_bin.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_bin.py b/tests/test_bin.py index dc7a71a38..d95aa5061 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -804,3 +804,17 @@ def test_relative_require(case, monkeypatch, tmp_path): output, _ = run_cmd('python3 -m out.b') assert 'HELLO' in output + + +def test_run_dir_or_zip(tmp_path): + + (tmp_path / 'dir').mkdir() + (tmp_path / 'dir' / '__main__.hy').write_text('(print (+ "A" "Z"))') + out, _ = run_cmd(['hy', tmp_path / 'dir']) + assert 'AZ' in out + + from zipfile import ZipFile + with ZipFile(tmp_path / 'zoom.zip', 'w') as o: + o.writestr('__main__.hy', '(print (+ "B" "Y"))') + out, _ = run_cmd(['hy', tmp_path / 'zoom.zip']) + assert 'BY' in out From 2e8ab66c57acb586872baffa9a57bc86c0954685 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 18 Jul 2023 15:59:53 -0400 Subject: [PATCH 229/342] Clean up `native_tests/repl.hy` --- tests/native_tests/repl.hy | 39 +++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/tests/native_tests/repl.hy b/tests/native_tests/repl.hy index 384d0f92a..a5abea802 100644 --- a/tests/native_tests/repl.hy +++ b/tests/native_tests/repl.hy @@ -5,23 +5,36 @@ sys pytest) -(defn test-preserve-ps1 [monkeypatch] +(defn [pytest.fixture] rt [monkeypatch capsys] + "Do a test run of the REPL." + (fn [[inp ""] [to-return 'out]] + (monkeypatch.setattr "sys.stdin" (io.StringIO inp)) + (.run (hy.REPL)) + (setv result (capsys.readouterr)) + (cond + (= to-return 'out) result.out + (= to-return 'err) result.err + (= to-return 'both) result))) + +(defmacro has [haystack needle] + "`in` backwards." + `(in ~needle ~haystack)) + + +(defn test-preserve-ps1 [rt] ; https://github.com/hylang/hy/issues/1323#issuecomment-1310837340 - (monkeypatch.setattr "sys.stdin" (io.StringIO "(+ 1 1)")) (setv sys.ps1 "chippy") (assert (= sys.ps1 "chippy")) - (.run (hy.REPL)) + (rt "(+ 1 1)") (assert (= sys.ps1 "chippy"))) -(defn test-repl-input-1char [monkeypatch capsys] +(defn test-input-1char [rt] ; https://github.com/hylang/hy/issues/2430 - (monkeypatch.setattr "sys.stdin" (io.StringIO "1\n")) - (.run (hy.REPL)) - (assert (= (. capsys (readouterr) out) "=> 1\n=> " ))) + (assert (= + (rt "1\n") + "=> 1\n=> "))) -(defn test-repl-no-shebangs [monkeypatch capsys] - (monkeypatch.setattr "sys.stdin" (io.StringIO "#!/usr/bin/env hy\n")) - (.run (hy.REPL)) - (assert (in - "hy.reader.exceptions.LexException" - (. capsys (readouterr) err)))) +(defn test-no-shebangs-allowed [rt] + (assert (has + (rt "#!/usr/bin/env hy\n" 'err) + "hy.reader.exceptions.LexException"))) From cac7fb01da49c08b1ab3861399898936f88961ff Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 18 Jul 2023 16:27:38 -0400 Subject: [PATCH 230/342] Remove a redundant test assertion `.rindex` raises an error if the substring isn't found. So this assertion is just checking that the substring wasn't found at position 0, which isn't very useful. --- tests/test_bin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_bin.py b/tests/test_bin.py index d95aa5061..6c1ee35f3 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -131,7 +131,6 @@ def test_stdin_error_underline_alignment(): _, err = run_cmd("hy", "(defmacro mabcdefghi [x] x)\n(mabcdefghi)") msg_idx = err.rindex(" (mabcdefghi)") - assert msg_idx err_parts = err[msg_idx:].splitlines() assert err_parts[1].startswith(" ^----------^") assert err_parts[2].startswith("expanding macro mabcdefghi") From a42acaba9129c7b432e94a9d77f4b8ec9db5caff Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 19 Jul 2023 11:39:28 -0400 Subject: [PATCH 231/342] Translate a lot of `test_bin` tests to `repl.hy` This decouples the tests in question from the behavior of the command-line interface, and it makes them run faster because they're all in-process now. --- tests/native_tests/repl.hy | 148 ++++++++++++++++++++++++++++++++- tests/test_bin.py | 162 +------------------------------------ 2 files changed, 147 insertions(+), 163 deletions(-) diff --git a/tests/native_tests/repl.hy b/tests/native_tests/repl.hy index a5abea802..f8c187374 100644 --- a/tests/native_tests/repl.hy +++ b/tests/native_tests/repl.hy @@ -1,15 +1,19 @@ -; Many other tests of the REPL are in `test_bin.py`. +; Some other tests of the REPL are in `test_bin.py`. (import io sys + re pytest) + (defn [pytest.fixture] rt [monkeypatch capsys] "Do a test run of the REPL." - (fn [[inp ""] [to-return 'out]] + (fn [[inp ""] [to-return 'out] [spy False] [py-repr False]] (monkeypatch.setattr "sys.stdin" (io.StringIO inp)) - (.run (hy.REPL)) + (.run (hy.REPL + :spy spy + :output-fn (when py-repr repr))) (setv result (capsys.readouterr)) (cond (= to-return 'out) result.out @@ -21,6 +25,144 @@ `(in ~needle ~haystack)) +(defn test-simple [rt] + (assert (has (rt #[[(.upper "hello")]]) "HELLO"))) + +(defn test-spy [rt] + (setv x (rt #[[(.upper "hello")]] :spy True)) + (assert (has x ".upper()")) + (assert (has x "HELLO")) + ; `spy` should work even when an exception is thrown + (assert (has (rt "(foof)" :spy True) "foof()"))) + +(defn test-multiline [rt] + (assert (has (rt "(+ 1 3\n5 9)") " 18\n=> "))) + +(defn test-history [rt] + (assert (has + (rt #[[ + (+ "a" "b") + (+ "c" "d") + (+ "e" "f") + (.format "*1: {}, *2: {}, *3: {}," *1 *2 *3)]]) + #[["*1: ef, *2: cd, *3: ab,"]])) + (assert (has + (rt #[[ + (raise (Exception "TEST ERROR")) + (+ "err: " (str *e))]]) + #[["err: TEST ERROR"]]))) + +(defn test-comments [rt] + (setv err-empty (rt "" 'err)) + (setv x (rt #[[(+ "a" "b") ; "c"]] 'both)) + (assert (has x.out "ab")) + (assert (= x.err err-empty)) + (assert (= (rt "; 1" 'err) err-empty))) + +(defn test-assignment [rt] + "If the last form is an assignment, don't print the value." + (assert (not (has (rt #[[(setv x (+ "A" "Z"))]]) "AZ"))) + (setv x (rt #[[(setv x (+ "A" "Z")) (+ "B" "Y")]])) + (assert (has x "BY")) + (assert (not (has x "AZ"))) + (setv x (rt #[[(+ "B" "Y") (setv x (+ "A" "Z"))]])) + (assert (not (has x "BY"))) + (assert (not (has x "AZ")))) + +(defn test-multi-setv [rt] + ; https://github.com/hylang/hy/issues/1255 + (assert (re.match + r"=>\s+2\s+=>" + (rt (.replace + "(do + (setv it 0 it (+ it 1) it (+ it 1)) + it)" + "\n" " "))))) + +(defn test-error-underline-alignment [rt] + (setv err (rt "(defmacro mabcdefghi [x] x)\n(mabcdefghi)" 'err)) + (setv msg-idx (.rindex err " (mabcdefghi)")) + (setv [_ e1 e2 e3 #* _] (.splitlines (cut err msg_idx None))) + (assert (.startswith e1 " ^----------^")) + (assert (.startswith e2 "expanding macro mabcdefghi")) + (assert (or + ; PyPy can use a function's `__name__` instead of + ; `__code__.co_name`. + (.startswith e3 " TypeError: mabcdefghi") + (.startswith e3 " TypeError: (mabcdefghi)")))) + +(defn test-except-do [rt] + ; https://github.com/hylang/hy/issues/533 + (assert (has + (rt #[[(try (/ 1 0) (except [ZeroDivisionError] "hello"))]]) + "hello")) + (setv x (rt + #[[(try (/ 1 0) (except [ZeroDivisionError] "aaa" "bbb" "ccc"))]])) + (assert (not (has x "aaa"))) + (assert (not (has x "bbb"))) + (assert (has x "ccc")) + (setv x (rt + #[[(when True "xxx" "yyy" "zzz")]])) + (assert (not (has x "xxx"))) + (assert (not (has x "yyy"))) + (assert (has x "zzz"))) + +(defn test-unlocatable-hytypeerror [rt] + ; https://github.com/hylang/hy/issues/1412 + ; The chief test of interest here is that the REPL isn't itself + ; throwing an error. + (assert (has + (rt :to-return 'err #[[ + (import hy.errors) + (raise (hy.errors.HyTypeError (+ "A" "Z") None '[] None))]]) + "AZ"))) + +(defn test-syntax-errors [rt] + ; https://github.com/hylang/hy/issues/2004 + (assert (has + (rt "(defn foo [/])\n(defn bar [a a])" 'err) + "SyntaxError: duplicate argument")) + ; https://github.com/hylang/hy/issues/2014 + (setv err (rt "(defn foo []\n(import re *))" 'err)) + (assert (has err "SyntaxError: import * only allowed")) + (assert (not (has err "PrematureEndOfInput")))) + +(defn test-bad-repr [rt] + ; https://github.com/hylang/hy/issues/1389 + (setv x (rt :to-return 'both #[[ + (defclass BadRepr [] (defn __repr__ [self] (/ 0))) + (BadRepr) + (+ "A" "Z")]])) + (assert (has x.err "ZeroDivisionError")) + (assert (has x.out "AZ"))) + +(defn test-py-repr [rt] + (assert (has + (rt "(+ [1] [2])") + "[1 2]")) + (assert (has + (rt "(+ [1] [2])" :py-repr True) + "[1, 2]")) + (setv x + (rt "(+ [1] [2])" :py-repr True :spy True)) + (assert (has x "[1] + [2]")) + (assert (has x "[1, 2]")) + ; --spy should work even when an exception is thrown + (assert (has + (rt "(+ [1] [2] (foof))" :py-repr True :spy True) + "[1] + [2]"))) + +(defn test-builtins [rt] + (assert (has + (rt "quit") + "Use (quit) or Ctrl-D (i.e. EOF) to exit")) + (assert (has + (rt "exit") + "Use (exit) or Ctrl-D (i.e. EOF) to exit")) + (assert (has + (rt "help") + "Use (help) for interactive help, or (help object) for help about object."))) + (defn test-preserve-ps1 [rt] ; https://github.com/hylang/hy/issues/1323#issuecomment-1310837340 (setv sys.ps1 "chippy") diff --git a/tests/test_bin.py b/tests/test_bin.py index 6c1ee35f3..db9dabb7c 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -63,116 +63,6 @@ def test_simple(): run_cmd("hy", "") -def test_stdin(): - output, _ = run_cmd("hy", '(.upper "hello")') - assert "HELLO" in output - - output, _ = run_cmd("hy --spy", '(.upper "hello")') - assert ".upper()" in output - assert "HELLO" in output - - # --spy should work even when an exception is thrown - output, _ = run_cmd("hy --spy", "(foof)") - assert "foof()" in output - - -def test_stdin_multiline(): - output, _ = run_cmd("hy", '(+ "a" "b"\n"c" "d")') - assert '"abcd"' in output - - -def test_history(): - output, _ = run_cmd("hy", '''(+ "a" "b") - (+ "c" "d") - (+ "e" "f") - (.format "*1: {}, *2: {}, *3: {}," *1 *2 *3)''') - assert '"*1: ef, *2: cd, *3: ab,"' in output - - output, _ = run_cmd("hy", '''(raise (Exception "TEST ERROR")) - (+ "err: " (str *e))''') - assert '"err: TEST ERROR"' in output - - -def test_stdin_comments(): - _, err_empty = run_cmd("hy", "") - - output, err = run_cmd("hy", '(+ "a" "b") ; "c"') - assert '"ab"' in output - assert err == err_empty - - _, err = run_cmd("hy", "; 1") - assert err == err_empty - - -def test_stdin_assignment(): - # If the last form is an assignment, don't print the value. - - output, _ = run_cmd("hy", '(setv x (+ "A" "Z"))') - assert "AZ" not in output - - output, _ = run_cmd("hy", '(setv x (+ "A" "Z")) (+ "B" "Y")') - assert "AZ" not in output - assert "BY" in output - - output, _ = run_cmd("hy", '(+ "B" "Y") (setv x (+ "A" "Z"))') - assert "AZ" not in output - assert "BY" not in output - - -def test_multi_setv(): - # https://github.com/hylang/hy/issues/1255 - output, _ = run_cmd("hy", """(do - (setv it 0 it (+ it 1) it (+ it 1)) - it)""".replace("\n", " ")) - assert re.match(r"=>\s+2\s+=>", output) - - -def test_stdin_error_underline_alignment(): - _, err = run_cmd("hy", "(defmacro mabcdefghi [x] x)\n(mabcdefghi)") - - msg_idx = err.rindex(" (mabcdefghi)") - err_parts = err[msg_idx:].splitlines() - assert err_parts[1].startswith(" ^----------^") - assert err_parts[2].startswith("expanding macro mabcdefghi") - assert ( - err_parts[3].startswith(" TypeError: mabcdefghi") - or - # PyPy can use a function's `__name__` instead of - # `__code__.co_name`. - err_parts[3].startswith(" TypeError: (mabcdefghi)") - ) - - -def test_stdin_except_do(): - # https://github.com/hylang/hy/issues/533 - - output, _ = run_cmd("hy", - '(try (/ 1 0) (except [ZeroDivisionError] "hello"))') - assert "hello" in output - - output, _ = run_cmd("hy", - '(try (/ 1 0) (except [ZeroDivisionError] "aaa" "bbb" "ccc"))') - assert "aaa" not in output - assert "bbb" not in output - assert "ccc" in output - - output, _ = run_cmd("hy", - '(when True "xxx" "yyy" "zzz")') - assert "xxx" not in output - assert "yyy" not in output - assert "zzz" in output - - -def test_stdin_unlocatable_hytypeerror(): - # https://github.com/hylang/hy/issues/1412 - # The chief test of interest here is the returncode assertion - # inside run_cmd. - _, err = run_cmd("hy", """ - (import hy.errors) - (raise (hy.errors.HyTypeError (+ "A" "Z") None '[] None))""") - assert "AZ" in err - - def test_error_parts_length(): """Confirm that exception messages print arrows surrounding the affected expression.""" @@ -232,43 +122,6 @@ def test_error_parts_length(): assert err_parts[2] == " ^----^" -def test_syntax_errors(): - # https://github.com/hylang/hy/issues/2004 - _, err = run_cmd("hy", "(defn foo [/])\n(defn bar [a a])") - assert "SyntaxError: duplicate argument" in err - - # https://github.com/hylang/hy/issues/2014 - _, err = run_cmd("hy", "(defn foo []\n(import re *))") - assert "SyntaxError: import * only allowed" in err - assert "PrematureEndOfInput" not in err - - -def test_stdin_bad_repr(): - # https://github.com/hylang/hy/issues/1389 - output, err = run_cmd("hy", """ - (defclass BadRepr [] (defn __repr__ [self] (/ 0))) - (BadRepr) - (+ "A" "Z")""") - assert "ZeroDivisionError" in err - assert "AZ" in output - - -def test_stdin_py_repr(): - output, _ = run_cmd("hy", "(+ [1] [2])") - assert "[1 2]" in output - - output, _ = run_cmd(pyr(), "(+ [1] [2])") - assert "[1, 2]" in output - - output, _ = run_cmd(pyr("--spy"), "(+ [1] [2])") - assert "[1]+[2]" in output.replace(" ", "") - assert "[1, 2]" in output - - # --spy should work even when an exception is thrown - output, _ = run_cmd(pyr("--spy"), "(+ [1] [2] (foof))") - assert "[1]+[2]" in output.replace(" ", "") - - def test_mangle_m(): # https://github.com/hylang/hy/issues/1445 @@ -374,19 +227,8 @@ def test_hyc_missing_file(): assert "[Errno 2]" in err -def test_builtins(): - # The REPL replaces `builtins.help` etc. - - output, _ = run_cmd("hy", 'quit') - assert "Use (quit) or Ctrl-D (i.e. EOF) to exit" in output - - output, _ = run_cmd("hy", 'exit') - assert "Use (exit) or Ctrl-D (i.e. EOF) to exit" in output - - output, _ = run_cmd("hy", 'help') - assert "Use (help) for interactive help, or (help object) for help about object." in output - - # Just importing `hy.cmdline` doesn't modify these objects. +def test_repl_builtins(): + "Just importing `hy.cmdline` doesn't modify `help` etc." import hy.cmdline assert "help(object)" in str(builtins.help) From 761d0b5ac5471f87013f3dd7532daba32c204b5e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 19 Jul 2023 12:28:30 -0400 Subject: [PATCH 232/342] Make a REPL test slightly stricter --- tests/native_tests/repl.hy | 6 +++++- tests/test_bin.py | 6 ------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/native_tests/repl.hy b/tests/native_tests/repl.hy index f8c187374..570f2e565 100644 --- a/tests/native_tests/repl.hy +++ b/tests/native_tests/repl.hy @@ -161,7 +161,11 @@ "Use (exit) or Ctrl-D (i.e. EOF) to exit")) (assert (has (rt "help") - "Use (help) for interactive help, or (help object) for help about object."))) + "Use (help) for interactive help, or (help object) for help about object.")) + ; The old values of these objects come back after the REPL ends. + (assert (.startswith + (str quit) + "Use quit() or"))) (defn test-preserve-ps1 [rt] ; https://github.com/hylang/hy/issues/1323#issuecomment-1310837340 diff --git a/tests/test_bin.py b/tests/test_bin.py index db9dabb7c..62fff0806 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -227,12 +227,6 @@ def test_hyc_missing_file(): assert "[Errno 2]" in err -def test_repl_builtins(): - "Just importing `hy.cmdline` doesn't modify `help` etc." - import hy.cmdline - assert "help(object)" in str(builtins.help) - - def test_no_main(): output, _ = run_cmd("hy tests/resources/bin/nomain.hy") assert "This Should Still Work" in output From 9774aa65d107dc8a7dd0dd9d9d8fe0f74658ad65 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 21 Jul 2023 13:21:05 -0400 Subject: [PATCH 233/342] Overhaul `hy -i` to work as a flag --- hy/cmdline.py | 113 ++++++++++++++++++++++++---------------------- tests/test_bin.py | 10 ++-- 2 files changed, 63 insertions(+), 60 deletions(-) diff --git a/hy/cmdline.py b/hy/cmdline.py index 2218ab1d9..719c1d021 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -49,28 +49,7 @@ def run_command(source, filename=None): return 0 -def run_icommand(source, **kwargs): - if Path(source).exists(): - filename = source - set_path(source) - with open(source, "r", encoding="utf-8") as f: - source = f.read() - else: - filename = "" - - hr = REPL(**kwargs) - with filtered_hy_exceptions(): - res = hr.runsource(source, filename=filename) - - # If the command was prematurely ended, show an error (just like Python - # does). - if res: - hy_exc_handler(sys.last_type, sys.last_value, sys.last_traceback) - - return hr.run() - - -USAGE = "hy [-h | -v | -i CMD | -c CMD | -m MODULE | FILE | -] [ARG]..." +USAGE = "hy [-h | -v | -i | -c CMD | -m MODULE | FILE | -] [ARG]..." VERSION = "hy " + hy.__version__ EPILOG = """ FILE @@ -114,9 +93,8 @@ def cmdline_handler(scriptname, argv): ), dict( name=["-i"], - dest="icommand", - terminate=True, - help="program passed in as string, then stay in REPL", + action="store_true", + help="launch REPL after running script; forces a prompt even if stdin does not appear to be a terminal", ), dict( name=["-m"], @@ -242,41 +220,56 @@ def proc_opt(opt, arg=None, item=None, i=None): print(VERSION) return 0 - if "command" in options: + action, action_arg = ( + ["eval_string", options["command"]] + if "command" in options else + ["run_module", options["mod"]] + if "mod" in options else + ["run_script_stdin", None] + if argv and argv[0] == "-" else + ["run_script_file", argv[0]] + if argv else + ["just_repl", None]) + repl = ( + REPL( + spy = options.get("spy"), + output_fn = options.get("repl_output_fn")) + if "i" in options or action == "just_repl" + else None) + source = '' + + if action == "eval_string": sys.argv = ["-c"] + argv - return run_command(options["command"], filename="") - - if "mod" in options: + if repl: + source = action_arg + filename = '' + else: + return run_command(action_arg, filename="") + elif action == "run_module": + if repl: raise ValueError() set_path("") sys.argv = [program] + argv - runpy.run_module(hy.mangle(options["mod"]), run_name="__main__", alter_sys=True) + runpy.run_module(hy.mangle(action_arg), run_name="__main__", alter_sys=True) return 0 - - if "icommand" in options: - return run_icommand( - options["icommand"], - spy=options.get("spy"), - output_fn=options.get("repl_output_fn"), - ) - - if argv: - if argv[0] == "-": - # Read the program from stdin + elif action == "run_script_stdin": + if repl: + source = sys.stdin.read() + filename = 'stdin' + else: return run_command(sys.stdin.read(), filename="") - + elif action == "run_script_file": + filename = Path(action_arg) + set_path(filename) + # Ensure __file__ is set correctly in the code we're about + # to run. + if PY3_9: + if not filename.is_absolute(): + filename = Path.cwd() / filename + if platform.system() == "Windows": + filename = os.path.normpath(filename) + if repl: + source = Path(filename).read_text() else: - # User did "hy " - - filename = Path(argv[0]) - set_path(filename) - # Ensure __file__ is set correctly in the code we're about - # to run. - if PY3_9: - if not filename.is_absolute(): - filename = Path.cwd() / filename - if platform.system() == "Windows": - filename = os.path.normpath(filename) - try: sys.argv = argv with filtered_hy_exceptions(): @@ -293,8 +286,18 @@ def proc_opt(opt, arg=None, item=None, i=None): except HyLanguageError: hy_exc_handler(*sys.exc_info()) sys.exit(1) + else: + assert action == "just_repl" - return REPL(spy=options.get("spy"), output_fn=options.get("repl_output_fn")).run() + # If we didn't return earlier, we'll be using the REPL. + if source: + with filtered_hy_exceptions(): + res = repl.runsource(source, filename=str(filename)) + # If the command was prematurely ended, show an error (just like Python + # does). + if res: + hy_exc_handler(sys.last_type, sys.last_value, sys.last_traceback) + return repl.run() # entry point for cmd line script "hy" diff --git a/tests/test_bin.py b/tests/test_bin.py index 62fff0806..5902a9714 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -175,8 +175,8 @@ def test_cmd(): assert "<-c|AA|ZZ|-m>" in output -def test_icmd(): - output, _ = run_cmd("""hy -i '(.upper "hello")'""", '(.upper "bye")') +def test_icmd_string(): + output, _ = run_cmd("""hy -i -c '(.upper "hello")'""", '(.upper "bye")') assert "HELLO" in output assert "BYE" in output @@ -187,7 +187,7 @@ def test_icmd_file(): def test_icmd_and_spy(): - output, _ = run_cmd('hy --spy -i "(+ [] [])"', "(+ 1 1)") + output, _ = run_cmd('hy --spy -i -c "(+ [] [])"', "(+ 1 1)") assert "[] + []" in output @@ -403,7 +403,7 @@ def req_err(x): assert len(error_lines) <= 4 req_err(error_lines[-1]) - output, error = run_cmd('hy -i "(require not-a-real-module)"') + output, error = run_cmd('hy -i -c "(require not-a-real-module)"') assert output.startswith("=> ") req_err(error.splitlines()[2]) @@ -430,7 +430,7 @@ def req_err(x): # ^ # SyntaxError: EOL while scanning string literal # >>> - output, error = run_cmd(r'hy -i "(print \""') + output, error = run_cmd(r'hy -i -c "(print \""') assert output.startswith("=> ") assert re.match(peoi_re, error) From 6bed4b75facee05c6c924b22884bd12266681ee1 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 19 Jul 2023 11:58:24 -0400 Subject: [PATCH 234/342] Run standard input as a script when it's not a TTY --- docs/cli.rst | 3 ++- hy/cmdline.py | 15 ++++++++++++--- tests/test_bin.py | 41 +++++++++++++++++++++++++++++------------ 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 0f8227a07..d552c59ba 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -11,7 +11,8 @@ hy ``hy`` is a command-line interface for Hy that in general imitates the program ``python`` provided by CPython. For example, ``hy`` without arguments launches -the :ref:`REPL `, whereas ``hy foo.hy a b`` runs the Hy program +the :ref:`REPL ` if standard input is a TTY and runs the standard input +as a script otherwise, whereas ``hy foo.hy a b`` runs the Hy program ``foo.hy`` with ``a`` and ``b`` as command-line arguments. See ``hy --help`` for a complete list of options and :py:ref:`Python's documentation ` for many details. Here are some Hy-specific details: diff --git a/hy/cmdline.py b/hy/cmdline.py index 719c1d021..d12bc45b8 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -229,7 +229,9 @@ def proc_opt(opt, arg=None, item=None, i=None): if argv and argv[0] == "-" else ["run_script_file", argv[0]] if argv else - ["just_repl", None]) + ["just_repl", None] + if sys.stdin.isatty() else + ["run_script_stdin", None]) repl = ( REPL( spy = options.get("spy"), @@ -253,7 +255,7 @@ def proc_opt(opt, arg=None, item=None, i=None): return 0 elif action == "run_script_stdin": if repl: - source = sys.stdin.read() + source = sys.stdin filename = 'stdin' else: return run_command(sys.stdin.read(), filename="") @@ -291,8 +293,15 @@ def proc_opt(opt, arg=None, item=None, i=None): # If we didn't return earlier, we'll be using the REPL. if source: + res = None + filename = str(filename) with filtered_hy_exceptions(): - res = repl.runsource(source, filename=str(filename)) + accum = '' + for chunk in ([source] if isinstance(source, str) else source): + accum += chunk + res = repl.runsource(accum, filename=filename) + if not res: + accum = '' # If the command was prematurely ended, show an error (just like Python # does). if res: diff --git a/tests/test_bin.py b/tests/test_bin.py index 5902a9714..3d3ae6876 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -63,6 +63,23 @@ def test_simple(): run_cmd("hy", "") +def test_stdin(): + # https://github.com/hylang/hy/issues/2438 + code = '(+ "P" "Q")\n(print (+ "R" "S"))\n(+ "T" "U")' + + # Without `-i`, the standard input is run as a script. + out, _ = run_cmd("hy", code) + assert "PQ" not in out + assert "RS" in out + assert "TU" not in out + + # With it, the standard input is fed to the REPL. + out, _ = run_cmd("hy -i", code) + assert "PQ" in out + assert "RS" in out + assert "TU" in out + + def test_error_parts_length(): """Confirm that exception messages print arrows surrounding the affected expression.""" @@ -84,7 +101,7 @@ def test_error_parts_length(): """ # Up-arrows right next to each other. - _, err = run_cmd("hy", prg_str.format(3, 3, 1, 2)) + _, err = run_cmd("hy -i", prg_str.format(3, 3, 1, 2)) msg_idx = err.rindex("HyLanguageError:") assert msg_idx @@ -104,7 +121,7 @@ def test_error_parts_length(): assert obs.startswith(exp) # Make sure only one up-arrow is printed - _, err = run_cmd("hy", prg_str.format(3, 3, 1, 1)) + _, err = run_cmd("hy -i", prg_str.format(3, 3, 1, 1)) msg_idx = err.rindex("HyLanguageError:") assert msg_idx @@ -113,7 +130,7 @@ def test_error_parts_length(): # Make sure lines are printed in between arrows separated by more than one # character. - _, err = run_cmd("hy", prg_str.format(3, 3, 1, 6)) + _, err = run_cmd("hy -i", prg_str.format(3, 3, 1, 6)) print(err) msg_idx = err.rindex("HyLanguageError:") @@ -390,13 +407,13 @@ def req_err(x): # Traceback (most recent call last): # File "", line 1, in # ImportError: No module named not_a_real_module - _, error = run_cmd("hy", "(require not-a-real-module)") + _, error = run_cmd("hy", "(require not-a-real-module)", expect=1) error_lines = error.splitlines() if error_lines[-1] == "": del error_lines[-1] assert len(error_lines) <= 10 # Rough check for the internal traceback filtering - req_err(error_lines[4]) + req_err(error_lines[-1]) _, error = run_cmd('hy -c "(require not-a-real-module)"', expect=1) error_lines = error.splitlines() @@ -430,8 +447,8 @@ def req_err(x): # ^ # SyntaxError: EOL while scanning string literal # >>> - output, error = run_cmd(r'hy -i -c "(print \""') - assert output.startswith("=> ") + output, error = run_cmd(r'hy -c "(print \""', expect=1) + assert output == '' assert re.match(peoi_re, error) # Modeled after @@ -471,28 +488,28 @@ def test_traceback_shebang(tmp_path): def test_hystartup(): # spy == True and custom repl-output-fn os.environ["HYSTARTUP"] = "tests/resources/hystartup.hy" - output, _ = run_cmd("hy", "[1 2]") + output, _ = run_cmd("hy -i", "[1 2]") assert "[1, 2]" in output assert "[1,_2]" in output - output, _ = run_cmd("hy", "(hello-world)") + output, _ = run_cmd("hy -i", "(hello-world)") assert "(hello-world)" not in output assert "1 + 1" in output assert "2" in output - output, _ = run_cmd("hy", "#rad") + output, _ = run_cmd("hy -i", "#rad") assert "#rad" not in output assert "'totally' + 'rad'" in output assert "'totallyrad'" in output - output, _ = run_cmd("hy --repl-output-fn repr", "[1 2 3 4]") + output, _ = run_cmd("hy -i --repl-output-fn repr", "[1 2 3 4]") assert "[1, 2, 3, 4]" in output assert "[1 2 3 4]" not in output assert "[1,_2,_3,_4]" not in output # spy == False and custom repl-output-fn os.environ["HYSTARTUP"] = "tests/resources/spy_off_startup.hy" - output, _ = run_cmd("hy --spy", "[1 2]") # overwrite spy with cmdline arg + output, _ = run_cmd("hy -i --spy", "[1 2]") # overwrite spy with cmdline arg assert "[1, 2]" in output assert "[1,~2]" in output From 8864931db229d22d0204e16cb17f474905c62367 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 21 Jul 2023 13:22:43 -0400 Subject: [PATCH 235/342] Add some comments to `cmdline.py` --- hy/cmdline.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hy/cmdline.py b/hy/cmdline.py index d12bc45b8..c270ba72f 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -221,14 +221,21 @@ def proc_opt(opt, arg=None, item=None, i=None): return 0 action, action_arg = ( + # If the `command` or `mod` options were provided, we'll run + # the corresponding code. ["eval_string", options["command"]] if "command" in options else ["run_module", options["mod"]] if "mod" in options else + # Otherwise, we'll run any provided filename as a script (or + # standard input, if the filename is "-"). ["run_script_stdin", None] if argv and argv[0] == "-" else ["run_script_file", argv[0]] if argv else + # With none of those arguments, we'll launch the REPL (if + # standard input is a TTY) or run a script from standard input + # (otherwise). ["just_repl", None] if sys.stdin.isatty() else ["run_script_stdin", None]) @@ -293,6 +300,7 @@ def proc_opt(opt, arg=None, item=None, i=None): # If we didn't return earlier, we'll be using the REPL. if source: + # Execute `source` in the REPL before entering interactive mode. res = None filename = str(filename) with filtered_hy_exceptions(): From 1abce37f43779826eef3546ec276860e0bec75bc Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 19 Jul 2023 12:19:21 -0400 Subject: [PATCH 236/342] Update NEWS --- NEWS.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 028a0f351..72334ed99 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,8 @@ Unreleased Breaking Changes ------------------------------ +* `hy` now only implicitly launches a REPL if standard input is a TTY. +* `hy -i` has been overhauled to work as a flag like `python3 -i`. * `hy2py` now requires `-m` to specify modules, and uses the same `sys.path` rules as Python when parsing a module vs a standalone script. From b5a1b6e3f7ec40beedefcd4bb6c0b490b70103f7 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 22 Jul 2023 11:48:06 -0400 Subject: [PATCH 237/342] Add support for type parameters --- docs/api.rst | 30 ++++++++++++-------- hy/core/result_macros.py | 48 ++++++++++++++++++++++++-------- hy/model_patterns.py | 14 ++++++---- tests/native_tests/defclass.hy | 8 ++++++ tests/native_tests/functions.hy | 49 +++++++++++++++++++++++++++++++++ tests/resources/tp.hy | 10 +++++++ 6 files changed, 131 insertions(+), 28 deletions(-) create mode 100644 tests/resources/tp.hy diff --git a/docs/api.rst b/docs/api.rst index 4ee722dd9..bdc51de86 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -87,10 +87,11 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:macro:: (fn [args]) As :hy:func:`defn`, but no name for the new function is required (or - allowed), and the newly created function object is returned. Decorators - aren't allowed, either. However, the function body is understood identically - to that of :hy:func:`defn`, without any of the restrictions of Python's - :py:keyword:`lambda`. See :hy:func:`fn/a` for the asynchronous equivalent. + allowed), and the newly created function object is returned. Decorators and + type parameters aren't allowed, either. However, the function body is + understood identically to that of :hy:func:`defn`, without any of the + restrictions of Python's :py:keyword:`lambda`. See :hy:func:`fn/a` for the + asynchronous equivalent. .. hy:macro:: (fn/a [name #* args]) @@ -114,12 +115,12 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. returned; thus, ``(defn f [] 5)`` is equivalent to ``(defn f [] (return 5))``. - ``defn`` accepts two additional, optional arguments: a bracketed list of - :term:`decorators ` and an annotation (see :hy:func:`annotate`) for - the return value. These are placed before the function name (in that order, - if both are present):: + ``defn`` accepts a few more optional arguments: a bracketed list of + :term:`decorators `, a list of type parameters (see below), + and an annotation (see :hy:func:`annotate`) for the return value. These are + placed before the function name (in that order, if several are present):: - (defn [decorator1 decorator2] ^annotation name [params] …) + (defn [decorator1 decorator2] :tp [T1 T2] #^ annotation name [params] …) To define asynchronous functions, see :hy:func:`defn/a` and :hy:func:`fn/a`. @@ -163,6 +164,12 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (print (hy.repr (f 1 2 :d 4 :e 5 :f 6))) ; => [1 2 3 4 5 {"f" 6}] + Type parameters require Python 3.12, and have the semantics specified by + :pep:`695`. The keyword ``:tp`` introduces the list of type parameters. Each + item of the list is a symbol, an annotated symbol (such as ``#^ int T``), or + an unpacked symbol (such as ``#* T`` or ``#** T``). As in Python, unpacking + and annotation can't be used with the same parameter. + .. hy:macro:: (defn/a [name lambda-list #* body]) As :hy:func:`defn`, but defines a :ref:`coroutine ` like @@ -716,12 +723,13 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. ``defclass`` compiles to a :py:keyword:`class` statement, which creates a new class. It always returns ``None``. Only one argument, specifying the name of the new class as a symbol, is required. A list of :term:`decorators - ` may be provided before the class name. After the name comes + ` (and type parameters, in the same way as for + :hy:func:`defn`) may be provided before the class name. After the name comes a list of superclasses (use the empty list ``[]`` for the typical case of no superclasses) and any number of body forms, the first of which may be a :term:`py:docstring`. :: - (defclass [decorator1 decorator2] MyClass [SuperClass1 SuperClass2] + (defclass [decorator1 decorator2] :tp [T1 T2] MyClass [SuperClass1 SuperClass2] "A class that does things at times." (setv diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 34fb2e81e..d494d01ec 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -79,6 +79,30 @@ def dotted(name): return Expression(map(Symbol, [".", *name.split(".")])) +type_params = sym(":tp") + brackets(many( + maybe_annotated(SYM) | unpack("either", Symbol))) + +def digest_type_params(compiler, tp): + "Return a `type_params` attribute for `FunctionDef` etc." + + if tp: + if not PY3_12: + compiler._syntax_error(tp, "`:tp` requires Python 3.12 or later") + tp, = tp + elif not PY3_12: + return {} + + return dict(type_params = [ + asty.TypeVarTuple(x[1], name = mangle(x[1])) + if is_unpack("iterable", x) else + asty.ParamSpec(x[1], name = mangle(x[1])) + if is_unpack("mapping", x) else + asty.TypeVar(x[0], + name = mangle(x[0]), + bound = x[1] and compiler.compile(x[1]).force_expr) + for x in (tp or [])]) + + # ------------------------------------------------ # * Fundamentals # ------------------------------------------------ @@ -1410,8 +1434,9 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo ) -@pattern_macro(["fn", "fn/a"], [maybe_annotated(lambda_list), many(FORM)]) -def compile_function_lambda(compiler, expr, root, params, body): +@pattern_macro(["fn", "fn/a"], + [maybe(type_params), maybe_annotated(lambda_list), many(FORM)]) +def compile_function_lambda(compiler, expr, root, tp, params, body): params, returns = params posonly, args, rest, kwonly, kwargs = params has_annotations = returns is not None or any( @@ -1423,13 +1448,13 @@ def compile_function_lambda(compiler, expr, root, params, body): body = compiler._compile_branch(body) # Compile to lambda if we can - if not has_annotations and not body.stmts and root != "fn/a": + if not (has_annotations or tp or body.stmts or root == "fn/a"): return ret + asty.Lambda(expr, args=args, body=body.force_expr) # Otherwise create a standard function node = asty.AsyncFunctionDef if root == "fn/a" else asty.FunctionDef name = compiler.get_anon_var() - ret += compile_function_node(compiler, expr, node, [], name, args, returns, body) + ret += compile_function_node(compiler, expr, node, [], tp, name, args, returns, body) # return its name as the final expr return ret + Result(expr=ret.temp_variables[0]) @@ -1437,9 +1462,9 @@ def compile_function_lambda(compiler, expr, root, params, body): @pattern_macro( ["defn", "defn/a"], - [maybe(brackets(many(FORM))), maybe_annotated(SYM), lambda_list, many(FORM)], + [maybe(brackets(many(FORM))), maybe(type_params), maybe_annotated(SYM), lambda_list, many(FORM)], ) -def compile_function_def(compiler, expr, root, decorators, name, params, body): +def compile_function_def(compiler, expr, root, decorators, tp, name, params, body): name, returns = name node = asty.FunctionDef if root == "defn" else asty.AsyncFunctionDef decorators, ret, _ = compiler._compile_collect(decorators[0] if decorators else []) @@ -1451,11 +1476,11 @@ def compile_function_def(compiler, expr, root, decorators, name, params, body): body = compiler._compile_branch(body) return ret + compile_function_node( - compiler, expr, node, decorators, name, args, returns, body + compiler, expr, node, decorators, tp, name, args, returns, body ) -def compile_function_node(compiler, expr, node, decorators, name, args, returns, body): +def compile_function_node(compiler, expr, node, decorators, tp, name, args, returns, body): ret = Result() if body.expr: @@ -1468,7 +1493,7 @@ def compile_function_node(compiler, expr, node, decorators, name, args, returns, body=body.stmts or [asty.Pass(expr)], decorator_list=decorators, returns=compiler.compile(returns).force_expr if returns is not None else None, - **({"type_params": []} if PY3_12 else {}), + **digest_type_params(compiler, tp), ) ast_name = asty.Name(expr, id=name, ctx=ast.Load()) @@ -1648,11 +1673,12 @@ def compile_yield_from_or_await_expression(compiler, expr, root, arg): "defclass", [ maybe(brackets(many(FORM))), + maybe(type_params), SYM, maybe(brackets(many(FORM)) + maybe(STR) + many(FORM)), ], ) -def compile_class_expression(compiler, expr, root, decorators, name, rest): +def compile_class_expression(compiler, expr, root, decorators, tp, name, rest): base_list, docstring, body = rest or ([[]], None, []) decorators, ret, _ = compiler._compile_collect(decorators[0] if decorators else []) @@ -1682,7 +1708,7 @@ def compile_class_expression(compiler, expr, root, decorators, name, rest): kwargs=None, bases=bases_expr, body=bodyr.stmts or [asty.Pass(expr)], - **({"type_params": []} if PY3_12 else {}), + **digest_type_params(compiler, tp) ) diff --git a/hy/model_patterns.py b/hy/model_patterns.py index 1257e912f..14d06a336 100644 --- a/hy/model_patterns.py +++ b/hy/model_patterns.py @@ -103,13 +103,15 @@ def notpexpr(*disallowed_heads): ) -def unpack(kind): +def unpack(kind, content_type = None): "Parse an unpacking form, returning it unchanged." - return some( - lambda x: isinstance(x, Expression) - and len(x) > 0 - and x[0] == Symbol("unpack-" + kind) - ) + return some(lambda x: + isinstance(x, Expression) and + len(x) > 0 and + x[0] in [Symbol("unpack-" + tail) for tail in + (["iterable", "mapping"] if kind == "either" else [kind])] and + (content_type is None or + (len(x) == 2 and isinstance(x[1], content_type)))) def times(lo, hi, parser): diff --git a/tests/native_tests/defclass.hy b/tests/native_tests/defclass.hy index 30aae9732..f26e38d11 100644 --- a/tests/native_tests/defclass.hy +++ b/tests/native_tests/defclass.hy @@ -148,3 +148,11 @@ (defclass Quest [QuestBase :swallow "african"]) (assert (= (. (Quest) swallow) "african"))) + + +(do-mac (when hy._compat.PY3_12 '(defn test-type-params [] + (import tests.resources.tp :as ttp) + (defclass :tp [#^ int A #** B] C) + (assert (= (ttp.show C) [ + [ttp.TypeVar "A" int #()] + [ttp.ParamSpec "B" None #()]]))))) diff --git a/tests/native_tests/functions.hy b/tests/native_tests/functions.hy index fc9dd05a7..2ac5dfcb6 100644 --- a/tests/native_tests/functions.hy +++ b/tests/native_tests/functions.hy @@ -96,6 +96,55 @@ (assert (= (. f __annotations__ [k]) v)))) +(do-mac (when hy._compat.PY3_12 '(defn test-type-params [] + (import tests.resources.tp :as ttp) + + (defn foo []) + (assert (= (ttp.show foo) [])) + + ; `defn` with a type parameter + (defn :tp [T] #^ T foo [#^ T x] + (+ x 1)) + (assert (= (foo 3) 4)) + (assert (= (ttp.show foo) [[ttp.TypeVar "T" None #()]])) + + ; `fn` with a type parameter + (setv foo (fn :tp [T] #^ T [#^ T x] + (+ x 2))) + (assert (= (foo 3) 5)) + (assert (= (ttp.show foo) [[ttp.TypeVar "T" None #()]])) + + ; Bounds and constraints + (defn :tp [#^ int T] foo []) + (assert (= (ttp.show foo) [[ttp.TypeVar "T" int #()]])) + (defn :tp [#^ #(int str) T] foo []) + (assert (= (ttp.show foo) [[ttp.TypeVar "T" None #(int str)]])) + + ; `TypeVarTuple`s and `ParamSpec`s + (defn :tp [#* T] foo []) + (assert (= (ttp.show foo) [[ttp.TypeVarTuple "T" None #()]])) + (defn :tp [#** T] foo []) + (assert (= (ttp.show foo) [[ttp.ParamSpec "T" None #()]])) + + ; A more complex case + (defn + :tp [A #^ int B #* C #** D #* E #^ #(bool float) F] + foo []) + (assert (= (ttp.show foo) [ + [ttp.TypeVar "A" None #()] + [ttp.TypeVar "B" int #()] + [ttp.TypeVarTuple "C" None #()] + [ttp.ParamSpec "D" None #()] + [ttp.TypeVarTuple "E" None #()] + [ttp.TypeVar "F" None #(bool float)]])) + + ; Illegal attempts to annotate unpacking + (with [(pytest.raises hy.errors.HySyntaxError)] + (hy.eval '(defn :tp [#^ int #* T] foo []))) + (with [(pytest.raises hy.errors.HySyntaxError)] + (hy.eval '(defn :tp [#^ #(int str) #** T] foo [])))))) + + (defn test-lambda-keyword-lists [] (defn foo [x #* xs #** kw] [x xs kw]) (assert (= (foo 10 20 30) [10 #(20 30) {}]))) diff --git a/tests/resources/tp.hy b/tests/resources/tp.hy new file mode 100644 index 000000000..e02a47299 --- /dev/null +++ b/tests/resources/tp.hy @@ -0,0 +1,10 @@ +"Helpers for testing type parameters." + +(import typing [TypeVar TypeVarTuple ParamSpec TypeAliasType]) + +(defn show [f] + (lfor + tp f.__type_params__ + [(type tp) tp.__name__ + (getattr tp "__bound__" None) + (getattr tp "__constraints__" #())])) From 613486e67bd48a9c1ca647c3d690396615c337ff Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 22 Jul 2023 11:50:06 -0400 Subject: [PATCH 238/342] Add `deftype` --- docs/api.rst | 10 ++++++++++ docs/conf.py | 1 + hy/core/result_macros.py | 8 ++++++++ tests/native_tests/deftype.hy | 21 +++++++++++++++++++++ 4 files changed, 40 insertions(+) create mode 100644 tests/native_tests/deftype.hy diff --git a/docs/api.rst b/docs/api.rst index bdc51de86..3da08e8bd 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1244,6 +1244,16 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (hy.repr (list (zip "abc" (myrange)))) ; => [#("a" 0) #("b" 1) #("c" 2)] +.. hy:macro:: (deftype [args]) + + ``deftype`` compiles to a :py3_12:keyword:`type` statement, which defines a + type alias. It requires Python 3.12. Its arguments optionally begin with + ``:tp`` and a list of type parameters (as in :hy:func:`defn`), then specify + the name for the new alias and its value. :: + + (deftype IntOrStr (| int str)) + (deftype :tp [T] ListOrSet (| (get list T) (get set T))) + .. hy:macro:: (pragma) ``pragma`` is reserved as a core macro name for future use, especially for diff --git a/docs/conf.py b/docs/conf.py index 03ad67207..3d6be0873 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -68,6 +68,7 @@ intersphinx_mapping = dict( py=("https://docs.python.org/3/", None), py3_10=("https://docs.python.org/3.10/", None), + py3_12=("https://docs.python.org/3.12/", None), hyrule=("https://hyrule.readthedocs.io/en/master/", None), ) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index d494d01ec..1fad8e85c 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1952,6 +1952,14 @@ def compile_let(compiler, expr, root, bindings, body): return res + compiler.compile(mkexpr("do", *body).replace(expr)) +@pattern_macro(((3, 12), "deftype"), [maybe(type_params), SYM, FORM]) +def compile_deftype(compiler, expr, root, tp, name, value): + return asty.TypeAlias(expr, + name = asty.Name(name, id = mangle(name), ctx = ast.Store()), + value = compiler.compile(value).force_expr, + **digest_type_params(compiler, tp)) + + @pattern_macro( "pragma unquote unquote-splice unpack-mapping except except* finally else".split(), [many(FORM)], diff --git a/tests/native_tests/deftype.hy b/tests/native_tests/deftype.hy new file mode 100644 index 000000000..8be8a9e3b --- /dev/null +++ b/tests/native_tests/deftype.hy @@ -0,0 +1,21 @@ +(do-mac (when hy._compat.PY3_12 '(do + +(import tests.resources.tp :as ttp) + + +(defn test-deftype [] + + (deftype Foo int) + (assert (is (type Foo) ttp.TypeAliasType)) + (assert (= Foo.__value__) int) + + (deftype Foo (| int bool)) + (assert (is (type Foo.__value__ hy.M.types.UnionType))) + + (deftype :tp [#^ int A #** B] Foo int) + (assert (= (ttp.show Foo) [ + [ttp.TypeVar "A" int #()] + [ttp.ParamSpec "B" None #()]]))))) + + +) From 65228c8a6cc25dc8ff3cda827090f955db4e8384 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 22 Jul 2023 11:45:53 -0400 Subject: [PATCH 239/342] Update NEWS --- NEWS.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 72334ed99..345094bb2 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -10,6 +10,11 @@ Breaking Changes * `hy2py` now requires `-m` to specify modules, and uses the same `sys.path` rules as Python when parsing a module vs a standalone script. +* New macro `deftype`. + +New Features +------------------------------ +* `defn`, `defn/a`, and `defclass` now support type parameters. 0.27.0 (released 2023-07-06) ============================= From 9dff46adaf2503341af27e79badba28ed3d2f3bd Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 25 Jul 2023 16:06:32 -0400 Subject: [PATCH 240/342] Fix `hy2py` on standard input --- hy/cmdline.py | 2 +- tests/test_bin.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hy/cmdline.py b/hy/cmdline.py index c270ba72f..be8d5a318 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -384,7 +384,7 @@ def printing_source(hst): hst.filename = filename with filtered_hy_exceptions(): - module_name = source_path.stem + module_name = source_path.stem if source_path else Path(filename).name if parent_module: module_name = f"{parent_module}.{module_name}" module = types.ModuleType(module_name) diff --git a/tests/test_bin.py b/tests/test_bin.py index 3d3ae6876..a4c41389d 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -592,6 +592,12 @@ def test_assert(tmp_path, monkeypatch): assert ("bye" in err) == show_msg +def test_hy2py_stdin(): + out, _ = run_cmd("hy2py", "(+ 482 223)") + assert "482 + 223" in out + assert "705" not in out + + def test_hy2py_compile_only(monkeypatch): def check(args): output, _ = run_cmd(f"hy2py {args}") From a0f08f5d0109ea690b3f6fe35a43c63765a85d0f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 29 Jul 2023 09:08:39 -0400 Subject: [PATCH 241/342] Improve model-pattern error msgs for core macros --- NEWS.rst | 5 ++++ hy/core/result_macros.py | 10 ++++---- hy/macros.py | 2 +- hy/model_patterns.py | 47 +++++++++++++++++-------------------- tests/compilers/test_ast.py | 12 ++++++++++ 5 files changed, 45 insertions(+), 31 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 345094bb2..f9acfdccc 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -16,6 +16,11 @@ New Features ------------------------------ * `defn`, `defn/a`, and `defclass` now support type parameters. +Misc. Improvements +------------------------------ +* Some syntax errors raised by core macros now have more informative + messages. + 0.27.0 (released 2023-07-06) ============================= diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 1fad8e85c..2db198d50 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -72,7 +72,9 @@ def pvalue(root, wanted): def maybe_annotated(target): - return pexpr(sym("annotate") + target + FORM) | target >> (lambda x: (x, None)) + return ( + pexpr(sym("annotate") + target + FORM).named('`annotate` form') | + (target >> (lambda x: (x, None)))) def dotted(name): @@ -727,9 +729,9 @@ def compile_if(compiler, expr, _, cond, body, orel_expr): ) -@pattern_macro( - ["for"], [brackets(loopers), many(notpexpr("else")) + maybe(dolike("else"))] -) +@pattern_macro(["for"], [ + brackets(loopers, name = 'square-bracketed loop clauses'), + many(notpexpr("else")) + maybe(dolike("else"))]) @pattern_macro(["lfor", "sfor", "gfor"], [loopers, FORM]) @pattern_macro(["dfor"], [loopers, finished]) # Here `finished` is a hack replacement for FORM + FORM: diff --git a/hy/macros.py b/hy/macros.py index d684310fc..c1ee5604b 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -68,7 +68,7 @@ def wrapper(hy_compiler, *args): raise hy_compiler._syntax_error( expr[min(e.state.pos + 1, len(expr) - 1)], "parse error for pattern macro '{}': {}".format( - root, e.msg.replace("", "end of form") + root, e.msg.replace("end of input", "end of macro call") ), ) return fn(hy_compiler, expr, root, *parse_tree) diff --git a/hy/model_patterns.py b/hy/model_patterns.py index 14d06a336..1d7033958 100644 --- a/hy/model_patterns.py +++ b/hy/model_patterns.py @@ -31,11 +31,11 @@ Tuple, ) -FORM = some(lambda _: True) -SYM = some(lambda x: isinstance(x, Symbol)) -KEYWORD = some(lambda x: isinstance(x, Keyword)) -STR = some(lambda x: isinstance(x, String)) # matches literal strings only! -LITERAL = some(lambda x: isinstance(x, (String, Integer, Float, Complex, Bytes))) +FORM = some(lambda _: True).named('form') +SYM = some(lambda x: isinstance(x, Symbol)).named('Symbol') +KEYWORD = some(lambda x: isinstance(x, Keyword)).named('Keyword') +STR = some(lambda x: isinstance(x, String)).named('String') # matches literal strings only! +LITERAL = some(lambda x: isinstance(x, (String, Integer, Float, Complex, Bytes))).named('literal') def sym(wanted): @@ -49,9 +49,10 @@ def keepsym(wanted): def _sym(wanted, f=lambda x: x): + name = '`' + wanted + '`' if wanted.startswith(":"): - return f(a(Keyword(wanted[1:]))) - return f(some(lambda x: x == Symbol(wanted))) + return f(a(Keyword(wanted[1:]))).named(name) + return f(some(lambda x: x == Symbol(wanted))).named(name) def whole(parsers): @@ -64,29 +65,23 @@ def whole(parsers): return reduce(add, parsers) + skip(finished) -def _grouped(group_type, parsers): - return some(lambda x: isinstance(x, group_type)) >> ( - lambda x: group_type(whole(parsers).parse(x)).replace(x, recursive=False) +def _grouped(group_type, syntax_example, name, parsers): + return ( + some(lambda x: isinstance(x, group_type)).named(name or + f'{group_type.__name__} (i.e., `{syntax_example}`)') >> + (lambda x: group_type(whole(parsers).parse(x)).replace(x, recursive=False)) ) - - -def brackets(*parsers): +def brackets(*parsers, name = None): "Parse the given parsers inside square brackets." - return _grouped(List, parsers) - - -def in_tuple(*parsers): - return _grouped(Tuple, parsers) - - -def braces(*parsers): + return _grouped(List, '[ … ]', name, parsers) +def in_tuple(*parsers, name = None): + return _grouped(Tuple, '#( … )', name, parsers) +def braces(*parsers, name = None): "Parse the given parsers inside curly braces" - return _grouped(Dict, parsers) - - -def pexpr(*parsers): + return _grouped(Dict, '{ … }', name, parsers) +def pexpr(*parsers, name = None): "Parse the given parsers inside a parenthesized expression." - return _grouped(Expression, parsers) + return _grouped(Expression, '( … )', name, parsers) def dolike(head): diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index d3d91a5d6..56e903894 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -628,3 +628,15 @@ def test_module_prelude(): def test_pragma(): cant_compile("(pragma)") cant_compile("(pragma :native-code :namespaced-symbols :give-user-a-pony)") + + +def test_error_with_expectation(): + def check(code, expected): + assert cant_compile(code).msg.endswith("expected: " + expected) + + check("(defmacro)", "Symbol") + check("(quote)", "form") + check("(py)", "String") + check("(py a)", "String") + check('(py "foo" a)', "end of macro call") + check('(for a)', "square-bracketed loop clauses") From 4dabf6916e5651adeef37a625bbb98c50b60f0d3 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Tue, 1 Aug 2023 12:05:25 +0200 Subject: [PATCH 242/342] Convert strings ourselves instead of using `eval` Fixes #2474. --- NEWS.rst | 4 +++ hy/reader/hy_reader.py | 46 ++++++++++++++++++++++++++--------- tests/native_tests/strings.hy | 2 ++ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index f9acfdccc..1c8753993 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -21,6 +21,10 @@ Misc. Improvements * Some syntax errors raised by core macros now have more informative messages. +Bug Fixes +------------------------------ +* Double quotes inside of bracketed f-strings are now properly handled. + 0.27.0 (released 2023-07-06) ============================= diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 923b820e2..b2593df86 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -1,5 +1,6 @@ "Character reader for parsing Hy source." +import codecs from itertools import islice import hy @@ -438,7 +439,7 @@ def delim_closing(c): index = -1 return 0 - return self.read_string_until(delim_closing, "fr" if is_fstring else None, is_fstring, brackets=delim) + return self.read_string_until(delim_closing, "r", is_fstring, brackets=delim) def read_string_until(self, closing, prefix, is_fstring, **kwargs): if is_fstring: @@ -449,6 +450,7 @@ def read_string_until(self, closing, prefix, is_fstring, **kwargs): def read_chars_until(self, closing, prefix, is_fstring): s = [] + in_named_escape = False for c in self.chars(): s.append(c) # check if c is closing @@ -457,19 +459,39 @@ def read_chars_until(self, closing, prefix, is_fstring): # string has ended s = s[:-n_closing_chars] break - # check if c is start of component - if is_fstring and c == "{" and s[-3:] != ["\\", "N", "{"]: - # check and handle "{{" - if self.peek_and_getc("{"): - s.append("{") - else: - # remove "{" from end of string component - s.pop() - break + if is_fstring: + # handle braces in f-strings + if c == "{": + if "r" not in prefix and s[-3:] == ["\\", "N", "{"]: + # ignore "\N{...}" + in_named_escape = True + elif not self.peek_and_getc("{"): + # start f-component if not "{{" + s.pop() + break + elif c == "}": + if in_named_escape: + in_named_escape = False + elif not self.peek_and_getc("}"): + raise SyntaxError("f-string: single '}' is not allowed") res = "".join(s).replace("\x0d\x0a", "\x0a").replace("\x0d", "\x0a") - if prefix is not None: - res = eval(f'{prefix}"""{res}"""') + if "b" in prefix: + try: + res = res.encode('ascii') + except UnicodeEncodeError: + raise SyntaxError("bytes can only contain ASCII literal characters") + + if "r" not in prefix: + # perform string escapes + if "b" in prefix: + res = codecs.escape_decode(res)[0] + else: + # formula taken from https://stackoverflow.com/a/57192592 + # encode first to ISO-8859-1 ("Latin 1") due to a Python bug, + # see https://github.com/python/cpython/issues/65530 + res = res.encode('ISO-8859-1', errors='backslashreplace').decode('unicode_escape') + if is_fstring: return res, n_closing_chars return res diff --git a/tests/native_tests/strings.hy b/tests/native_tests/strings.hy index 15b7df194..d0f580e08 100644 --- a/tests/native_tests/strings.hy +++ b/tests/native_tests/strings.hy @@ -101,6 +101,8 @@ cee"} dee" "ey bee\ncee dee")) (assert (= #[f[{{escaped braces}} \n {"not escaped"}]f] "{escaped braces} \\n not escaped")) + ; https://github.com/hylang/hy/issues/2474 + (assert (= #[f["{0}"]f] "\"0\"")) ; Quoting shouldn't evaluate the f-string immediately ; https://github.com/hylang/hy/issues/1844 From 48c44d61b455dad35b5ae154152b38af4e656519 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Sun, 30 Jul 2023 00:32:24 +0200 Subject: [PATCH 243/342] update error message in prior test --- tests/test_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_reader.py b/tests/test_reader.py index 873d5b4c8..01c2ef687 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -159,9 +159,9 @@ def test_lex_strings_exception(): ' File "", line 1', ' "\\x8"', " ^", - "hy.reader.exceptions.LexException: (unicode error)" + "hy.reader.exceptions.LexException:" " 'unicodeescape' codec can't decode bytes in position 0-2:" - " truncated \\xXX escape (, line 1)", + " truncated \\xXX escape", ], ) From 5cf300acd841c34d35e77f589d5f64ba7b031ec5 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 3 Aug 2023 13:04:41 -0400 Subject: [PATCH 244/342] Correct `root` in pattern macros with dotted head --- NEWS.rst | 2 ++ hy/macros.py | 7 +++---- tests/native_tests/functions.hy | 9 +++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 1c8753993..20a508b7c 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -24,6 +24,8 @@ Misc. Improvements Bug Fixes ------------------------------ * Double quotes inside of bracketed f-strings are now properly handled. +* Fixed incomplete recognition of macro calls with a unary dotted + head like `((. defn) f [])`. 0.27.0 (released 2023-07-06) ============================= diff --git a/hy/macros.py b/hy/macros.py index c1ee5604b..bdc373322 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -52,13 +52,12 @@ def wrapper(hy_compiler, *args): ).replace(hy_compiler.this) expr = hy_compiler.this - root = unmangle(expr[0]) if py_version_required and sys.version_info < py_version_required: raise hy_compiler._syntax_error( expr, "`{}` requires Python {} or later".format( - root, ".".join(map(str, py_version_required)) + name, ".".join(map(str, py_version_required)) ), ) @@ -68,10 +67,10 @@ def wrapper(hy_compiler, *args): raise hy_compiler._syntax_error( expr[min(e.state.pos + 1, len(expr) - 1)], "parse error for pattern macro '{}': {}".format( - root, e.msg.replace("end of input", "end of macro call") + name, e.msg.replace("end of input", "end of macro call") ), ) - return fn(hy_compiler, expr, root, *parse_tree) + return fn(hy_compiler, expr, name, *parse_tree) return wrapper diff --git a/tests/native_tests/functions.hy b/tests/native_tests/functions.hy index 2ac5dfcb6..a1095cc25 100644 --- a/tests/native_tests/functions.hy +++ b/tests/native_tests/functions.hy @@ -189,6 +189,15 @@ (assert (= (asyncio.run (coro-test)) [1 2 3]))) +(defn test-root-set-correctly [] + ; https://github.com/hylang/hy/issues/2475 + ((. defn) not-async [] "ok") + (assert (= (not-async) "ok")) + (require builtins) + (builtins.defn also-not-async [] "ok") + (assert (= (also-not-async) "ok"))) + + (defn test-return [] ; `return` in main line From ae93d3adddccd756a7fff28ff6acdbf137cf38d8 Mon Sep 17 00:00:00 2001 From: wrobell Date: Fri, 4 Aug 2023 18:55:49 +0100 Subject: [PATCH 245/342] Improve note about `defmacro` parameters in the API docs --- docs/api.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 3da08e8bd..240dc355e 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -198,10 +198,9 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. => (infix (1 + 1)) 2 - .. note:: because all values are passed to macros unevaluated, ``defmacro`` - cannot use keyword arguments, or kwargs. All arguments are passed - in positionally. Parameters can still be given default values - however:: + .. note:: ``defmacro`` cannot use keyword arguments, because all values + are passed to macros unevaluated. All arguments are passed + positionally, but they can have default values:: => (defmacro a-macro [a [b 1]] ... `[~a ~b]) From f99b4ed4676c94bda7a2f741ad1608117da37645 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 6 Aug 2023 08:13:51 -0400 Subject: [PATCH 246/342] Remove the unused argument `ast_callback` from `hy_eval` --- hy/compiler.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 32e10cd86..ba2bdd5e2 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -688,7 +688,6 @@ def hy_eval( hytree, locals=None, module=None, - ast_callback=None, compiler=None, filename=None, source=None, @@ -727,11 +726,6 @@ def hy_eval( When neither `module` nor `compiler` is specified, the calling frame's module is used. - ast_callback (Optional[Callable]): - A callback that is passed the Hy compiled tree and resulting - expression object, in that order, after compilation but before - evaluation. - compiler (Optional[HyASTCompiler]): An existing Hy compiler to use for compilation. Also serves as the `module` value when given. @@ -776,9 +770,6 @@ def hy_eval( import_stdlib=import_stdlib, ) - if ast_callback: - ast_callback(_ast, expr) - # Two-step eval: eval() the body of the exec call eval(ast_compile(_ast, filename, "exec"), module.__dict__, locals) From 8d730f30d292c26e4cc2c789cf94393a6acccd65 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 7 Aug 2023 14:19:09 -0400 Subject: [PATCH 247/342] Slight cleanup of `native-tests.hy-eval` --- tests/native_tests/hy_eval.hy | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/native_tests/hy_eval.hy b/tests/native_tests/hy_eval.hy index ff664f617..7d3179c1b 100644 --- a/tests/native_tests/hy_eval.hy +++ b/tests/native_tests/hy_eval.hy @@ -1,3 +1,6 @@ +"Tests of the user-facing function `hy.eval`." + + (import re pytest) @@ -45,15 +48,6 @@ (hy.eval (quote x) d2))) -(defn test-eval-failure [] - ; yo dawg - (with [(pytest.raises TypeError)] (hy.eval '(hy.eval))) - (defclass C) - (with [(pytest.raises TypeError)] (hy.eval (C))) - (with [(pytest.raises TypeError)] (hy.eval 'False [])) - (with [(pytest.raises TypeError)] (hy.eval 'False {} 1))) - - (defn test-eval-quasiquote [] ; https://github.com/hylang/hy/issues/1174 @@ -84,3 +78,12 @@ (setv d {"a" 1 "b" 2}) (setv k "b") (assert (= (hy.eval `(get ~d ~k)) 2))) + + +(defn test-eval-failure [] + ; yo dawg + (with [(pytest.raises TypeError)] (hy.eval '(hy.eval))) + (defclass C) + (with [(pytest.raises TypeError)] (hy.eval (C))) + (with [(pytest.raises TypeError)] (hy.eval 'False [])) + (with [(pytest.raises TypeError)] (hy.eval 'False {} 1))) From 5c1720468326cd54eb8095998921487ff7ff8552 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 9 Aug 2023 13:42:17 -0400 Subject: [PATCH 248/342] Provide a new `hy.eval` `hy.eval` is now equal to `hy.compiler.hy_eval_user`, which has a new signature more like Python's `eval`, and is better documented and tested. --- NEWS.rst | 1 + hy/__init__.py | 2 +- hy/compiler.py | 57 +++++++++--- tests/compilers/test_ast.py | 5 +- tests/importer/test_importer.py | 4 +- tests/native_tests/hy_eval.hy | 151 ++++++++++++++++++++++++++++++-- tests/test_reader.py | 6 +- 7 files changed, 198 insertions(+), 28 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 20a508b7c..c91665a85 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,7 @@ Unreleased Breaking Changes ------------------------------ +* `hy.eval` has been overhauled to be more like Python's `eval`. * `hy` now only implicitly launches a REPL if standard input is a TTY. * `hy -i` has been overhauled to work as a flag like `python3 -i`. * `hy2py` now requires `-m` to specify modules, and uses diff --git a/hy/__init__.py b/hy/__init__.py index 263f8c2ef..03fc2e2d5 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -40,7 +40,7 @@ def __getattr__(self, s): read_many="hy.reader", mangle="hy.reader", unmangle="hy.reader", - eval=["hy.compiler", "hy_eval"], + eval=["hy.compiler", "hy_eval_user"], repr=["hy.core.hy_repr", "hy_repr"], repr_register=["hy.core.hy_repr", "hy_repr_register"], gensym="hy.core.util", diff --git a/hy/compiler.py b/hy/compiler.py index ba2bdd5e2..83c9aa7d1 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -686,12 +686,13 @@ def get_compiler_module(module=None, compiler=None, calling_frame=False): def hy_eval( hytree, - locals=None, + locals, module=None, compiler=None, filename=None, source=None, import_stdlib=True, + globals=None, ): """Evaluates a quoted expression and returns the value. @@ -749,13 +750,6 @@ def hy_eval( module = get_compiler_module(module, compiler, True) - if locals is None: - frame = inspect.stack()[1][0] - locals = inspect.getargvalues(frame).locals - - if not isinstance(locals, dict): - raise TypeError("Locals must be a dictionary") - # Does the Hy AST object come with its own information? filename = getattr(hytree, "filename", filename) or "" source = getattr(hytree, "source", source) @@ -770,11 +764,54 @@ def hy_eval( import_stdlib=import_stdlib, ) + if globals is None: + globals = module.__dict__ + # Two-step eval: eval() the body of the exec call - eval(ast_compile(_ast, filename, "exec"), module.__dict__, locals) + eval(ast_compile(_ast, filename, "exec"), globals, locals) # Then eval the expression context and return that - return eval(ast_compile(expr, filename, "eval"), module.__dict__, locals) + return eval(ast_compile(expr, filename, "eval"), globals, locals) + + +def hy_eval_user(model, globals = None, locals = None, module = None): + # This function is advertised as `hy.eval`. + """An equivalent of Python's :func:`eval` for evaluating Hy code. The chief difference is that the first argument should be a :ref:`model ` rather than source text. If you have a string of source text you want to evaluate, convert it to a model first with :hy:func:`hy.read` or :hy:func:`hy.read-many`:: + + (hy.eval '(+ 1 1)) ; => 2 + (hy.eval (hy.read "(+ 1 1)")) ; => 2 + + The optional arguments ``globals`` and ``locals`` work as in the case of ``eval``. There's one more optional argument, ``module``, which can be a module object or a string naming a module. The module's ``__dict__`` attribute can fill in for ``globals`` (and hence also for ``locals``) if ``module`` is provided but ``globals`` isn't, but the primary purpose of ``module`` is to control where macro calls are looked up. Without this argument, the calling module of ``hy.eval`` is used instead. :: + + (defmacro my-test-mac [] 3) + (hy.eval '(my-test-mac)) ; => 3 + (import hyrule) + (hy.eval '(my-test-mac) :module hyrule) ; NameError + (hy.eval '(list-n 3 1) :module hyrule) ; => [1 1 1]""" + + if locals is None: + locals = globals + hy_was = None + if locals and 'hy' in locals: + hy_was = (locals['hy'],) + try: + value = hy_eval( + hytree = model, + globals = globals, + locals = (inspect.getargvalues(inspect.stack()[1][0]).locals + if locals is None and module is None + else locals), + module = get_compiler_module(module, None, True)) + finally: + if locals is not None: + if hy_was: + # Restore the old value of `hy`. + locals['hy'], = hy_was + else: + # Remove the implicitly added `hy` (if execution + # reached far enough to add it). + locals.pop('hy', None) + return value def hy_compile( diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 56e903894..fc92493e1 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -3,8 +3,9 @@ import pytest +import hy from hy._compat import PY3_11 -from hy.compiler import hy_compile, hy_eval +from hy.compiler import hy_compile from hy.errors import HyError, HyLanguageError from hy.reader import read_many from hy.reader.exceptions import LexException, PrematureEndOfInput @@ -24,7 +25,7 @@ def can_compile(expr, import_stdlib=False, iff=True): def can_eval(expr): - return hy_eval(read_many(expr)) + return hy.eval(read_many(expr)) def cant_compile(expr): diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index 921fbaf7e..4a7d66c0c 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -8,7 +8,7 @@ import pytest import hy -from hy.compiler import hy_compile, hy_eval +from hy.compiler import hy_compile from hy.errors import HyLanguageError, hy_exc_handler from hy.importer import HyLoader from hy.reader import read_many @@ -103,7 +103,7 @@ def import_from_path(path): def test_eval(): def eval_str(s): - return hy_eval(hy.read(s), filename="", source=s) + return hy.eval(hy.read(s)) assert eval_str("[1 2 3]") == [1, 2, 3] assert eval_str('{"dog" "bark" "cat" "meow"}') == {"dog": "bark", "cat": "meow"} diff --git a/tests/native_tests/hy_eval.hy b/tests/native_tests/hy_eval.hy index 7d3179c1b..f758e8b85 100644 --- a/tests/native_tests/hy_eval.hy +++ b/tests/native_tests/hy_eval.hy @@ -39,15 +39,6 @@ (assert (= (hy.eval '#{}) #{}))) -(defn test-eval-global-dict [] - (assert (= 'bar (hy.eval (quote foo) {"foo" 'bar}))) - (assert (= 1 (do (setv d {}) (hy.eval '(setv x 1) d) (hy.eval (quote x) d)))) - (setv d1 {} d2 {}) - (hy.eval '(setv x 1) d1) - (with [e (pytest.raises NameError)] - (hy.eval (quote x) d2))) - - (defn test-eval-quasiquote [] ; https://github.com/hylang/hy/issues/1174 @@ -80,6 +71,148 @@ (assert (= (hy.eval `(get ~d ~k)) 2))) +(setv outer "O") + +(defn test-globals [] + (assert (= (hy.eval 'foo {"foo" 2}) 2)) + (with [(pytest.raises NameError)] + (hy.eval 'foo {})) + + (assert (= outer "O")) + (assert (= (hy.eval 'outer) "O")) + (with [(pytest.raises NameError)] + (hy.eval 'outer {})) + + (hy.eval '(do + (global outer) + (setv outer "O2"))) + (assert (= outer "O2")) + + (hy.eval :globals {"outer" "I"} '(do + (global outer) + (setv outer "O3"))) + (assert (= outer "O2")) + + ; If `globals` is provided but not `locals`, then `globals` + ; substitutes in for `locals`. + (defn try-it [#** eval-args] + (setv d (dict :g1 1 :g2 2)) + (hy.eval :globals d #** eval-args '(do + (global g2 g3) + (setv g2 "newv" g3 3 l 4))) + (del (get d "__builtins__")) + d) + (setv ls {}) + (assert (= (try-it) (dict :g1 1 :g2 "newv" :g3 3 :l 4))) + (assert (= (try-it :locals ls) (dict :g1 1 :g2 "newv" :g3 3))) + (assert (= ls {"l" 4})) + + ; If `module` is provided but `globals` isn't, the dictionary of + ; `module` is used for globals. If `locals` also isn't provided, + ; the same dictionary is used for that, too. + (import string) + (assert (= + (hy.eval 'digits :module string) + "0123456789")) + (assert (= + (hy.eval 'digits :module "string") + "0123456789")) + (assert (= + (hy.eval 'digits :module string :globals {"digits" "boo"}) + "boo")) + (with [(pytest.raises NameError)] + (hy.eval 'digits :module string :globals {})) + (hy.eval :module string '(do + (global hytest-string-g) + (setv hytest-string-g "hi") + (setv hytest-string-l "bye"))) + (assert (= string.hytest-string-g "hi")) + (assert (= string.hytest-string-l "bye"))) + + +(defn test-locals [] + (assert (= (hy.eval 'foo :locals {"foo" 2}) 2)) + (with [(pytest.raises NameError)] + (hy.eval 'foo :locals {})) + + (setv d (dict :l1 1 :l2 2 :hippopotamus "local_v")) + (hy.eval :locals d '(do + (global hippopotamus) + (setv l2 "newv" l3 3 hippopotamus "global_v"))) + (assert (= d (dict :l1 1 :l2 "newv" :l3 3 :hippopotamus "local_v"))) + (assert (= (get (globals) "hippopotamus") "global_v")) + (assert (= hippopotamus "global_v")) + + ; `hy` is implicitly available even when `locals` and `globals` are + ; provided. + (assert (= + (hy.eval :locals {"foo" "A"} :globals {"bar" "B"} + '(hy.repr (+ foo bar))) + #[["AB"]])) + ; Even though `hy.eval` deletes the `hy` implicitly added to + ; `locals`, references in returned code still work. + (setv d {"a" 1}) + (setv f (hy.eval '(fn [] (hy.repr "hello")) :locals d)) + (assert (= d {"a" 1})) + (assert (= (f) #[["hello"]]))) + + +(defn test-globals-and-locals [] + (setv gd (dict :g1 "apple" :g2 "banana")) + (setv ld (dict :l1 "Austin" :l2 "Boston")) + (hy.eval :globals gd :locals ld '(do + (global g2 g3) + (setv g2 "newg-val" g3 "newg-var" l2 "newl-val" l3 "newl-var"))) + (del (get gd "__builtins__")) + (assert (= gd (dict :g1 "apple" :g2 "newg-val" :g3 "newg-var"))) + (assert (= ld (dict :l1 "Austin" :l2 "newl-val" :l3 "newl-var")))) + + +(defn test-no-extra-hy-removal [] + "`hy.eval` shouldn't remove `hy` from a provided namespace if it + was already there." + (setv g {}) + (exec "import hy" g) + (assert (= (hy.eval '(hy.repr [1 2]) g) "[1 2]")) + (assert (in "hy" g))) + + +(defmacro test-macro [] + '(setv blah "test from here")) +(defmacro cheese [] + "gorgonzola") + +(defn test-macros [] + (setv M "tests.resources.macros") + + ; Macros defined in `module` can be called. + (assert (= (hy.eval '(do (test-macro) blah)) "test from here")) + (assert (= (hy.eval '(do (test-macro) blah) :module M) 1)) + + ; `defmacro` creates a new macro in the module. + (hy.eval '(defmacro bilb-ono [] "creative consulting") :module M) + (assert (= (hy.eval '(bilb-ono) :module M) "creative consulting")) + (with [(pytest.raises NameError)] + (hy.eval '(bilb-ono))) + + ; When `module` is provided, implicit access to macros in the + ; current scope is lost. + (assert (= (hy.eval '(cheese)) "gorgonzola")) + (with [(pytest.raises NameError)] + (hy.eval '(cheese) :module M)) + + ; You can still use `require` inside `hy.eval`. + (hy.eval '(require tests.resources.tlib [qplah])) + (assert (= (hy.eval '(qplah 1)) [8 1]))) + + +(defn test-filename [] + (setv m (hy.read "(/ 1 0)" :filename "bad_math.hy")) + (with [e (pytest.raises ZeroDivisionError)] + (hy.eval m)) + (assert (in "bad_math.hy" (get (hy.M.traceback.format-tb e.tb) -1)))) + + (defn test-eval-failure [] ; yo dawg (with [(pytest.raises TypeError)] (hy.eval '(hy.eval))) diff --git a/tests/test_reader.py b/tests/test_reader.py index 01c2ef687..6127a11cc 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -672,13 +672,11 @@ def test_read_error(): pointing to the source position where the error arose.""" import traceback - - from hy.compiler import hy_eval + import hy from hy.errors import HySyntaxError, hy_exc_handler - from hy.reader import read with pytest.raises(HySyntaxError) as e: - hy_eval(read("(do (defn))")) + hy.eval(hy.read("(do (defn))")) assert "".join(traceback.format_exception_only(e.type, e.value)).startswith( ' File "", line 1\n (do (defn))\n ^\n' ) From bc88a4fd7074d88f06bf80f8de16ccf15c53f044 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 7 Aug 2023 15:01:18 -0400 Subject: [PATCH 249/342] Remove the docstring of `hy.compiler.hy_eval` It's not quite correct. It may've been once, but it hasn't been for a while (e.g., `read-str` is long gone). Time that might be spent on making sure the description is correct would be better spent on cleaning up the implementation. The larger issue is that we already have a lot of undermaintained user documentation; purely internal documentation is an additional maintenance burden. --- hy/compiler.py | 53 -------------------------------------------------- 1 file changed, 53 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 83c9aa7d1..8d9868906 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -694,59 +694,6 @@ def hy_eval( import_stdlib=True, globals=None, ): - """Evaluates a quoted expression and returns the value. - - If you're evaluating hand-crafted AST trees, make sure the line numbers - are set properly. Try `fix_missing_locations` and related functions in the - Python `ast` library. - - Examples: - :: - - => (hy.eval '(print "Hello World")) - "Hello World" - - If you want to evaluate a string, use ``read-str`` to convert it to a - form first:: - - => (hy.eval (hy.read-str "(+ 1 1)")) - 2 - - Args: - hytree (Object): - The Hy AST object to evaluate. - - locals (Optional[dict]): - Local environment in which to evaluate the Hy tree. Defaults to the - calling frame. - - module (Optional[Union[str, types.ModuleType]]): - Module, or name of the module, to which the Hy tree is assigned and - the global values are taken. - The module associated with `compiler` takes priority over this value. - When neither `module` nor `compiler` is specified, the calling frame's - module is used. - - compiler (Optional[HyASTCompiler]): - An existing Hy compiler to use for compilation. Also serves as - the `module` value when given. - - filename (Optional[str]): - The filename corresponding to the source for `tree`. This will be - overridden by the `filename` field of `tree`, if any; otherwise, it - defaults to "". When `compiler` is given, its `filename` field - value is always used. - - source (Optional[str]): - A string containing the source code for `tree`. This will be - overridden by the `source` field of `tree`, if any; otherwise, - if `None`, an attempt will be made to obtain it from the module given by - `module`. When `compiler` is given, its `source` field value is always - used. - - Returns: - Any: Result of evaluating the Hy compiled tree. - """ module = get_compiler_module(module, compiler, True) From 8c3907239567016d6d8ae660e81211fe12998f32 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 8 Aug 2023 15:33:51 -0400 Subject: [PATCH 250/342] Remove some undocumented behavior of `hy.eval` in which angle-bracketed module names were treated specially. --- hy/compiler.py | 5 +---- hy/importer.py | 4 +++- tests/native_tests/comprehensions.hy | 2 +- tests/native_tests/let.hy | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 8d9868906..55070b87b 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -670,10 +670,7 @@ def get_compiler_module(module=None, compiler=None, calling_frame=False): module = getattr(compiler, "module", None) or module if isinstance(module, str): - if module.startswith("<") and module.endswith(">"): - module = types.ModuleType(module) - else: - module = importlib.import_module(mangle(module)) + module = importlib.import_module(mangle(module)) if calling_frame and not module: module = calling_module(n=2) diff --git a/hy/importer.py b/hy/importer.py index 2f7ce9519..7d9f39e4d 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -135,10 +135,12 @@ def _hy_source_to_code(self, data, path, _optimize=-1): def _hy_compile_source(pathname, source): if not pathname.endswith(".hy"): return _py_compile_source(pathname, source) + mname = f"" + sys.modules[mname] = types.ModuleType(mname) return compile( hy_compile( read_many(source.decode("UTF-8"), filename=pathname, skip_shebang=True), - f"", + sys.modules[mname], ), pathname, "exec", diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy index 7280cafed..bc7166759 100644 --- a/tests/native_tests/comprehensions.hy +++ b/tests/native_tests/comprehensions.hy @@ -264,7 +264,7 @@ (defmacro eval-isolated [#* body] - `(hy.eval '(do ~@body) :module "" :locals {})) + `(hy.eval '(do ~@body) :module (hy.M.types.ModuleType "") :locals {})) (defn test-lfor-nonlocal [] diff --git a/tests/native_tests/let.hy b/tests/native_tests/let.hy index ffed3f376..591612179 100644 --- a/tests/native_tests/let.hy +++ b/tests/native_tests/let.hy @@ -506,7 +506,7 @@ (defmacro eval-isolated [#* body] - `(hy.eval '(do ~@body) :module "" :locals {})) + `(hy.eval '(do ~@body) :module (hy.M.types.ModuleType "") :locals {})) (defn test-let-bound-nonlocal [] From c9796af071afda018fadc1e820585bb39122dbb0 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 8 Aug 2023 14:30:07 -0400 Subject: [PATCH 251/342] Small documentation improvements --- docs/api.rst | 6 +++--- docs/syntax.rst | 4 +++- hy/models.py | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 240dc355e..2de2cba55 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -67,17 +67,17 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. As a convenience, ``.`` supports two other kinds of arguments in place of a plain attribute. A parenthesized expression is understood as a method call: - ``(. foo (bar a b))`` compiles to ``x.foo.bar(a, b)``. A bracketed form is + ``(. foo (bar a b))`` compiles to ``foo.bar(a, b)``. A bracketed form is understood as a subscript: ``(. foo ["bar"])`` compiles to ``foo["bar"]``. All these options can be mixed and matched in a single ``.`` call, so :: - (. a (b 1 2) c [d] [(e)]) + (. a (b 1 2) c [d] [(e 3 4)]) compiles to .. code-block:: python - a.b(1, 2).c[d][e()] + a.b(1, 2).c[d][e(3, 4)] :ref:`Dotted identifiers ` provide syntactic sugar for common uses of this macro. In particular, syntax like ``foo.bar`` ends up diff --git a/docs/syntax.rst b/docs/syntax.rst index 5773cbb73..6b051878d 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -189,7 +189,9 @@ equivalent to ``{"a" 1 "b" 2}``, which is different from ``{:a 1 :b 2}`` (see :ref:`dict-literals`). The empty keyword ``:`` is syntactically legal, but you can't compile a -function call with an empty keyword argument. +function call with an empty keyword argument due to Python limitations. Thus +``(foo : 3)`` must be rewritten to use runtime unpacking, as in +``(foo #** {"" 3})``. .. autoclass:: hy.models.Keyword :members: __bool__, __call__ diff --git a/hy/models.py b/hy/models.py index abe9701d8..4ddf06812 100644 --- a/hy/models.py +++ b/hy/models.py @@ -203,7 +203,7 @@ class Symbol(Object, str): """ Represents a symbol. - Symbol objects behave like strings under operations like :hy:func:`get`, + Symbol objects behave like strings under operations like :hy:func:`get `, :func:`len`, and :class:`bool`; in particular, ``(bool (hy.models.Symbol "False"))`` is true. Use :hy:func:`hy.eval` to evaluate a symbol. """ @@ -283,8 +283,8 @@ def __call__(self, data, default=_sentinel): :class:`hy.models.Keyword` objects). The optional second parameter is a default value; if provided, any - :class:`KeyError` from :hy:func:`get` will be caught, and the default returned - instead.""" + :class:`KeyError` from :hy:func:`get ` will be caught, + and the default returned instead.""" from hy.reader import mangle From 9b162c5de9c6c34c04e105927acbde04b6706b29 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 8 Aug 2023 14:59:23 -0400 Subject: [PATCH 252/342] Simplify some gensym tests --- tests/native_tests/hy_misc.hy | 8 ++++---- tests/native_tests/macros.hy | 35 +++++++++++------------------------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index 63c0492b1..124aad571 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -8,12 +8,12 @@ (defn test-gensym [] (setv s1 (hy.gensym)) (assert (isinstance s1 hy.models.Symbol)) - (assert (= 0 (.find s1 "_G\uffff"))) + (assert (.startswith s1 "_G\uffff")) (setv s2 (hy.gensym "xx")) (setv s3 (hy.gensym "xx")) - (assert (= 0 (.find s2 "_xx\uffff"))) - (assert (not (= s2 s3))) - (assert (not (= (str s2) (str s3))))) + (assert (.startswith s2 "_xx\uffff")) + (assert (!= s2 s3)) + (assert (!= (str s2) (str s3)))) (defmacro mac [x expr] diff --git a/tests/native_tests/macros.hy b/tests/native_tests/macros.hy index 01bc44365..f35789988 100644 --- a/tests/native_tests/macros.hy +++ b/tests/native_tests/macros.hy @@ -120,31 +120,18 @@ (assert initialized) (assert (test-initialized)) + +(defmacro gensym-example [] + `(setv ~(hy.gensym) 1)) + (defn test-gensym-in-macros [] - (import ast) - (import hy.compiler [hy-compile]) - (import hy.reader [read-many]) - (setv macro1 "(defmacro nif [expr pos zero neg] - (setv g (hy.gensym)) - `(do - (setv ~g ~expr) - (cond (> ~g 0) ~pos - (= ~g 0) ~zero - (< ~g 0) ~neg))) - - (print (nif (inc -1) 1 0 -1)) - ") - ;; expand the macro twice, should use a different - ;; gensym each time - (setv _ast1 (hy-compile (read-many macro1) __name__)) - (setv _ast2 (hy-compile (read-many macro1) __name__)) - (setv s1 (ast.unparse _ast1)) - (setv s2 (ast.unparse _ast2)) - ;; and make sure there is something new that starts with _G\uffff - (assert (in (hy.mangle "_G\uffff") s1)) - (assert (in (hy.mangle "_G\uffff") s2)) - ;; but make sure the two don't match each other - (assert (not (= s1 s2)))) + ; Call `gensym-example` twice, getting a distinct gensym each time. + (defclass C [] + (gensym-example) + (gensym-example)) + (assert (= + (len (sfor a (dir C) :if (not (.startswith a "__")) a)) + 2))) (defn test-macro-errors [] From ebc95dddd3f252c87da7567e54f5b100cfff5d9f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 10 Aug 2023 14:10:44 -0400 Subject: [PATCH 253/342] Change the gensym format The new format uses the `_hy_` prefix the we've already officially reserved for a while, and it doesn't need mangling. Since I construe the gensym format as an internal matter, it remains undocumented. --- hy/core/util.hy | 20 ++++++++++++-------- tests/native_tests/hy_misc.hy | 7 ++++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/hy/core/util.hy b/hy/core/util.hy index 27aba123d..a8002b438 100644 --- a/hy/core/util.hy +++ b/hy/core/util.hy @@ -39,7 +39,7 @@ (setv _gensym_counter 0) (setv _gensym_lock (Lock)) -(defn gensym [[g "G"]] +(defn gensym [[g ""]] #[[Generate a symbol with a unique name. The argument will be included in the generated symbol, as an aid to debugging. Typically one calls ``hy.gensym`` without an argument. @@ -65,14 +65,18 @@ 4) (print (selfadd (f)))]] - (setv new_symbol None) - (global _gensym_counter) - (global _gensym_lock) (.acquire _gensym_lock) - (try (do (setv _gensym_counter (+ _gensym_counter 1)) - (setv new_symbol (Symbol (.format "_{}\uffff{}" g _gensym_counter)))) - (finally (.release _gensym_lock))) - new_symbol) + (try + (global _gensym_counter) + (+= _gensym_counter 1) + (setv n _gensym_counter) + (finally (.release _gensym_lock))) + (setv g (hy.mangle (.format "_hy_gensym_{}_{}" g n))) + (Symbol (if (.startswith g "_hyx_") + ; Remove the mangle prefix, if it's there, so the result always + ; starts with our reserved prefix `_hy_`. + (+ "_" (cut g (len "_hyx_") None)) + g))) (defn _calling-module-name [[n 1]] "Get the name of the module calling `n` levels up the stack from the diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index 124aad571..35f62f609 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -8,12 +8,13 @@ (defn test-gensym [] (setv s1 (hy.gensym)) (assert (isinstance s1 hy.models.Symbol)) - (assert (.startswith s1 "_G\uffff")) + (assert (.startswith s1 "_hy_gensym__")) (setv s2 (hy.gensym "xx")) (setv s3 (hy.gensym "xx")) - (assert (.startswith s2 "_xx\uffff")) + (assert (.startswith s2 "_hy_gensym_xx_")) (assert (!= s2 s3)) - (assert (!= (str s2) (str s3)))) + (assert (!= (str s2) (str s3))) + (assert (.startswith (hy.gensym "•ab") "_hy_gensym_XbulletXab_"))) (defmacro mac [x expr] From 3206396a5c04b5b9338a6955b1d09994d74b598a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 8 Aug 2023 16:05:15 -0400 Subject: [PATCH 254/342] Forbid `~@ #*` --- NEWS.rst | 1 + hy/core/result_macros.py | 2 ++ tests/native_tests/quote.hy | 10 ++++++++++ 3 files changed, 13 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index c91665a85..184314f8d 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -27,6 +27,7 @@ Bug Fixes * Double quotes inside of bracketed f-strings are now properly handled. * Fixed incomplete recognition of macro calls with a unary dotted head like `((. defn) f [])`. +* `~@ #*` now produces a syntax error instead of a nonsensical result. 0.27.0 (released 2023-07-06) ============================= diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 2db198d50..c0849b87e 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -222,6 +222,8 @@ def render_quoted_form(compiler, form, level): for x in form: f_contents, splice = render_quoted_form(compiler, x, level) if splice: + if is_unpack("iterable", f_contents): + raise compiler._syntax_error(f_contents, "`unpack-iterable` is not allowed here") f_contents = Expression( [ Symbol("unpack-iterable"), diff --git a/tests/native_tests/quote.hy b/tests/native_tests/quote.hy index 0f874c691..156a4161f 100644 --- a/tests/native_tests/quote.hy +++ b/tests/native_tests/quote.hy @@ -1,3 +1,7 @@ +(import + pytest) + + (defn test-quote [] (setv q (quote (a b c))) (assert (= (len q) 3)) @@ -83,3 +87,9 @@ [1 2 3] [4 5 6]) [4 5 6]))) + + +(defn test-unquote-splice-unpack [] + ; https://github.com/hylang/hy/issues/2336 + (with [(pytest.raises hy.errors.HySyntaxError)] + (hy.eval '`[~@ #* [[1]]]))) From f237bbb6da432072d8f8c45d4cc0779b67a19650 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Mon, 14 Aug 2023 17:03:40 +0200 Subject: [PATCH 255/342] fix broken link --- hy/core/macros.hy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index e5eb4a779..b3a81553c 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -87,7 +87,7 @@ => #slice a:(+ 1 2):\"column\" (slice 42 3 column) - See the :ref:`reader macros docs ` for more detailed + See the :ref:`reader macros docs ` for more detailed information on how reader macros work and are defined. " (when (not (isinstance &compiler.scope hy.scoping.ScopeGlobal)) From 4917db8a70e01766960c94e6d71bc88af578a602 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Mon, 14 Aug 2023 17:34:03 +0200 Subject: [PATCH 256/342] improve docs for reader macros --- docs/api.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index 2de2cba55..7107cbdc7 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1321,6 +1321,11 @@ the following methods Reader Macros ------------- +Reader macros allow one to hook directly into Hy's reader to customize how +different forms are parsed. Reader macros can be imported from other libraries +using :hy:func:`require`, and can be defined directly using +:hy:func:`defreader`. + Like regular macros, reader macros should return a Hy form that will then be passed to the compiler for execution. Reader macros access the Hy reader using the ``&reader`` name. It gives access to all of the text- and form-parsing logic @@ -1328,6 +1333,30 @@ that Hy uses to parse itself. See :py:class:`HyReader ` for details regarding the available processing methods. +Note that Hy reads and parses each top-level form completely before it is executed, +so the following code will throw an exception: + +.. code-block:: hylang + + => (do + ... (defreader up + ... (.slurp-space &reader) + ... (.upper (.read-one-form &reader))) + ... (print #up "hello?")) + ;; !! ERROR reader macro '#up' is not defined + +Since the entire ``do`` block is read at once, the ``defreader`` will not have +yet been evaluated when the parser encounters the call to ``#up``. However, if +the reader macro isn't used until a later top-level form, then it will work: + +.. code-block:: hylang + + => (defreader up + ... (.slurp-space &reader) + ... (.upper (.read-one-form &reader))) + => (print #up "hy there!") + HY THERE! + .. autoclass:: hy.reader.hy_reader.HyReader :members: parse, parse_one_form, parse_forms_until, read_default, fill_pos From e03d5c1a48bff74121fb707aa54ec4e9b8f6dcb6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 13 Aug 2023 10:21:06 -0400 Subject: [PATCH 257/342] Remove some old leftover debugging prints --- tests/native_tests/functions.hy | 11 ++++++++--- tests/native_tests/model_patterns.hy | 1 - tests/resources/importer/circular.hy | 1 - tests/resources/importer/foo/__init__.hy | 1 - tests/resources/importer/foo/__init__.py | 1 - tests/resources/importer/foo/some_mod.hy | 1 - tests/resources/importer/foo/some_mod.py | 1 - tests/test_bin.py | 3 +-- 8 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/native_tests/functions.hy b/tests/native_tests/functions.hy index a1095cc25..a2fd36d26 100644 --- a/tests/native_tests/functions.hy +++ b/tests/native_tests/functions.hy @@ -299,12 +299,17 @@ (defn test-yield-in-try [] + (setv hit-finally False) (defn gen [] (setv x 1) - (try (yield x) - (finally (print x)))) + (try + (yield x) + (finally + (nonlocal hit-finally) + (setv hit-finally True)))) (setv output (list (gen))) - (assert (= [1] output))) + (assert (= [1] output)) + (assert hit-finally)) (defn test-midtree-yield [] diff --git a/tests/native_tests/model_patterns.hy b/tests/native_tests/model_patterns.hy index dd3ccebb9..8aa5e6ee7 100644 --- a/tests/native_tests/model_patterns.hy +++ b/tests/native_tests/model_patterns.hy @@ -39,7 +39,6 @@ (defn f [loopers] (setv head (if loopers (get loopers 0) None)) (setv tail (cut loopers 1 None)) - (print head) (cond (is head None) `(do ~@body) diff --git a/tests/resources/importer/circular.hy b/tests/resources/importer/circular.hy index 4825b5f29..c5120a67c 100644 --- a/tests/resources/importer/circular.hy +++ b/tests/resources/importer/circular.hy @@ -2,4 +2,3 @@ (defn f [] (import circular) circular.a) -(print (f)) diff --git a/tests/resources/importer/foo/__init__.hy b/tests/resources/importer/foo/__init__.hy index 5909d57b4..2902aa116 100644 --- a/tests/resources/importer/foo/__init__.hy +++ b/tests/resources/importer/foo/__init__.hy @@ -1,2 +1 @@ -(print "This is __init__.hy") (setv ext "hy") diff --git a/tests/resources/importer/foo/__init__.py b/tests/resources/importer/foo/__init__.py index 7e89b4607..306d46cb8 100644 --- a/tests/resources/importer/foo/__init__.py +++ b/tests/resources/importer/foo/__init__.py @@ -1,2 +1 @@ -print("This is __init__.py") ext = "py" diff --git a/tests/resources/importer/foo/some_mod.hy b/tests/resources/importer/foo/some_mod.hy index 10db45cff..2902aa116 100644 --- a/tests/resources/importer/foo/some_mod.hy +++ b/tests/resources/importer/foo/some_mod.hy @@ -1,2 +1 @@ -(print "This is test_mod.hy") (setv ext "hy") diff --git a/tests/resources/importer/foo/some_mod.py b/tests/resources/importer/foo/some_mod.py index 40c80e5c4..306d46cb8 100644 --- a/tests/resources/importer/foo/some_mod.py +++ b/tests/resources/importer/foo/some_mod.py @@ -1,2 +1 @@ -print("This is test_mod.py") ext = "py" diff --git a/tests/test_bin.py b/tests/test_bin.py index a4c41389d..f283b3e1d 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -131,7 +131,6 @@ def test_error_parts_length(): # Make sure lines are printed in between arrows separated by more than one # character. _, err = run_cmd("hy -i", prg_str.format(3, 3, 1, 6)) - print(err) msg_idx = err.rindex("HyLanguageError:") assert msg_idx @@ -420,7 +419,7 @@ def req_err(x): assert len(error_lines) <= 4 req_err(error_lines[-1]) - output, error = run_cmd('hy -i -c "(require not-a-real-module)"') + output, error = run_cmd('hy -i -c "(require not-a-real-module)"', '') assert output.startswith("=> ") req_err(error.splitlines()[2]) From fe3edd749aa22b09188353b98648189e7adad347 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 12 Aug 2023 10:10:38 -0400 Subject: [PATCH 258/342] Correct a documentation example --- hy/reader/mangling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/reader/mangling.py b/hy/reader/mangling.py index b12100dd0..0ce46d94c 100644 --- a/hy/reader/mangling.py +++ b/hy/reader/mangling.py @@ -12,7 +12,7 @@ def mangle(s): to :ref:`Hy's mangling rules `. :: (hy.mangle 'foo-bar) ; => "foo_bar" - (hy.mangle "🦑") ; => "hyx_squid" + (hy.mangle "🦑") ; => "hyx_XsquidX" If the stringified argument is already both legal as a Python identifier and normalized according to Unicode normalization form KC (NFKC), it will From 1830f669da48697aefb13ae746ed9b6432213533 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 11 Aug 2023 09:28:12 -0400 Subject: [PATCH 259/342] Simplify `compile_quote` --- hy/core/result_macros.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index c0849b87e..8dbce14f9 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -176,11 +176,9 @@ def compile_inline_python(compiler, expr, root, code): @pattern_macro(["quote", "quasiquote"], [FORM]) def compile_quote(compiler, expr, root, arg): - level = Inf if root == "quote" else 0 # Only quasiquotes can unquote - stmts, _ = render_quoted_form(compiler, arg, level) - ret = compiler.compile(stmts) - return ret - + return compiler.compile(render_quoted_form(compiler, arg, + level = Inf if root == "quote" else 0)[0]) + # Only quasiquotes can unquote def render_quoted_form(compiler, form, level): """ From e3382143daff318c662485a2d6f650162462bb9f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 11 Aug 2023 09:42:25 -0400 Subject: [PATCH 260/342] Avoid a call to `hy.unmangle` to avoid crashes from names that look like malformed mangles. --- hy/core/result_macros.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 8dbce14f9..a09ecbb26 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -196,7 +196,7 @@ def render_quoted_form(compiler, form, level): op = None if isinstance(form, Expression) and form and isinstance(form[0], Symbol): - op = unmangle(mangle(form[0])) + op = mangle(form[0]).replace('_', '-') if level == 0 and op in ("unquote", "unquote-splice"): if len(form) != 2: raise HyTypeError( From 9e42b6e74ce376dbc1b7632e844032a1b1e7066b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 11 Aug 2023 09:52:42 -0400 Subject: [PATCH 261/342] Minor simplification of `render_quoted_form` --- hy/core/result_macros.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index a09ecbb26..b17b8730a 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -197,20 +197,18 @@ def render_quoted_form(compiler, form, level): op = None if isinstance(form, Expression) and form and isinstance(form[0], Symbol): op = mangle(form[0]).replace('_', '-') - if level == 0 and op in ("unquote", "unquote-splice"): - if len(form) != 2: - raise HyTypeError( - "`%s' needs 1 argument, got %s" % op, - len(form) - 1, - compiler.filename, - form, - compiler.source, - ) - return form[1], op == "unquote-splice" - elif op == "quasiquote": - level += 1 - elif op in ("unquote", "unquote-splice"): - level -= 1 + if op in ("unquote", "unquote-splice", "quasiquote"): + if level == 0 and op != "quasiquote": + if len(form) != 2: + raise HyTypeError( + "`%s' needs 1 argument, got %s" % op, + len(form) - 1, + compiler.filename, + form, + compiler.source, + ) + return form[1], op == "unquote-splice" + level += 1 if op == "quasiquote" else -1 name = form.__class__.__name__ body = [form] From c9df0b17a3bf0cd761138fe406317b5dd9dceeb3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 14 Aug 2023 12:30:00 -0400 Subject: [PATCH 262/342] Clean up `native-tests.quote` --- tests/native_tests/quote.hy | 109 +++++++++++++++++------------------- 1 file changed, 50 insertions(+), 59 deletions(-) diff --git a/tests/native_tests/quote.hy b/tests/native_tests/quote.hy index 156a4161f..08cb790a7 100644 --- a/tests/native_tests/quote.hy +++ b/tests/native_tests/quote.hy @@ -1,95 +1,86 @@ +"Tests of `quote` and `quasiquote`." + + (import pytest) - -(defn test-quote [] - (setv q (quote (a b c))) - (assert (= (len q) 3)) - (assert (= q (hy.models.Expression [(quote a) (quote b) (quote c)])))) +(setv E hy.models.Expression) +(setv S hy.models.Symbol) -(defn test-basic-quoting [] - (assert (= (type (quote (foo bar))) hy.models.Expression)) - (assert (= (type (quote foo)) hy.models.Symbol)) - (assert (= (type (quote "string")) hy.models.String)) - (assert (= (type (quote b"string")) hy.models.Bytes))) +(defn test-quote-basic [] + (assert (= '3 (hy.models.Integer 3))) + (assert (= 'a (S "a"))) + (assert (= 'False (S "False"))) + (assert (= '"hello" (hy.models.String "hello"))) + (assert (= 'b"hello" (hy.models.Bytes b"hello"))) + (assert (= '(a b) (E [(S "a") (S "b")]))) + (assert (= + '{foo bar baz quux} + (hy.models.Dict (map S ["foo" "bar" "baz" "quux"]))))) (defn test-quoted-hoistable [] - (setv f (quote (if True True True))) - (assert (= (get f 0) (quote if))) - (assert (= (cut f 1 None) (quote (True True True))))) + (setv f '(if True True True)) + (assert (= (get f 0) 'if)) + (assert (= (cut f 1 None) '(True True True)))) + + +(defn test-quasiquote-trivial [] + "Quasiquote and quote are equivalent for simple cases." + (assert (= `(a b c) '(a b c)))) (defn test-quoted-macroexpand [] "Don't expand macros in quoted expressions." (require tests.resources.macros [test-macro]) - (setv q1 (quote (test-macro))) - (setv q2 (quasiquote (test-macro))) + (setv q1 '(test-macro)) + (setv q2 `(test-macro)) (assert (= q1 q2)) - (assert (= (get q1 0) (quote test-macro)))) - - -(defn test-quote-dicts [] - (setv q (quote {foo bar baz quux})) - (assert (= (len q) 4)) - (assert (= (get q 0) (quote foo))) - (assert (= (get q 1) (quote bar))) - (assert (= (get q 2) (quote baz))) - (assert (= (get q 3) (quote quux))) - (assert (= (type q) hy.models.Dict))) + (assert (= (get q1 0) 'test-macro))) (defn test-quote-expr-in-dict [] - (setv q (quote {(foo bar) 0})) - (assert (= (len q) 2)) - (setv qq (get q 0)) - (assert (= qq (quote (foo bar))))) - - -(defn test-quasiquote [] - "Quasiquote and quote are equivalent for simple cases." - (setv q (quote (a b c))) - (setv qq (quasiquote (a b c))) - (assert (= q qq))) + (assert (= + '{(foo bar) 0} + (hy.models.Dict [(E [(S "foo") (S "bar")]) (hy.models.Integer 0)])))) (defn test-unquote [] - (setv q (quote (unquote foo))) + (setv q '~foo) (assert (= (len q) 2)) - (assert (= (get q 1) (quote foo))) - (setv qq (quasiquote (a b c (unquote (+ 1 2))))) - (assert (= (len qq) 4)) - (assert (= (hy.as-model qq) (quote (a b c 3))))) + (assert (= (get q 1) 'foo)) + (setv qq `(a b c ~(+ 1 2))) + (assert (= (hy.as-model qq) '(a b c 3)))) (defn test-unquote-splice [] - (setv q (quote (c d e))) + (setv q '(c d e)) (setv qq `(a b ~@q f ~@q ~@0 ~@False ~@None g ~@(when False 1) h)) - (assert (= (len qq) 11)) - (assert (= qq (quote (a b c d e f c d e g h))))) - - -(defn test-nested-quasiquote [] - (setv qq (hy.as-model `(1 `~(+ 1 ~(+ 2 3) ~@None) 4))) - (setv q (quote (1 `~(+ 1 5) 4))) - (assert (= (len q) 3)) - (assert (= (get qq 1) (quote `~(+ 1 5)))) - (assert (= q qq))) + (assert (= qq '(a b c d e f c d e g h)))) -(defmacro doodle [#* body] - `(do ~@body)) +(defmacro doodle [#* args] + `[1 ~@args 2]) -(defn test-unquote-splice [] +(defn test-unquote-splice-in-mac [] (assert (= (doodle - [1 2 3] - [4 5 6]) - [4 5 6]))) + (setv x 5) + (+= x 1) + x) + [1 None None 6 2]))) (defn test-unquote-splice-unpack [] ; https://github.com/hylang/hy/issues/2336 (with [(pytest.raises hy.errors.HySyntaxError)] (hy.eval '`[~@ #* [[1]]]))) + + +(defn test-nested-quasiquote [] + (setv qq (hy.as-model `(1 `~(+ 1 ~(+ 2 3) ~@None) 4))) + (setv q '(1 `~(+ 1 5) 4)) + (assert (= (len q) 3)) + (assert (= (get qq 1) '`~(+ 1 5))) + (assert (= qq q))) From dae8f21c2c45f00b129d8485fe5c6e612a805b22 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 14 Aug 2023 12:26:27 -0400 Subject: [PATCH 263/342] Add more tests of nested quasiquotation --- tests/native_tests/quote.hy | 77 +++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/native_tests/quote.hy b/tests/native_tests/quote.hy index 08cb790a7..2d2771228 100644 --- a/tests/native_tests/quote.hy +++ b/tests/native_tests/quote.hy @@ -84,3 +84,80 @@ (assert (= (len q) 3)) (assert (= (get qq 1) '`~(+ 1 5))) (assert (= qq q))) + + +(defn test-nested-quasiquote--nested-struct [] + (assert (= + (hy.as-model `(try + ~@(lfor + i [1 2 3] + `(setv ~(S (+ "x" (str i))) (+ "x" (str ~i)))) + (finally + (print "done")))) + '(try + (setv x1 (+ "x" (str 1))) + (setv x2 (+ "x" (str 2))) + (setv x3 (+ "x" (str 3))) + (finally + (print "done")))))) + + +(defmacro macroify-programs [#* names] + `(do + ~@(lfor name names + `(defmacro ~name [#* args] + `(.append imaginary-syscalls #(~'~(str name) ~@(map str args))))))) + +(defn test-nested-quasiquote--macro-writing-macro-1 [] + "A test of the construct ~'~ (to substitute in a variable from a + higher-level quasiquote) inspired by + https://github.com/hylang/hy/discussions/2251" + + (setv imaginary-syscalls []) + (macroify-programs ls mkdir touch) + (mkdir mynewdir) + (touch mynewdir/file) + (ls -lA mynewdir) + (assert (= imaginary-syscalls [ + #("mkdir" "mynewdir") + #("touch" "mynewdir/file") + #("ls" "-lA" "mynewdir")]))) + + +(defmacro def-caller [abbrev proc] + `(defmacro ~abbrev [var form] + `(~'~proc + (fn [~var] ~form)))) +(def-caller smoo-caller smoo) + +(defn test-nested-quasiquote--macro-writing-macro-2 [] + "A similar test to the previous one, based on section 3.2 of + Bawden, A. (1999). Quasiquotation in Lisp. ACM SIGPLAN Workshop on Partial Evaluation and Program Manipulation. Retrieved from http://web.archive.org/web/20230105083805id_/https://3e8.org/pub/scheme/doc/Quasiquotation%20in%20Lisp%20(Bawden).pdf" + + (setv accum []) + (defn smoo [f] + (.append accum "entered smoo") + (f "in smoo") + (.append accum "exiting smoo")) + (smoo-caller arg + (.append accum (+ "in the lambda: " arg))) + (assert (= accum [ + "entered smoo" + "in the lambda: in smoo" + "exiting smoo"]))) + + +(defn test-nested-quasiquote--triple [] + "N.B. You can get the same results with an analogous test in Emacs + Lisp or Common Lisp." + + (setv + a 1 b 1 c 1 + x ```[~a ~~b ~~~c] + ; `x` has been implicitly evaluated once. Let's explicitly + ; evaluate it twice more, so no backticks are left. + a 2 b 2 c 2 + x (hy.eval x) + a 3 b 3 c 3 + x (hy.eval x)) + (assert (= (hy.as-model x) '[3 2 1]))) From 151f0a131d9d7b77eb70cf6418ff2e93915f9741 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 13 Aug 2023 11:48:56 -0400 Subject: [PATCH 264/342] Rewrite the documentation of the quoting macros --- docs/api.rst | 113 ++++++++++++++++++++------------------------------- 1 file changed, 43 insertions(+), 70 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 7107cbdc7..e98dac919 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -792,42 +792,60 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. beware that significant leading whitespace in embedded string literals will be removed. -.. hy:macro:: (quasiquote [form]) +.. hy:macro:: (quasiquote [model]) +.. hy:macro:: (unquote [model]) +.. hy:macro:: (unquote-splice [model]) - ``quasiquote`` allows you to quote a form, but also selectively evaluate - expressions. Expressions inside a ``quasiquote`` can be selectively evaluated - using ``unquote`` (``~``). The evaluated form can also be spliced using - ``unquote-splice`` (``~@``). Quasiquote can be also written using the backquote - (`````) symbol. + ``quasiquote`` is like :hy:func:`quote` except that it treats the model as a template, in which certain special :ref:`expressions ` indicate that some code should evaluated and its value substituted there. The idea is similar to C's ``sprintf`` or Python's various string-formatting constructs. For example:: - :strong:`Examples` + (setv x 2) + (quasiquote (+ 1 (unquote x))) ; => '(+ 1 2) - :: + ``unquote`` indicates code to be evaluated, so ``x`` becomes ``2`` and the ``2`` gets inserted in the parent model. ``quasiquote`` can be :ref:`abbreviated ` as a backtick (\`), with no parentheses, and likewise ``unquote`` can be abbreviated as a tilde (``~``), so one can instead write simply :: - ;; let `qux' be a variable with value (bar baz) - `(foo ~qux) - ; equivalent to '(foo (bar baz)) - `(foo ~@qux) - ; equivalent to '(foo bar baz) + `(+ 1 ~x) + (In the bulk of Lisp tradition, unquotation is written ``,``. Hy goes with Clojure's choice of ``~``, which has the advantage of being more visible in most programming fonts.) -.. hy:macro:: (quote [form]) + Quasiquotation is convenient for writing macros:: - ``quote`` returns the form passed to it without evaluating it. ``quote`` can - alternatively be written using the apostrophe (``'``) symbol. + (defmacro set-foo [value] + `(setv foo ~value)) + (set-foo (+ 1 2 3)) + (print foo) ; => 6 - :strong:`Examples` + Another kind of unquotation operator, ``unquote-splice``, abbreviated ``~@``, is analogous to ``unpack-iterable`` in that it splices an iterable object into the sequence of the parent :ref:`sequential model `. Compare the effects of ``unquote`` to ``unquote-splice``:: - :: + (setv X [1 2 3]) + (hy.repr `[a b ~X c d ~@X e f]) + ; => '[a b [1 2 3] c d 1 2 3 e f] + + If ``unquote-splice`` is given any sort of false value (such as ``None``), it's treated as an empty list. To be precise, ``~@x`` splices in the result of ``(or x [])``. + + Note that while a symbol name can begin with ``@`` in Hy, ``~@`` takes precedence in the parser, so if you want to unquote the symbol ``@foo`` with ``~``, you must use whitespace to separate ``~`` and ``@``, as in ``~ @foo``. - => (setv x '(print "Hello World")) - => x ; variable x is set to unevaluated expression - hy.models.Expression([ - hy.models.Symbol('print'), - hy.models.String('Hello World')]) - => (hy.eval x) - Hello World +.. hy:macro:: (quote [model]) + + Return the given :ref:`model ` without evaluating it. Or to be more pedantic, ``quote`` complies to code that produces and returns the model it was originally called on. Thus ``quote`` serves as syntactic sugar for model constructors:: + + (quote a) + ; Equivalent to: (hy.models.Symbol "a") + (quote (+ 1 1)) + ; Equivalent to: (hy.models.Expression [ + ; (hy.models.Symbol "+") + ; (hy.models.Integer 1) + ; (hy.models.Integer 1)]) + + ``quote`` itself is conveniently :ref:`abbreviated ` as the single-quote character ``'``, which needs no parentheses, allowing one to instead write:: + + 'a + '(+ 1 1) + + See also: + - :hy:func:`quasiquote` to substitute values into a quoted form + - :hy:func:`hy.eval` to evaluate models as code + - :hy:func:`hy.repr` to stringify models into Hy source text that uses ``'`` .. hy:macro:: (require [#* args]) @@ -1078,51 +1096,6 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. => (f #* [1] #* [2] #** {"c" 3} #** {"d" 4}) [1 2 3 4] -.. hy:macro:: (unquote [symbol]) - - Within a quasiquoted form, ``unquote`` forces evaluation of a symbol. ``unquote`` - is aliased to the tilde (``~``) symbol. - - :: - - => (setv nickname "Cuddles") - => (quasiquote (= nickname (unquote nickname))) - '(= nickname "Cuddles") - => `(= nickname ~nickname) - '(= nickname "Cuddles") - - -.. hy:macro:: (unquote-splice [symbol]) - - ``unquote-splice`` forces the evaluation of a symbol within a quasiquoted form, - much like ``unquote``. ``unquote-splice`` can be used when the symbol - being unquoted contains an iterable value, as it "splices" that iterable into - the quasiquoted form. ``unquote-splice`` can also be used when the value - evaluates to a false value such as ``None``, ``False``, or ``0``, in which - case the value is treated as an empty list and thus does not splice anything - into the form. ``unquote-splice`` is aliased to the ``~@`` syntax. - - :: - - => (setv nums [1 2 3 4]) - => (quasiquote (+ (unquote-splice nums))) - '(+ 1 2 3 4) - => `(+ ~@nums) - '(+ 1 2 3 4) - => `[1 2 ~@(when (< (get nums 0) 0) nums)] - '[1 2] - - Here, the last example evaluates to ``('+' 1 2)``, since the condition - ``(< (nth nums 0) 0)`` is ``False``, which makes this ``if`` expression - evaluate to ``None``, because the ``if`` expression here does not have an - else clause. ``unquote-splice`` then evaluates this as an empty value, - leaving no effects on the list it is enclosed in, therefore resulting in - ``('+' 1 2)``. - - A symbol name can begin with ``@`` in Hy, but ``~@`` takes precedence in the - parser. So, if you want to unquote the symbol ``@foo`` with ``~``, you must - use whitespace to separate ``~`` and ``@``, as in ``~ @foo``. - .. hy:macro:: (while [condition #* body]) ``while`` compiles to a :py:keyword:`while` statement, which executes some From 9f0ebf3bb406ef0d8b3e06b07ddd19b7896c059e Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Sun, 13 Aug 2023 19:49:12 +0200 Subject: [PATCH 265/342] functions don't always have __module__ set --- hy/macros.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hy/macros.py b/hy/macros.py index bdc373322..96c1da640 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -32,7 +32,7 @@ def macro(name): def reader_macro(name, fn): fn = rename_function(fn, name) - inspect.getmodule(fn).__dict__.setdefault("_hy_reader_macros", {})[name] = fn + fn.__globals__.setdefault("_hy_reader_macros", {})[name] = fn def pattern_macro(names, pattern, shadow=None): @@ -84,13 +84,12 @@ def wrapper(hy_compiler, *args): def install_macro(name, fn, module_of): name = mangle(name) fn = rename_function(fn, name) - calling_module = inspect.getmodule(module_of) - macros_obj = calling_module.__dict__.setdefault("_hy_macros", {}) + macros_obj = module_of.__globals__.setdefault("_hy_macros", {}) if name in getattr(builtins, "_hy_macros", {}): warnings.warn( ( f"{name} already refers to: `{name}` in module: `builtins`," - f" being replaced by: `{calling_module.__name__}.{name}`" + f" being replaced by: `{module_of.__globals__.get('__name__', '(globals)')}.{name}`" ), RuntimeWarning, stacklevel=3, From fb46808b84a9d0bf403a3e3d65810056a1e36195 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Sun, 13 Aug 2023 19:49:59 +0200 Subject: [PATCH 266/342] fix hy.eval trying to access hy.&reader --- NEWS.rst | 2 + hy/cmdline.py | 22 +++++---- hy/compiler.py | 5 +- hy/core/macros.hy | 2 +- hy/core/result_macros.py | 2 +- hy/importer.py | 6 +-- hy/reader/__init__.py | 6 ++- hy/reader/hy_reader.py | 74 ++++++++++++++++++----------- tests/native_tests/reader_macros.hy | 43 +++++++++++++++++ 9 files changed, 114 insertions(+), 48 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 184314f8d..0fafcfeae 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -28,6 +28,8 @@ Bug Fixes * Fixed incomplete recognition of macro calls with a unary dotted head like `((. defn) f [])`. * `~@ #*` now produces a syntax error instead of a nonsensical result. +* Fixed `hy.eval` failing on `defreader` or `require` forms that + install a new reader. 0.27.0 (released 2023-07-06) ============================= diff --git a/hy/cmdline.py b/hy/cmdline.py index be8d5a318..cf79b1c9e 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -372,16 +372,18 @@ def hy2py_worker(source, options, filename=None, parent_module=None, output_file ) as output_file: def printing_source(hst): - for node in hst: - if options.with_source: - print(node, file=output_file) - yield node - - hst = hy.models.Lazy( - printing_source(read_many(source, filename, skip_shebang=True)) - ) - hst.source = source - hst.filename = filename + def _printing_gen(hst): + for node in hst: + if options.with_source: + print(node, file=output_file) + yield node + printing_hst = hy.models.Lazy(_printing_gen(hst)) + printing_hst.source = hst.source + printing_hst.filename = hst.filename + printing_hst.reader = hst.reader + return printing_hst + + hst = printing_source(read_many(source, filename, skip_shebang=True)) with filtered_hy_exceptions(): module_name = source_path.stem if source_path else Path(filename).name diff --git a/hy/compiler.py b/hy/compiler.py index 55070b87b..7bd78c98f 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -31,7 +31,7 @@ as_model, is_unpack, ) -from hy.reader import mangle +from hy.reader import mangle, HyReader from hy.scoping import ScopeGlobal hy_ast_compile_flags = 0 @@ -795,6 +795,7 @@ def hy_compile( filename = getattr(tree, "filename", filename) source = getattr(tree, "source", source) + reader = getattr(tree, "reader", None) tree = as_model(tree) if not isinstance(tree, Object): @@ -804,7 +805,7 @@ def hy_compile( compiler = compiler or HyASTCompiler(module, filename=filename, source=source) - with compiler.scope: + with HyReader.using_reader(reader, create=False), compiler.scope: result = compiler.compile(tree) expr = result.force_expr diff --git a/hy/core/macros.hy b/hy/core/macros.hy index b3a81553c..9746aaaab 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -110,7 +110,7 @@ ~@(if docstr [docstr] []) ~@body))) (eval-when-compile - (setv (get hy.&reader.reader-macros ~dispatch-key) + (setv (get (. (hy.reader.HyReader.current-reader) reader-macros) ~dispatch-key) (get _hy_reader_macros ~dispatch-key))))) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index b17b8730a..6601cd160 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1845,7 +1845,7 @@ def compile_require(compiler, expr, root, entries): mkexpr( dotted("hy.macros.enable-readers"), "None", - dotted("hy.&reader"), + mkexpr(dotted("hy.reader.HyReader.current-reader")), [reader_assignments], ), ), diff --git a/hy/importer.py b/hy/importer.py index 7d9f39e4d..24f8dfcb5 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -11,7 +11,7 @@ import hy from hy.compiler import hy_compile -from hy.reader import read_many +from hy.reader import read_many, HyReader @contextmanager @@ -118,7 +118,7 @@ def _hy_source_to_code(self, data, path, _optimize=-1): if os.environ.get("HY_MESSAGE_WHEN_COMPILING"): print("Compiling", path, file=sys.stderr) source = data.decode("utf-8") - hy_tree = read_many(source, filename=path, skip_shebang=True) + hy_tree = read_many(source, filename=path, skip_shebang=True, reader=HyReader()) with loader_module_obj(self) as module: data = hy_compile(hy_tree, module) @@ -139,7 +139,7 @@ def _hy_compile_source(pathname, source): sys.modules[mname] = types.ModuleType(mname) return compile( hy_compile( - read_many(source.decode("UTF-8"), filename=pathname, skip_shebang=True), + read_many(source.decode("UTF-8"), filename=pathname, skip_shebang=True, reader=HyReader()), sys.modules[mname], ), pathname, diff --git a/hy/reader/__init__.py b/hy/reader/__init__.py index 8744c715f..09147c8c4 100644 --- a/hy/reader/__init__.py +++ b/hy/reader/__init__.py @@ -33,10 +33,12 @@ def read_many(stream, filename="", reader=None, skip_shebang=False): source = stream.read() stream.seek(pos) - m = hy.models.Lazy((reader or HyReader()).parse( + reader = reader or HyReader() + m = hy.models.Lazy(reader.parse( stream, filename, skip_shebang)) m.source = source m.filename = filename + m.reader = reader return m @@ -52,5 +54,5 @@ def read(stream, filename=None, reader=None): except StopIteration: raise EOFError() else: - m.source, m.filename = it.source, it.filename + m.source, m.filename, m.reader = it.source, it.filename, it.reader return m diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index b2593df86..110e69211 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -1,6 +1,7 @@ "Character reader for parsing Hy source." import codecs +from contextlib import contextmanager, nullcontext from itertools import islice import hy @@ -116,6 +117,7 @@ class HyReader(Reader): ### NON_IDENT = set("()[]{};\"'`~") + _current_reader = None def __init__(self): super().__init__() @@ -128,6 +130,27 @@ def __init__(self): self.reader_macros[tag[1:]] = self.reader_table.pop(tag) + @classmethod + def current_reader(cls, override=None, create=True): + return override or HyReader._current_reader or (cls() if create else None) + + @contextmanager + def as_current_reader(self): + old_reader = HyReader._current_reader + HyReader._current_reader = self + try: + yield + finally: + HyReader._current_reader = old_reader + + @classmethod + @contextmanager + def using_reader(cls, override=None, create=True): + reader = cls.current_reader(override, create) + with reader.as_current_reader() if reader else nullcontext(): + yield + + def fill_pos(self, model, start): """Attach line/col information to a model. @@ -159,8 +182,6 @@ def read_default(self, key): def parse(self, stream, filename=None, skip_shebang=False): """Yields all `hy.models.Object`'s in `source` - Additionally exposes `self` as ``hy.&reader`` during read/compile time. - Args: source: Hy source to be parsed. @@ -178,17 +199,7 @@ def parse(self, stream, filename=None, skip_shebang=False): if c == "\n": break - rname = mangle("&reader") - old_reader = getattr(hy, rname, None) - setattr(hy, rname, self) - - try: - yield from self.parse_forms_until("") - finally: - if old_reader is None: - delattr(hy, rname) - else: - setattr(hy, rname, old_reader) + yield from self.parse_forms_until("") ### # Reading forms @@ -210,23 +221,28 @@ def try_parse_one_form(self): fully parsing a form. LexException: If there is an error during form parsing. """ - try: - self.slurp_space() - c = self.getc() - start = self._pos - if not c: - raise PrematureEndOfInput.from_reader( - "Premature end of input while attempting to parse one form", self + with self.as_current_reader(): + try: + self.slurp_space() + c = self.getc() + start = self._pos + if not c: + raise PrematureEndOfInput.from_reader( + "Premature end of input while attempting to parse one form", self + ) + handler = self.reader_table.get(c) + model = handler(self, c) if handler else self.read_default(c) + if model is not None: + model = self.fill_pos(model, start) + model.reader = self + return model + return None + except LexException: + raise + except Exception as e: + raise LexException.from_reader( + str(e) or "Exception thrown attempting to parse one form", self ) - handler = self.reader_table.get(c) - model = handler(self, c) if handler else self.read_default(c) - return self.fill_pos(model, start) if model is not None else None - except LexException: - raise - except Exception as e: - raise LexException.from_reader( - str(e) or "Exception thrown attempting to parse one form", self - ) def parse_one_form(self): """Read from the stream until a form is parsed. diff --git a/tests/native_tests/reader_macros.hy b/tests/native_tests/reader_macros.hy index 9d62e7b22..f6eea5735 100644 --- a/tests/native_tests/reader_macros.hy +++ b/tests/native_tests/reader_macros.hy @@ -4,6 +4,7 @@ types contextlib [contextmanager] hy.errors [HyMacroExpansionError] + hy.reader [HyReader] hy.reader.exceptions [PrematureEndOfInput] pytest) @@ -89,3 +90,45 @@ (eval-module #[[(require tests.resources.tlib :readers [taggart] :readers [upper!])]])) (with [(pytest.raises hy.errors.HyRequireError)] (eval-module #[[(require tests.resources.tlib :readers [not-a-real-reader])]]))) + +(defn test-eval-read [] + ;; https://github.com/hylang/hy/issues/2291 + ;; hy.eval should not raise an exception when + ;; defining readers using hy.read or with quoted forms + (with [module (temp-module "")] + (hy.eval (hy.read "(defreader r 5)") :module module) + (hy.eval '(defreader test-read 4) :module module) + (hy.eval '(require tests.resources.tlib :readers [upper!]) :module module) + ;; these reader macros should not exist in any current reader + (for [tag #("#r" "#test-read" "#upper!")] + (with [(pytest.raises hy.errors.HySyntaxError)] + (hy.read tag))) + ;; but they should be installed in the module + (setv reader (HyReader)) + (hy.macros.enable-readers module reader "ALL") + (for [[s val] [["#r" 5] + ["#test-read" 4] + ["#upper! \"hi there\"" "HI THERE"]]] + (assert (= (hy.eval (hy.read s :reader reader) :module module) val)))) + + ;; passing a reader explicitly should work as expected + (with [module (temp-module "")] + (setv reader (HyReader)) + (defn eval1 [s] + (hy.eval (hy.read s :reader reader) :module module)) + (eval1 "(defreader fbaz 32)") + (eval1 "(require tests.resources.tlib :readers [upper!])") + (assert (= (eval1 "#fbaz") 32)) + (assert (= (eval1 "#upper! \"hello\"") "HELLO")))) + + +(defn test-interleaving-readers [] + (with [module1 (temp-module "") + module2 (temp-module "")] + (setv stream1 (hy.read-many #[[(do (defreader foo "foo1") (defreader bar "bar1")) #foo #bar]]) + stream2 (hy.read-many #[[(do (defreader foo "foo2") (defreader bar "bar2")) #foo #bar]]) + valss [[None None] ["foo1" "foo2"] ["bar1" "bar2"]]) + (for [[form1 form2 vals] (zip stream1 stream2 valss)] + (assert (= vals + [(hy.eval form1 :module module1) + (hy.eval form2 :module module2)]))))) From f5aff259ed6d7f3cf37ab8181064ae08c54bf877 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Tue, 15 Aug 2023 00:28:30 +0200 Subject: [PATCH 267/342] Allow initializing HyReader with current module's readers --- NEWS.rst | 2 ++ hy/reader/hy_reader.py | 12 ++++++++++-- tests/native_tests/reader_macros.hy | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 0fafcfeae..c497bc5c9 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -16,6 +16,8 @@ Breaking Changes New Features ------------------------------ * `defn`, `defn/a`, and `defclass` now support type parameters. +* `HyReader` now has an optional parameter to install existing + reader macros from the calling module. Misc. Improvements ------------------------------ diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py index 110e69211..6e89e269b 100644 --- a/hy/reader/hy_reader.py +++ b/hy/reader/hy_reader.py @@ -1,6 +1,7 @@ "Character reader for parsing Hy source." import codecs +import inspect from contextlib import contextmanager, nullcontext from itertools import islice @@ -110,7 +111,10 @@ def err(msg): class HyReader(Reader): - """A modular reader for Hy source.""" + """A modular reader for Hy source. + + When ``use_current_readers`` is true, initialize this reader + with all reader macros from the calling module.""" ### # Components necessary for Reader implementation @@ -119,7 +123,7 @@ class HyReader(Reader): NON_IDENT = set("()[]{};\"'`~") _current_reader = None - def __init__(self): + def __init__(self, *, use_current_readers=False): super().__init__() # move any reader macros declared using @@ -129,6 +133,10 @@ def __init__(self): if tag[0] == '#' and tag[1:]: self.reader_macros[tag[1:]] = self.reader_table.pop(tag) + if use_current_readers: + self.reader_macros.update( + inspect.stack()[1].frame.f_globals.get("_hy_reader_macros", {}) + ) @classmethod def current_reader(cls, override=None, create=True): diff --git a/tests/native_tests/reader_macros.hy b/tests/native_tests/reader_macros.hy index f6eea5735..5f613d12d 100644 --- a/tests/native_tests/reader_macros.hy +++ b/tests/native_tests/reader_macros.hy @@ -104,8 +104,8 @@ (with [(pytest.raises hy.errors.HySyntaxError)] (hy.read tag))) ;; but they should be installed in the module - (setv reader (HyReader)) - (hy.macros.enable-readers module reader "ALL") + (hy.eval '(setv reader (hy.reader.HyReader :use-current-readers True)) :module module) + (setv reader module.reader) (for [[s val] [["#r" 5] ["#test-read" 4] ["#upper! \"hi there\"" "HI THERE"]]] From 2d96067234ab505198d78a70c2cca4a61b06932d Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Tue, 15 Aug 2023 02:12:15 +0200 Subject: [PATCH 268/342] Bypass validity checks when quoting Symbols and Keywords --- hy/core/result_macros.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 6601cd160..2bf35d089 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -235,10 +235,10 @@ def render_quoted_form(compiler, form, level): body.extend([Keyword("conversion"), String(form.conversion)]) elif isinstance(form, Symbol): - body = [String(form)] + body = [String(form), Keyword("from_parser"), Symbol("True")] elif isinstance(form, Keyword): - body = [String(form.name)] + body = [String(form.name), Keyword("from_parser"), Symbol("True")] elif isinstance(form, String): if form.brackets is not None: From c74dcc4462d41b116b493d5f4946795999af58db Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 18 Aug 2023 16:10:50 -0400 Subject: [PATCH 269/342] Remove partial macro namespacing --- NEWS.rst | 3 ++ hy/macros.py | 8 +--- hy/models.py | 2 +- tests/native_tests/import.hy | 89 ++++++++---------------------------- 4 files changed, 25 insertions(+), 77 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index c497bc5c9..2eae76998 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,9 @@ Unreleased Breaking Changes ------------------------------ +* When a macro is `require`\d from another module, that module is no + longer implicitly included when checking for further macros in + the expansion. * `hy.eval` has been overhauled to be more like Python's `eval`. * `hy` now only implicitly launches a REPL if standard input is a TTY. * `hy -i` has been overhauled to work as a flag like `python3 -i`. diff --git a/hy/macros.py b/hy/macros.py index 96c1da640..e2d96d39b 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -391,14 +391,11 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): else: break - expr_modules = ([] if not hasattr(tree, "module") else [tree.module]) + [module] - expr_modules.append(builtins) - # Choose the first namespace with the macro. m = next( ( mod._hy_macros[fn] - for mod in expr_modules + for mod in (module, builtins) if fn in getattr(mod, "_hy_macros", ()) ), None, @@ -413,9 +410,6 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): if isinstance(obj, (hy.compiler.Result, AST)): return obj if result_ok else tree - if isinstance(obj, Expression): - obj.module = inspect.getmodule(m) - tree = replace_hy_obj(obj, tree) if once: diff --git a/hy/models.py b/hy/models.py index 4ddf06812..3a1ce2c69 100644 --- a/hy/models.py +++ b/hy/models.py @@ -34,7 +34,7 @@ class Object: `end_column` 3. """ - properties = ["module", "_start_line", "_end_line", "_start_column", "_end_column"] + properties = ["_start_line", "_end_line", "_start_column", "_end_column"] def replace(self, other, recursive=False): if isinstance(other, Object): diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy index 817c47f63..4cb006309 100644 --- a/tests/native_tests/import.hy +++ b/tests/native_tests/import.hy @@ -214,75 +214,26 @@ in expansions." (require-macros)) -(defn [(pytest.mark.xfail)] test-macro-from-module [] - " - Macros loaded from an external module, which itself `require`s macros, should - work without having to `require` the module's macro dependencies (due to - [minimal] macro namespace resolution). - - In doing so we also confirm that a module's `_hy_macros` attribute is correctly - loaded and used. - - Additionally, we confirm that `require` statements are executed via loaded bytecode. - " - - (setv pyc-file (importlib.util.cache-from-source - (os.path.realpath - (os.path.join - "tests" "resources" "macro_with_require.hy")))) - - ;; Remove any cached byte-code, so that this runs from source and - ;; gets evaluated in this module. - (when (os.path.isfile pyc-file) - (os.unlink pyc-file) - (.clear sys.path_importer_cache) - (when (in "tests.resources.macro_with_require" sys.modules) - (del (get sys.modules "tests.resources.macro_with_require")) - (_hy_macros.clear))) - - ;; Ensure that bytecode isn't present when we require this module. - (assert (not (os.path.isfile pyc-file))) - - (defn test-requires-and-macros [] - (require tests.resources.macro-with-require - [test-module-macro]) - - ;; Make sure that `require` didn't add any of its `require`s - (assert (not (in (hy.mangle "nonlocal-test-macro") _hy_macros))) - ;; and that it didn't add its tags. - (assert (not (in (hy.mangle "#test-module-tag") _hy_macros))) - - ;; Now, require everything. - (require tests.resources.macro-with-require *) - - ;; Again, make sure it didn't add its required macros and/or tags. - (assert (not (in (hy.mangle "nonlocal-test-macro") _hy_macros))) - - ;; Its tag(s) should be here now. - (assert (in (hy.mangle "#test-module-tag") _hy_macros)) - - ;; The test macro expands to include this symbol. - (setv module-name-var "tests.native_tests.native_macros") - (assert (= (+ "This macro was created in tests.resources.macros, " - "expanded in tests.native_tests.native_macros " - "and passed the value 1.") - (test-module-macro 1)))) - - (test-requires-and-macros) - - ;; Now that bytecode is present, reload the module, clear the `require`d - ;; macros and tags, and rerun the tests. - (assert (os.path.isfile pyc-file)) - - ;; Reload the module and clear the local macro context. - (.clear sys.path_importer_cache) - (del (get sys.modules "tests.resources.macro_with_require")) - (.clear _hy_macros) - - ;; There doesn't seem to be a way--via standard import mechanisms--to - ;; ensure that an imported module used the cached bytecode. We'll simply have - ;; to trust that the .pyc loading convention was followed. - (test-requires-and-macros)) +(defn test-no-surprise-shadow [tmp-path monkeypatch] + "Check that an out-of-module macro doesn't shadow a function." + ; https://github.com/hylang/hy/issues/2451 + + (monkeypatch.syspath-prepend tmp-path) + (.write-text (/ tmp-path "wexter_a.hy") #[[ + (defmacro helper [] + "helper a (macro)") + (defmacro am [form] + form)]]) + (.write-text (/ tmp-path "wexter_b.hy") #[[ + (require wexter-a [am]) + (defn helper [] + "helper b (function)") + (setv v1 (helper)) + (setv v2 (am (helper)))]]) + + (import wexter-b) + (assert (= wexter-b.v1 "helper b (function)")) + (assert (= wexter-b.v2 "helper b (function)"))) (defn test-recursive-require-star [] From 89ae8bc9ca7b1ced6b35608f4448770c40046876 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 18 Aug 2023 16:50:01 -0400 Subject: [PATCH 270/342] Don't use `self` for a non-method --- hy/core/result_macros.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 2bf35d089..dd11d2d88 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -111,8 +111,8 @@ def digest_type_params(compiler, tp): @pattern_macro("do", [many(FORM)]) -def compile_do(self, expr, root, body): - return self._compile_branch(body) +def compile_do(compiler, expr, root, body): + return compiler._compile_branch(body) @pattern_macro(["eval-and-compile", "eval-when-compile", "do-mac"], [many(FORM)]) From 8e46477fa411fdd661ee0c8428568fc74b6576fc Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 18 Aug 2023 16:27:09 -0400 Subject: [PATCH 271/342] Clean up some imports --- hy/core/result_macros.py | 2 +- hy/core/util.hy | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index dd11d2d88..6b754d588 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -57,7 +57,7 @@ as_model, is_unpack, ) -from hy.reader import mangle, unmangle +from hy.reader import mangle from hy.scoping import ScopeFn, ScopeGen, ScopeLet, is_inside_function_scope # ------------------------------------------------ diff --git a/hy/core/util.hy b/hy/core/util.hy index a8002b438..1b4e71f3c 100644 --- a/hy/core/util.hy +++ b/hy/core/util.hy @@ -1,8 +1,6 @@ (import itertools) (import collections.abc [Iterable]) -(import hy.models [Keyword Symbol] - hy.reader [mangle unmangle] - hy.compiler [HyASTCompiler calling-module]) +(import hy.compiler [HyASTCompiler calling-module]) (defn disassemble [tree [codegen False]] "Return the python AST for a quoted Hy `tree` as a string. @@ -72,7 +70,7 @@ (setv n _gensym_counter) (finally (.release _gensym_lock))) (setv g (hy.mangle (.format "_hy_gensym_{}_{}" g n))) - (Symbol (if (.startswith g "_hyx_") + (hy.models.Symbol (if (.startswith g "_hyx_") ; Remove the mangle prefix, if it's there, so the result always ; starts with our reserved prefix `_hy_`. (+ "_" (cut g (len "_hyx_") None)) From 64459613e0d81b09beabe3f7c3c7ebe8a63962dc Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 18 Aug 2023 16:25:45 -0400 Subject: [PATCH 272/342] Remove some dead code --- hy/macros.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/hy/macros.py b/hy/macros.py index e2d96d39b..180afe223 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -20,7 +20,7 @@ ) from hy.model_patterns import whole from hy.models import Expression, Symbol, as_model, is_unpack, replace_hy_obj -from hy.reader import mangle, unmangle +from hy.reader import mangle EXTRA_MACROS = ["hy.core.result_macros", "hy.core.macros"] @@ -273,11 +273,7 @@ def require(source_module, target_module, assignments, prefix="", target_module_ ) ): _name = mangle(name) - alias = mangle( - "#" + prefix + unmangle(alias)[1:] - if unmangle(alias).startswith("#") - else prefix + alias - ) + alias = mangle(prefix + alias) if _name in source_module._hy_macros: target_macros[alias] = source_macros[_name] else: From 7badc3aa970c0501791fb72ee4dac93a8e21bf80 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 18 Aug 2023 17:08:45 -0400 Subject: [PATCH 273/342] Remove more dead code found with Vulture --- hy/cmdline.py | 6 +++--- hy/compiler.py | 8 ++------ hy/errors.py | 10 ---------- hy/importer.py | 13 ------------- hy/models.py | 2 +- hy/repl.py | 12 ++++-------- hy/scoping.py | 1 - 7 files changed, 10 insertions(+), 42 deletions(-) diff --git a/hy/cmdline.py b/hy/cmdline.py index cf79b1c9e..1fa129582 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -65,7 +65,7 @@ class HyArgError(Exception): pass -def cmdline_handler(scriptname, argv): +def cmdline_handler(argv): # We need to terminate interpretation of options after certain # options, such as `-c`. So, we can't use `argparse`. @@ -321,7 +321,7 @@ def proc_opt(opt, arg=None, item=None, i=None): def hy_main(): sys.path.insert(0, "") try: - sys.exit(cmdline_handler("hy", sys.argv)) + sys.exit(cmdline_handler(sys.argv)) except HyArgError as e: print(e) exit(1) @@ -473,7 +473,7 @@ def hy2py_main(): f"{filename} is a directory but the output directory is not specified. Use --output or -o in command line arguments to specify the output directory." ) os.makedirs(options.output, exist_ok=True) - for path, subdirs, files in os.walk(filename): + for path, _, files in os.walk(filename): for name in files: filename_raw, filename_ext = os.path.splitext(name) if filename_ext == ".hy": diff --git a/hy/compiler.py b/hy/compiler.py index 7bd78c98f..1ef850928 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -316,10 +316,6 @@ def mkexpr(*items, **kwargs): return make_hy_model(Expression, items, kwargs.get("rest")) -def mklist(*items, **kwargs): - return make_hy_model(List, items, kwargs.get("rest")) - - def is_annotate_expression(model): return isinstance(model, Expression) and model and model[0] == Symbol("annotate") @@ -512,7 +508,7 @@ def _nonconst(self, name): return name @builds_model(Expression) - def compile_expression(self, expr, *, allow_annotation_expression=False): + def compile_expression(self, expr): # Perform macro expansions expr = macroexpand(expr, self.module, self) if isinstance(expr, (Result, ast.AST)): @@ -566,7 +562,7 @@ def compile_expression(self, expr, *, allow_annotation_expression=False): if is_annotate_expression(root): # Flatten and compile the annotation expression. ann_expr = Expression(root + args).replace(root) - return self.compile_expression(ann_expr, allow_annotation_expression=True) + return self.compile_expression(ann_expr) if not func: func = self.compile(root) diff --git a/hy/errors.py b/hy/errors.py index 440f0dc30..c594e1dcd 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -146,10 +146,6 @@ class HyTypeError(HyLanguageError, TypeError): """TypeError occurring during the normal use of Hy.""" -class HyNameError(HyLanguageError, NameError): - """NameError occurring during the normal use of Hy.""" - - class HyRequireError(HyLanguageError): """Errors arising during the use of `require` @@ -175,12 +171,6 @@ class HyEvalError(HyLanguageError): """ -class HyIOError(HyInternalError, IOError): - """Subclass used to distinguish between IOErrors raised by Hy itself as - opposed to Hy programs. - """ - - class HySyntaxError(HyLanguageError, SyntaxError): """Error during the Lexing of a Hython expression.""" diff --git a/hy/importer.py b/hy/importer.py index 24f8dfcb5..554281e14 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -158,11 +158,6 @@ def _hy_compile_source(pathname, source): # Do this one just in case? importlib.invalidate_caches() -# These aren't truly cross-compliant. -# They're useful for testing, though. -class HyImporter(importlib.machinery.FileFinder): - pass - class HyLoader(importlib.machinery.SourceFileLoader): pass @@ -182,14 +177,6 @@ class HyLoader(importlib.machinery.SourceFileLoader): runpy._get_code_from_file = _get_code_from_file -def _import_from_path(name, path): - """A helper function that imports a module from the given path.""" - spec = importlib.util.spec_from_file_location(name, path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - def _inject_builtins(): """Inject the Hy core macros into Python's builtins if necessary""" if hasattr(builtins, "__hy_injected__"): diff --git a/hy/models.py b/hy/models.py index 3a1ce2c69..63f8a529c 100644 --- a/hy/models.py +++ b/hy/models.py @@ -222,7 +222,7 @@ def __new__(cls, s, from_parser=False): _wrappers[bool] = lambda x: Symbol("True") if x else Symbol("False") -_wrappers[type(None)] = lambda foo: Symbol("None") +_wrappers[type(None)] = lambda _: Symbol("None") class Keyword(Object): diff --git a/hy/repl.py b/hy/repl.py index 5382a70f6..bde468727 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -14,7 +14,7 @@ from contextlib import contextmanager import hy -from hy.compiler import HyASTCompiler, hy_ast_compile_flags, hy_compile +from hy.compiler import HyASTCompiler, hy_compile from hy.completer import Completer, completion from hy.errors import ( HyLanguageError, @@ -126,8 +126,6 @@ def __init__( self.module, self.reader, self.module._hy_reader_macros.keys() ) - self.flags |= hy_ast_compile_flags - self.cmdline_cache = cmdline_cache def _cache(self, source, name): @@ -338,7 +336,7 @@ def ast_callback(self, exec_ast, eval_ast): msg = "Exception in AST callback:\n{}\n".format(traceback.format_exc()) self.write(msg) - def _error_wrap(self, error_fn, exc_info_override=False, *args, **kwargs): + def _error_wrap(self, exc_info_override=False, *args, **kwargs): sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() if exc_info_override: @@ -358,12 +356,10 @@ def showsyntaxerror(self, filename=None): filename = self.filename self.print_last_value = False - self._error_wrap( - super().showsyntaxerror, exc_info_override=True, filename=filename - ) + self._error_wrap(exc_info_override=True, filename=filename) def showtraceback(self): - self._error_wrap(super().showtraceback) + self._error_wrap() def runcode(self, code): try: diff --git a/hy/scoping.py b/hy/scoping.py index d7bb03ea7..488087043 100644 --- a/hy/scoping.py +++ b/hy/scoping.py @@ -323,7 +323,6 @@ def __init__(self, compiler): super().__init__(compiler) self.iterators = set() self.assignments = [] - self.nonlocals = set() self.exposing_assignments = False def __enter__(self): From c9977905c844d5d64d527ed9fb11c542dc45e8f8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 20 Aug 2023 10:21:53 -0400 Subject: [PATCH 274/342] Remove some unnecessary mangling --- hy/core/result_macros.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 6b754d588..24053129f 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -143,9 +143,9 @@ def compile_eval_foo_compile(compiler, expr, root, body): return ( compiler.compile(as_model(value)) - if mangle(root) == "do_mac" + if root == "do-mac" else compiler._compile_branch(body) - if mangle(root) == "eval_and_compile" + if root == "eval-and-compile" else Result() ) From c0a1b1054ccf00958607695076c5d27e530843e0 Mon Sep 17 00:00:00 2001 From: Tuula Turto Date: Wed, 23 Aug 2023 07:49:11 +0300 Subject: [PATCH 275/342] Update AUTHORS Change Tuukka Turto to Tuula TUrto --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index d28bcd1b9..9702eadb7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -33,7 +33,7 @@ * Matt Fenwick * Sean B. Palmer * Thom Neale -* Tuukka Turto +* Tuula Turto * Vasudev Kamath * Yuval Langer * Fatih Kadir Akın From dffad067f1425dc7cbe1b021f0d70d20a7bc4d98 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 23 Aug 2023 15:56:57 -0400 Subject: [PATCH 276/342] Improve the documentation of `eval-foo-compile` --- docs/api.rst | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index e98dac919..64a05c759 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -453,7 +453,24 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:macro:: (eval-and-compile [#* body]) - ``eval-and-compile`` takes any number of forms as arguments. The input forms are evaluated as soon as the ``eval-and-compile`` form is compiled, instead of being deferred until run-time. The input forms are also left in the program so they can be executed at run-time as usual. So, if you compile and immediately execute a program (as calling ``hy foo.hy`` does when ``foo.hy`` doesn't have an up-to-date byte-compiled version), ``eval-and-compile`` forms will be evaluated twice. The return value is the final argument, as in ``do``. + ``eval-and-compile`` takes any number of forms as arguments. The input forms are evaluated as soon as the ``eval-and-compile`` form is compiled, then left in the program so they can be executed at run-time as usual; contrast with :hy:func:`eval-when-compile`. So, if you compile and immediately execute a program (as calling ``hy foo.hy`` does when ``foo.hy`` doesn't have an up-to-date byte-compiled version), ``eval-and-compile`` forms will be evaluated twice. For example, the following program :: + + (eval-when-compile + (print "Compiling")) + (print "Running") + (eval-and-compile + (print "Hi")) + + prints + + .. code-block:: text + + Compiling + Hi + Running + Hi + + The return value of ``eval-and-compile`` is its final argument, as for :hy:func:`do`. One possible use of ``eval-and-compile`` is to make a function available both at compile-time (so a macro can call it while expanding) and run-time (so it can be called like any other function):: @@ -469,9 +486,35 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Had the ``defn`` not been wrapped in ``eval-and-compile``, ``m`` wouldn't be able to call ``add``, because when the compiler was expanding ``(m 3)``, ``add`` wouldn't exist yet. + While ``eval-and-compile`` executes the same code at both compile-time and run-time, bear in mind that the same code can have different meanings in the two contexts. Consider, for example, issues of scoping:: + + (eval-when-compile + (print "Compiling")) + (print "Running") + (eval-and-compile + (setv x 1)) + (defn f [] + (setv x 2) + (eval-and-compile + (setv x 3)) + (print "local x =" x)) + (f) + (eval-and-compile + (print "global x =" x)) + + The form ``(setv x 3)`` above refers to the global ``x`` at compile-time, but the local ``x`` at run-time, so the result is: + + .. code-block:: text + + Compiling + global x = 3 + Running + local x = 3 + global x = 1 + .. hy:macro:: (eval-when-compile [#* body]) - As ``eval-and-compile``, but the code isn't executed at run-time, and ``None`` is returned. Hence, ``eval-when-compile`` doesn't directly contribute any code to the final program, although it can still change Hy's state while compiling (e.g., by defining a function). :: + ``eval-when-compile`` executes the given forms at compile-time, but discards them at run-time and simply returns :data:`None` instead; contrast :hy:func:`eval-and-compile`. Hence, while ``eval-when-compile`` doesn't directly contribute code to the final program, it can change Hy's state while compiling, as by defining a function:: (eval-when-compile (defn add [x y] From c3b504811d978cd20b1e5ff5cec23b45e30da5e8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 24 Aug 2023 17:16:35 -0400 Subject: [PATCH 277/342] Correct a typo in the manual --- docs/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index 64a05c759..00b2e249c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -839,7 +839,7 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:macro:: (unquote [model]) .. hy:macro:: (unquote-splice [model]) - ``quasiquote`` is like :hy:func:`quote` except that it treats the model as a template, in which certain special :ref:`expressions ` indicate that some code should evaluated and its value substituted there. The idea is similar to C's ``sprintf`` or Python's various string-formatting constructs. For example:: + ``quasiquote`` is like :hy:func:`quote` except that it treats the model as a template, in which certain special :ref:`expressions ` indicate that some code should be evaluated and its value substituted there. The idea is similar to C's ``sprintf`` or Python's various string-formatting constructs. For example:: (setv x 2) (quasiquote (+ 1 (unquote x))) ; => '(+ 1 2) From 4bb6a9602bd251decb19b17c48d0d7dcf25f653c Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 24 Aug 2023 17:17:09 -0400 Subject: [PATCH 278/342] Improve the documentation of `hy.unmangle` --- hy/reader/mangling.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/hy/reader/mangling.py b/hy/reader/mangling.py index 0ce46d94c..74b2c932b 100644 --- a/hy/reader/mangling.py +++ b/hy/reader/mangling.py @@ -71,32 +71,19 @@ def mangle(s): def unmangle(s): """Stringify the argument and try to convert it to a pretty unmangled - form. See :ref:`Hy's mangling rules `. + form. See :ref:`Hy's mangling rules `. :: + + (hy.unmangle "hyx_XsquidX") ; => "🦑" Unmangling may not round-trip, because different Hy symbol names can mangle to the same Python identifier. In particular, Python itself already considers distinct strings that have the same normalized form (according to NFKC), such as ``hello`` and ``𝔥𝔢𝔩𝔩𝔬``, to be the same identifier. - Examples: - :: - - => (hy.unmangle 'foo_bar) - "foo-bar" - - => (hy.unmangle 'hyx_XasteriskX) - "*" - - => (hy.unmangle 'hyx_XhyphenHminusX_XgreaterHthan_signX) - "-->" - - => (hy.unmangle 'hyx_XlessHthan_signX__) - "<--" - - => (hy.unmangle '__dunder_name__) - "__dunder-name__" - - """ + It's an error to call ``hy.unmangle`` on something that looks like a + properly mangled name but isn't. For example, ``(hy.unmangle + "hyx_XpizzazzX")`` is erroneous, because there is no Unicode character + named "PIZZAZZ" (yet).""" s = str(s) From 1de91eb5d3d4d67119beb8ada1bddd7d926d853f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 24 Aug 2023 17:17:29 -0400 Subject: [PATCH 279/342] Improve the documentation of `hy.repr(-register)` --- hy/core/hy_repr.hy | 67 +++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/hy/core/hy_repr.hy b/hy/core/hy_repr.hy index 663b80ca1..2df8611a9 100644 --- a/hy/core/hy_repr.hy +++ b/hy/core/hy_repr.hy @@ -8,39 +8,26 @@ (setv _registry {}) (defn hy-repr-register [types f [placeholder None]] - "``hy.repr-register`` lets you set the function that ``hy.repr`` calls to - represent a type. - - Examples: - :: - - => (hy.repr-register the-type fun) - - => (defclass C) - => (hy.repr-register C (fn [x] \"cuddles\")) - => (hy.repr [1 (C) 2]) - \"[1 cuddles 2]\" - - If the type of an object passed to ``hy.repr`` doesn't have a registered - function, ``hy.repr`` falls back on ``repr``. - - Registered functions often call ``hy.repr`` themselves. ``hy.repr`` will - automatically detect self-references, even deeply nested ones, and - output ``\"...\"`` for them instead of calling the usual registered - function. To use a placeholder other than ``\"...\"``, pass a string of - your choice to the keyword argument ``:placeholder`` of - ``hy.repr-register``. - - => (defclass Container [object] - ... (defn __init__ (fn [self value] - ... (setv self.value value)))) - => (hy.repr-register Container :placeholder \"HY THERE\" (fn [x] - ... (+ \"(Container \" (hy.repr x.value) \")\"))) - => (setv container (Container 5)) - => (setv container.value container) - => (print (hy.repr container)) - '(Container HY THERE)' - " + #[[``hy.repr-register`` lets you set the function that :hy:func:`hy.repr` calls to + represent a type:: + + (defclass C) + (hy.repr-register C (fn [x] "cuddles")) + (hy.repr [1 (C) 2]) ; => "[1 cuddles 2]" + + Registered functions often call ``hy.repr`` themselves. ``hy.repr`` will + automatically detect self-references, even deeply nested ones, and + output ``"..."`` for them instead of calling the usual registered + function. To use a placeholder other than ``"..."``, pass a string of + your choice as the ``placeholder`` argument:: + + (defclass Container) + (hy.repr-register Container :placeholder "HY THERE" + (fn [x] f"(Container {(hy.repr x.value)})")) + (setv container (Container)) + (setv container.value container) + (hy.repr container) ; => "(Container HY THERE)"]] + (for [typ (if (isinstance types list) types [types])] (setv (get _registry typ) #(f placeholder)))) @@ -50,16 +37,14 @@ #[[This function is Hy's equivalent of Python's :func:`repr`. It returns a string representing the input object in Hy syntax. :: - => (hy.repr [1 2 3]) - "[1 2 3]" - => (repr [1 2 3]) - "[1, 2, 3]" + (hy.repr [1 2 3]) ; => "[1 2 3]" + (repr [1 2 3]) ; => "[1, 2, 3]" Like ``repr`` in Python, ``hy.repr`` can round-trip many kinds of values. Round-tripping implies that given an object ``x``, ``(hy.eval (hy.read (hy.repr x)))`` returns ``x``, or at least a value that's equal to ``x``. A notable exception to round-tripping - is that if a :class:`hy.models.Object` contains a non-model, the + is that if a model contains a non-model, the latter will be promoted to a model in the output:: (setv @@ -68,7 +53,11 @@ y (hy.eval (hy.read output))) (print output) ; '[5] (print (type (get x 0))) ; - (print (type (get y 0))) ; ]] + (print (type (get y 0))) ; + + When ``hy.repr`` doesn't know how to represent an object, it falls + back on :func:`repr`. Use :hy:func:`hy.repr-register` to add your + own conversion function for a type instead.]] (setv [f placeholder] (.get _registry (type obj) [_base-repr None])) From d9b13554d631866151b4ae9449ee476840d961e8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 26 Aug 2023 09:10:27 -0400 Subject: [PATCH 280/342] Change some tests of `defmacro` and `require` to call them at the top level instead of in the test function. I've also changed some uses of the words "local" and "nonlocal" in names to "home" and "remote" to reduce confusion with a new kind of local macros that I'm about to implement. --- tests/native_tests/doc.hy | 7 +++--- tests/native_tests/hy_misc.hy | 7 +++--- tests/native_tests/import.hy | 31 +++++++++++++-------------- tests/native_tests/macros.hy | 4 +++- tests/resources/macro_with_require.hy | 8 +++---- tests/resources/macros.hy | 2 +- 6 files changed, 31 insertions(+), 28 deletions(-) diff --git a/tests/native_tests/doc.hy b/tests/native_tests/doc.hy index 147ef3962..85afc50f0 100644 --- a/tests/native_tests/doc.hy +++ b/tests/native_tests/doc.hy @@ -6,6 +6,10 @@ "Some tag macro" '1) +(defmacro <-mangle-> [] + "a fancy docstring" + '(+ 2 2)) + (defn test-doc [capsys] ;; https://github.com/hylang/hy/issues/1970 ;; Let's first make sure we can doc the builtin macros @@ -18,9 +22,6 @@ (setv [out err] (.readouterr capsys)) (assert (in "Some tag macro" out)) - (defmacro <-mangle-> [] - "a fancy docstring" - '(+ 2 2)) (doc <-mangle->) (setv [out err] (.readouterr capsys)) ;; https://github.com/hylang/hy/issues/1946 diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index 35f62f609..96a818bd4 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -28,11 +28,12 @@ '(a b 5)))) +(defmacro m-with-named-import [] + (import math [pow]) + (pow 2 3)) + (defn test-macroexpand-with-named-import [] ; https://github.com/hylang/hy/issues/1207 - (defmacro m-with-named-import [] - (import math [pow]) - (pow 2 3)) (assert (= (hy.macroexpand '(m-with-named-import)) (hy.models.Float (** 2 3))))) diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy index 4cb006309..29e6f6bbc 100644 --- a/tests/native_tests/import.hy +++ b/tests/native_tests/import.hy @@ -135,24 +135,26 @@ (assert (in "b.xyzzy" _hy_macros))) +;; `remote-test-macro` is a macro used within +;; `tests.resources.macro-with-require.test-module-macro`. +;; Here, we introduce an equivalently named version that, when +;; used, will expand to a different output string. +(defmacro remote-test-macro [x] + "this is the home version of `remote-test-macro`!") + +(require tests.resources.macro-with-require *) +(defmacro home-test-macro [x] + (.format "This is the home version of `remote-test-macro` returning {}!" (int x))) + (defn test-macro-namespace-resolution [] - "Confirm that local versions of macro-macro dependencies do not shadow the + "Confirm that new versions of macro-macro dependencies do not shadow the versions from the macro's own module, but do resolve unbound macro references in expansions." - ;; `nonlocal-test-macro` is a macro used within - ;; `tests.resources.macro-with-require.test-module-macro`. - ;; Here, we introduce an equivalently named version in local scope that, when - ;; used, will expand to a different output string. - (defmacro nonlocal-test-macro [x] - (print "this is the local version of `nonlocal-test-macro`!")) - ;; Was the above macro created properly? - (assert (in "nonlocal_test_macro" _hy_macros)) + (assert (in "remote_test_macro" _hy_macros)) - (setv nonlocal-test-macro (get _hy_macros "nonlocal_test_macro")) - - (require tests.resources.macro-with-require *) + (setv remote-test-macro (get _hy_macros "remote_test_macro")) (setv module-name-var "tests.native_tests.native_macros.test-macro-namespace-resolution") (assert (= (+ "This macro was created in tests.resources.macros, " @@ -162,10 +164,7 @@ in expansions." ;; Now, let's use a `require`d macro that depends on another macro defined only ;; in this scope. - (defmacro local-test-macro [x] - (.format "This is the local version of `nonlocal-test-macro` returning {}!" (int x))) - - (assert (= "This is the local version of `nonlocal-test-macro` returning 3!" + (assert (= "This is the home version of `remote-test-macro` returning 3!" (test-module-macro-2 3)))) diff --git a/tests/native_tests/macros.hy b/tests/native_tests/macros.hy index f35789988..90a7743c8 100644 --- a/tests/native_tests/macros.hy +++ b/tests/native_tests/macros.hy @@ -167,10 +167,12 @@ (wrap-error-test)))) (assert (in "HyWrapperError" (str excinfo.value)))) + +(defmacro delete-me [] "world") + (defn test-delmacro [] ;; test deletion of user defined macro - (defmacro delete-me [] "world") (delmacro delete-me) (with [exc (pytest.raises NameError)] (delete-me)) diff --git a/tests/resources/macro_with_require.hy b/tests/resources/macro_with_require.hy index c1b69f56f..4650ec32e 100644 --- a/tests/resources/macro_with_require.hy +++ b/tests/resources/macro_with_require.hy @@ -4,11 +4,11 @@ (defmacro test-module-macro [a] "The variable `macro-level-var' here should not bind to the same-named symbol -in the expansion of `nonlocal-test-macro'." +in the expansion of `remote-test-macro'." (setv macro-level-var "tests.resources.macros.macro-with-require") - `(nonlocal-test-macro ~a)) + `(remote-test-macro ~a)) (defmacro test-module-macro-2 [a] - "The macro `local-test-macro` isn't in this module's namespace, so it better + "The macro `home-test-macro` isn't in this module's namespace, so it better be in the expansion's!" - `(local-test-macro ~a)) + `(home-test-macro ~a)) diff --git a/tests/resources/macros.hy b/tests/resources/macros.hy index 004d55a42..2b8094eae 100644 --- a/tests/resources/macros.hy +++ b/tests/resources/macros.hy @@ -6,7 +6,7 @@ (defmacro test-macro-2 [] '(setv qup 2)) -(defmacro nonlocal-test-macro [x] +(defmacro remote-test-macro [x] "When called from `macro-with-require`'s macro(s), the first instance of `module-name-var` should resolve to the value in the module where this is defined, then the expansion namespace/module" From 6451e510ef33ed27744f6d499b273455d23e14e1 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 26 Aug 2023 10:07:44 -0400 Subject: [PATCH 281/342] Allow `defmacro` to define local macros --- hy/compiler.py | 24 +++++++++++ hy/core/result_macros.py | 64 ++++++++++++------------------ hy/macros.py | 18 +++++---- tests/native_tests/macros_local.hy | 54 +++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 46 deletions(-) create mode 100644 tests/native_tests/macros_local.hy diff --git a/hy/compiler.py b/hy/compiler.py index 1ef850928..f93ed0607 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -4,6 +4,7 @@ import inspect import traceback import types +from contextlib import contextmanager from funcparserlib.parser import NoParseError, many @@ -335,6 +336,10 @@ def __init__(self, module, filename=None, source=None): """ self.anon_var_count = 0 self.temp_if = None + self.local_macro_stack = [] + # A list of dictionaries that map mangled names to local + # macros. The last element is considered the top of the + # stack. if not inspect.ismodule(module): self.module = importlib.import_module(module) @@ -507,6 +512,25 @@ def _nonconst(self, name): raise self._syntax_error(name, "Can't assign to constant") return name + def eval(self, model): + return hy_eval( + model, + locals = self.module.__dict__, + module = self.module, + filename = self.filename, + source = self.source, + import_stdlib = False) + + @contextmanager + def local_macros(self): + """Make `defmacro` and `require` assign to a new element of + `self.local_macro_stack` instead of a module.""" + self.local_macro_stack.append({}) + try: + yield + finally: + self.local_macro_stack.pop() + @builds_model(Expression) def compile_expression(self, expr): # Perform macro expansions diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 24053129f..1a53050b9 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -16,7 +16,7 @@ from funcparserlib.parser import finished, forward_decl, many, maybe, oneplus, some from hy._compat import PY3_11, PY3_12 -from hy.compiler import Result, asty, hy_eval, mkexpr +from hy.compiler import Result, asty, mkexpr from hy.errors import HyEvalError, HyInternalError, HyTypeError from hy.macros import pattern_macro, require, require_reader from hy.model_patterns import ( @@ -120,14 +120,7 @@ def compile_eval_foo_compile(compiler, expr, root, body): new_expr = Expression([Symbol("do").replace(expr[0])]).replace(expr) try: - value = hy_eval( - new_expr + body, - compiler.module.__dict__, - compiler.module, - filename=compiler.filename, - source=compiler.source, - import_stdlib=False, - ) + value = compiler.eval(new_expr + body) except HyInternalError: # Unexpected "meta" compilation errors need to be treated # like normal (unexpected) compilation errors at this level @@ -745,7 +738,8 @@ def compile_comprehension(compiler, expr, root, parts, final): is_for = root == "for" ctx = nullcontext() if is_for else compiler.scope.create(ScopeGen) - with ctx as scope: + mac_con = nullcontext() if is_for else compiler.local_macros() + with mac_con, ctx as scope: # Compile the parts. if is_for: @@ -1444,7 +1438,7 @@ def compile_function_lambda(compiler, expr, root, tp, params, body): for param in (posonly or []) + args + kwonly + [rest, kwargs] ) args, ret = compile_lambda_list(compiler, params) - with compiler.scope.create(ScopeFn, args): + with compiler.local_macros(), compiler.scope.create(ScopeFn, args): body = compiler._compile_branch(body) # Compile to lambda if we can @@ -1472,7 +1466,7 @@ def compile_function_def(compiler, expr, root, decorators, tp, name, params, bod ret += ret2 name = mangle(compiler._nonconst(name)) compiler.scope.define(name) - with compiler.scope.create(ScopeFn, args): + with compiler.local_macros(), compiler.scope.create(ScopeFn, args): body = compiler._compile_branch(body) return ret + compile_function_node( @@ -1513,31 +1507,25 @@ def compile_function_node(compiler, expr, node, decorators, tp, name, args, retu ], ) def compile_macro_def(compiler, expr, root, name, params, body): - ret = Result() + compiler.compile( - Expression( - [ - Symbol("eval-and-compile"), - Expression( - [ - Expression( - [ - dotted("hy.macros.macro"), - str(name), - ] - ), - Expression( - [ - Symbol("fn"), - List([Symbol("&compiler")] + list(expr[2])), - *body, - ] - ), - ] - ), - ] - ).replace(expr) - ) - + def E(*x): return Expression(x) + S = Symbol + + ret = [] + fn_def = E( + S("fn"), + List([S("&compiler"), *expr[2]]), + *body) + if compiler.local_macro_stack: + # We're in a local macro scope, so define the new macro + # locally. + compiler.local_macro_stack[-1][mangle(name)] = ( + compiler.eval(fn_def.replace(expr))) + return Result() + # Otherwise, define the macro module-wide. + ret.append(E( + E(dotted("hy.macros.macro"), str(name)), + fn_def)) + ret = compiler.compile(E(S("eval-and-compile"), *ret).replace(expr)) return ret + ret.expr_as_stmt() @@ -1695,7 +1683,7 @@ def compile_class_expression(compiler, expr, root, decorators, tp, name, rest): name = mangle(compiler._nonconst(name)) compiler.scope.define(name) - with compiler.scope.create(ScopeFn): + with compiler.local_macros(), compiler.scope.create(ScopeFn): e = compiler._compile_branch(body) bodyr += e + e.expr_as_stmt() diff --git a/hy/macros.py b/hy/macros.py index 180afe223..dd7e2e177 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -388,14 +388,16 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): break # Choose the first namespace with the macro. - m = next( - ( - mod._hy_macros[fn] - for mod in (module, builtins) - if fn in getattr(mod, "_hy_macros", ()) - ), - None, - ) + m = ((compiler and next( + (d[fn] + for d in reversed(compiler.local_macro_stack) + if fn in d), + None)) or + next( + (mod._hy_macros[fn] + for mod in (module, builtins) + if fn in getattr(mod, "_hy_macros", ())), + None)) if not m: break diff --git a/tests/native_tests/macros_local.hy b/tests/native_tests/macros_local.hy new file mode 100644 index 000000000..0f6a81960 --- /dev/null +++ b/tests/native_tests/macros_local.hy @@ -0,0 +1,54 @@ +"Tests of local macro definitions." + + +(defn test-nonleaking [] + (defn fun [] + (defmacro helper [] + "helper macro in fun") + (helper)) + (defclass C [] + (defmacro helper [] + "helper macro in class") + (setv attribute (helper))) + (defn helper [] + "helper function") + (assert (= (helper) "helper function")) + (assert (= (fun) "helper macro in fun")) + (assert (= C.attribute "helper macro in class")) + (assert (= + (lfor + x [1 2 3] + :do (defmacro helper [] + "helper macro in lfor") + y [1 2 3] + (if (= x y 2) (helper) (+ (* x 10) y))) + [11 12 13 21 "helper macro in lfor" 23 31 32 33])) + (assert (= (helper) "helper function"))) + + +(defmacro shadowable [] + "global version") + +(defn test-shadowing-global [] + (defn inner [] + (defmacro shadowable [] + "local version") + (shadowable)) + (assert (= (shadowable) "global version")) + (assert (= (inner) "local version")) + (assert (= (shadowable) "global version"))) + + +(defn test-nested-local-shadowing [] + (defn inner1 [] + (defmacro shadowable [] + "local version 1") + (defn inner2 [] + (defmacro shadowable [] + "local version 2") + (shadowable)) + [(inner2) (shadowable)]) + (assert (= (shadowable) "global version")) + (print (inner1)) + (assert (= (inner1) ["local version 2" "local version 1"])) + (assert (= (shadowable) "global version"))) From 71b39e2b3f8a3f81153b0b9274fa53434546e7a9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 25 Aug 2023 10:25:10 -0400 Subject: [PATCH 282/342] Overhaul some tests of `require` The most notable change here is that the `require`s are now at the module level instead of in functions. --- tests/native_tests/import.hy | 72 ++++++++++++++--------------- tests/resources/more_test_macros.hy | 8 ++++ 2 files changed, 43 insertions(+), 37 deletions(-) create mode 100644 tests/resources/more_test_macros.hy diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy index 29e6f6bbc..a00642ee7 100644 --- a/tests/native_tests/import.hy +++ b/tests/native_tests/import.hy @@ -59,54 +59,52 @@ (assert (in "_null_fn_for_import_test" (dir tests.resources.bin)))) -(defn test-require [] - (with [(pytest.raises NameError)] - (qplah 1 2 3 4)) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) - (with [(pytest.raises NameError)] - (✈ 1 2 3 4)) - (with [(pytest.raises NameError)] - (hyx_XairplaneX 1 2 3 4)) - - (require tests.resources.tlib [qplah]) - (assert (= (qplah 1 2 3) [8 1 2 3])) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) - - (require tests.resources.tlib) +(require + tests.resources.tlib + tests.resources.tlib :as TL + tests.resources.tlib [qplah] + tests.resources.tlib [parald :as parald-alias] + tests.resources [tlib macros :as TM exports-none] + os [path]) + ; The last one is a no-op, since the module `os.path` exists but + ; contains no macros. + +(defn test-require-global [] (assert (= (tests.resources.tlib.parald 1 2 3) [9 1 2 3])) (assert (= (tests.resources.tlib.✈ "silly") "plane silly")) (assert (= (tests.resources.tlib.hyx_XairplaneX "foolish") "plane foolish")) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) - (require tests.resources.tlib :as T) - (assert (= (T.parald 1 2 3) [9 1 2 3])) - (assert (= (T.✈ "silly") "plane silly")) - (assert (= (T.hyx_XairplaneX "foolish") "plane foolish")) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) + (assert (= (TL.parald 1 2 3) [9 1 2 3])) + (assert (= (TL.✈ "silly") "plane silly")) + (assert (= (TL.hyx_XairplaneX "foolish") "plane foolish")) - (require tests.resources.tlib [parald :as p]) - (assert (= (p 1 2 3) [9 1 2 3])) - (with [(pytest.raises NameError)] - (parald 1 2 3 4)) + (assert (= (qplah 1 2 3) [8 1 2 3])) - (require tests.resources.tlib *) - (assert (= (parald 1 2 3) [9 1 2 3])) - (assert (= (✈ "silly") "plane silly")) - (assert (= (hyx_XairplaneX "foolish") "plane foolish")) + (assert (= (parald-alias 1 2 3) [9 1 2 3])) - (require tests.resources [tlib macros :as m exports-none]) (assert (in "tlib.qplah" _hy_macros)) - (assert (in (hy.mangle "m.test-macro") _hy_macros)) + (assert (in (hy.mangle "TM.test-macro") _hy_macros)) (assert (in (hy.mangle "exports-none.cinco") _hy_macros)) - (require os [path]) + + (with [(pytest.raises NameError)] + (parald 1 2 3 4)) + (with [(pytest.raises hy.errors.HyRequireError)] - (hy.eval '(require tests.resources [does-not-exist]))) + (hy.eval '(require tests.resources [does-not-exist])))) + + +(require tests.resources.more-test-macros *) + +(defn test-require-global-star-without-exports [] + (assert (= (bairn 1 2 3) [14 1 2 3])) + (assert (= (cairn 1 2 3) [15 1 2 3])) + (with [(pytest.raises NameError)] + (_dairn 1 2 3 4))) + + +(require tests.resources.exports *) - (require tests.resources.exports *) +(defn test-require-global-star-with-exports [] (assert (= (casey 1 2 3) [11 1 2 3])) (assert (= (☘ 1 2 3) [13 1 2 3])) (with [(pytest.raises NameError)] diff --git a/tests/resources/more_test_macros.hy b/tests/resources/more_test_macros.hy new file mode 100644 index 000000000..478c87bf4 --- /dev/null +++ b/tests/resources/more_test_macros.hy @@ -0,0 +1,8 @@ +(defmacro bairn [#* tree] + `[14 ~@tree]) + +(defmacro cairn [ #* tree] + `[15 ~@tree]) + +(defmacro _dairn [] + `[16 ~@tree]) From f6e81d8605307082b5f9280d7dee3cfc116d0001 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 25 Aug 2023 10:27:49 -0400 Subject: [PATCH 283/342] Rewrite `test-requires-pollutes-core` This version avoids some complexities about `require` possibly executing too soon, and it relieves us from managing the bytecode manually. --- tests/native_tests/import.hy | 45 ------------------------------------ tests/test_bin.py | 23 ++++++++++++++++++ 2 files changed, 23 insertions(+), 45 deletions(-) diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy index a00642ee7..3d6c3b1b5 100644 --- a/tests/native_tests/import.hy +++ b/tests/native_tests/import.hy @@ -166,51 +166,6 @@ in expansions." (test-module-macro-2 3)))) -(defn test-requires-pollutes-core [] - ;; https://github.com/hylang/hy/issues/1978 - ;; Macros loaded from an external module should not pollute `_hy_macros` - ;; with macros from core. - - (setv pyc-file (importlib.util.cache-from-source - (os.path.realpath - (os.path.join - "tests" "resources" "macros.hy")))) - - ;; Remove any cached byte-code, so that this runs from source and - ;; gets evaluated in this module. - (when (os.path.isfile pyc-file) - (os.unlink pyc-file) - (.clear sys.path_importer_cache) - (when (in "tests.resources.macros" sys.modules) - (del (get sys.modules "tests.resources.macros")) - (_hy_macros.clear))) - - ;; Ensure that bytecode isn't present when we require this module. - (assert (not (os.path.isfile pyc-file))) - - (defn require-macros [] - (require tests.resources.macros :as m) - - (assert (in (hy.mangle "m.test-macro") _hy_macros)) - (for [macro-name _hy_macros] - (assert (not (and (in "with" macro-name) - (!= "with" macro-name)))))) - - (require-macros) - - ;; Now that bytecode is present, reload the module, clear the `require`d - ;; macros and tags, and rerun the tests. - (when (not PYODIDE) - (assert (os.path.isfile pyc-file))) - - ;; Reload the module and clear the local macro context. - (.clear sys.path_importer_cache) - (del (get sys.modules "tests.resources.macros")) - (.clear _hy_macros) - - (require-macros)) - - (defn test-no-surprise-shadow [tmp-path monkeypatch] "Check that an out-of-module macro doesn't shadow a function." ; https://github.com/hylang/hy/issues/2451 diff --git a/tests/test_bin.py b/tests/test_bin.py index f283b3e1d..6a732c40d 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -663,6 +663,29 @@ def test_relative_require(case, monkeypatch, tmp_path): assert 'HELLO' in output +def test_require_doesnt_pollute_core(monkeypatch, tmp_path): + # https://github.com/hylang/hy/issues/1978 + """Macros loaded from an external module should not pollute + `_hy_macros` with macros from core.""" + + (tmp_path / 'aaa.hy').write_text(''' + (defmacro foo [] + '(setv x (.upper "argelfraster")))''') + (tmp_path / 'bbb.hy').write_text(''' + (require aaa :as A) + (A.foo) + (print + x + (not-in "if" _hy_macros) + (not-in "cond" _hy_macros))''') + # `if` is a result macro; `cond` is a regular macro. + monkeypatch.chdir(tmp_path) + + # Try it without and then with bytecode. + for _ in (1, 2): + assert 'ARGELFRASTER True True' in run_cmd('hy bbb.hy')[0] + + def test_run_dir_or_zip(tmp_path): (tmp_path / 'dir').mkdir() From d219efa580ba417fe24568e7dbf20f5ee23e8b24 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 25 Aug 2023 14:50:47 -0400 Subject: [PATCH 284/342] Remove a redundant `require` test --- tests/native_tests/import.hy | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy index 3d6c3b1b5..99db0bd5f 100644 --- a/tests/native_tests/import.hy +++ b/tests/native_tests/import.hy @@ -111,17 +111,6 @@ (brother 1 2 3 4))) -(defn test-require-native [] - (with [(pytest.raises NameError)] - (test-macro-2)) - (import tests.resources.macros) - (with [(pytest.raises NameError)] - (test-macro-2)) - (require tests.resources.macros [test-macro-2]) - (test-macro-2) - (assert (= qup 2))) - - (defn test-relative-require [] (require ..resources.macros [test-macro]) (assert (in "test_macro" _hy_macros)) From c89d7bfc5ccafc975adbb1ee908a80798ab7e0f0 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 25 Aug 2023 14:52:31 -0400 Subject: [PATCH 285/342] Make `test-relative-require` req at the top level --- tests/native_tests/import.hy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy index 99db0bd5f..c3644cfde 100644 --- a/tests/native_tests/import.hy +++ b/tests/native_tests/import.hy @@ -111,15 +111,15 @@ (brother 1 2 3 4))) -(defn test-relative-require [] - (require ..resources.macros [test-macro]) - (assert (in "test_macro" _hy_macros)) +(require + ..resources.macros [test-macro-2] + .beside [xyzzy] + . [beside :as BS]) - (require .beside [xyzzy]) +(defn test-require-global-relative [] + (assert (in "test_macro_2" _hy_macros)) (assert (in "xyzzy" _hy_macros)) - - (require . [beside :as b]) - (assert (in "b.xyzzy" _hy_macros))) + (assert (in "BS.xyzzy" _hy_macros))) ;; `remote-test-macro` is a macro used within From 07ea69f0e723762f64547e97b4394cdbbbd6978f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 26 Aug 2023 10:33:54 -0400 Subject: [PATCH 286/342] Allow `require` to define local macros --- hy/core/result_macros.py | 4 +- hy/macros.py | 56 ++++++++++++---------------- tests/native_tests/macros_local.hy | 23 +++++++++++- tests/resources/local_req_example.hy | 6 +++ 4 files changed, 54 insertions(+), 35 deletions(-) create mode 100644 tests/resources/local_req_example.hy diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 1a53050b9..122870c3a 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1784,7 +1784,9 @@ def compile_require(compiler, expr, root, entries): # we don't want to import all macros as prefixed if we're specifically # importing readers but not macros # (require a-module :readers ["!"]) - if (rest or not readers) and require( + if (rest or not readers) and compiler.local_macro_stack: + require(module_name, compiler.local_macro_stack[-1], assignments=assignments, prefix=prefix) + elif (rest or not readers) and require( module_name, compiler.module, assignments=assignments, prefix=prefix ): # Actually calling `require` is necessary for macro expansions diff --git a/hy/macros.py b/hy/macros.py index dd7e2e177..9841b6d15 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -195,42 +195,32 @@ def enable_readers(module, reader, names): reader.reader_macros[name] = namespace["_hy_reader_macros"][name] -def require(source_module, target_module, assignments, prefix="", target_module_name=None): - """Load macros from one module into the namespace of another. +def require(source_module, target, assignments, prefix="", target_module_name=None): + """Load macros from a module. Return a `bool` indicating whether + macros were actually transferred. - This function is called from the macro also named `require`. + - `target` can be a a string (naming a module), a module object, + a dictionary, or `None` (meaning the calling module). + - `assignments` can be "ALL", "EXPORTS", or a list of (macro + name, alias) pairs.""" - Args: - source_module (Union[str, ModuleType]): The module from which macros are - to be imported. - target_module (Optional[Union[str, ModuleType]]): The module into which the - macros will be loaded. If `None`, then the caller's namespace. - The latter is useful during evaluation of generated AST/bytecode. - assignments (Union[str, typing.Sequence[str]]): The string "ALL", the string - "EXPORTS", or a list of macro name and alias pairs. - prefix (str): If nonempty, its value is prepended to the name of each imported macro. - This allows one to emulate namespaced macros, like "mymacromodule.mymacro", - which looks like an attribute of a module. Defaults to "" - target_module_name: If true, overrides the apparent name of `target_module`. - - Returns: - bool: Whether or not macros were actually transferred. - """ - target_module, target_namespace = derive_target_module( - target_module, inspect.stack()[1][0] - ) - - # Let's do a quick check to make sure the source module isn't actually - # the module being compiled (e.g. when `runpy` executes a module's code - # in `__main__`). - # We use the module's underlying filename for this (when they exist), since - # it's the most "fixed" attribute. - if _same_modules(source_module, target_module): - return False + if type(target) is dict: + target_module = None + else: + target_module, target_namespace = derive_target_module( + target, inspect.stack()[1][0] + ) + # Let's do a quick check to make sure the source module isn't actually + # the module being compiled (e.g. when `runpy` executes a module's code + # in `__main__`). + # We use the module's underlying filename for this (when they exist), since + # it's the most "fixed" attribute. + if _same_modules(source_module, target_module): + return False if not inspect.ismodule(source_module): source_module = import_module_from_string(source_module, - target_module_name or target_module) + target_module_name or target_module or '') source_macros = source_module.__dict__.setdefault("_hy_macros", {}) source_exports = getattr( @@ -246,7 +236,7 @@ def require(source_module, target_module, assignments, prefix="", target_module_ try: require( f"{source_module.__name__}.{mangle(name)}", - target_module, + target_module or target, "ALL", prefix=alias, ) @@ -258,7 +248,7 @@ def require(source_module, target_module, assignments, prefix="", target_module_ ) return True - target_macros = target_namespace.setdefault("_hy_macros", {}) + target_macros = target_namespace.setdefault("_hy_macros", {}) if target_module else target if prefix: prefix += "." diff --git a/tests/native_tests/macros_local.hy b/tests/native_tests/macros_local.hy index 0f6a81960..af18772aa 100644 --- a/tests/native_tests/macros_local.hy +++ b/tests/native_tests/macros_local.hy @@ -1,4 +1,4 @@ -"Tests of local macro definitions." +"Tests of local `defmacro` and `require`." (defn test-nonleaking [] @@ -52,3 +52,24 @@ (print (inner1)) (assert (= (inner1) ["local version 2" "local version 1"])) (assert (= (shadowable) "global version"))) + + +(defmacro local-require-test [arg] `(do + (defmacro wiz [] + "local wiz") + + (defn fun [] + (require tests.resources.local-req-example ~arg) + [(get-wiz) (helper)]) + (defn helper [] + "local helper function") + + (assert (= [(wiz) (helper)] ["local wiz" "local helper function"])) + (assert (= (fun) ["remote wiz" "remote helper macro"])) + (assert (= [(wiz) (helper)] ["local wiz" "local helper function"])))) + +(defn test-require [] + (local-require-test [get-wiz helper])) + +(defn test-require-star [] + (local-require-test *)) diff --git a/tests/resources/local_req_example.hy b/tests/resources/local_req_example.hy new file mode 100644 index 000000000..82467fc1a --- /dev/null +++ b/tests/resources/local_req_example.hy @@ -0,0 +1,6 @@ +(defmacro wiz [] + "remote wiz") +(defmacro get-wiz [] + (wiz)) +(defmacro helper [] + "remote helper macro") From 0469c410b156bb303ff3b9658921d7b16e532407 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 26 Aug 2023 10:44:38 -0400 Subject: [PATCH 287/342] Add local macros in doc of `defmacro`, `require` --- docs/api.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 00b2e249c..2cb36ba2c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -198,6 +198,10 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. => (infix (1 + 1)) 2 + If ``defmacro`` appears in a function definition, a class definition, or a + comprehension other than :hy:func:`for` (such as :hy:func:`lfor`), the new + macro is defined locally rather than module-wide. + .. note:: ``defmacro`` cannot use keyword arguments, because all values are passed to macros unevaluated. All arguments are passed positionally, but they can have default values:: @@ -895,7 +899,8 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. ``require`` is used to import macros and reader macros from one or more given modules. It allows parameters in all the same formats as ``import``. ``require`` imports each named module and then makes each requested macro - available in the current module. + available in the current module, or in the current local scope if called + locally (using the same notion of locality as :hy:func:`defmacro`). The following are all equivalent ways to call a macro named ``foo`` in the module ``mymodule``. @@ -939,8 +944,8 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. NameError: name 'mymodule' is not defined Unlike requiring regular macros, reader macros cannot be renamed - with ``:as``, and are not made available under their absolute paths - to their source module:: + with ``:as``, are not made available under their absolute paths + to their source module, and can't be required locally:: => (require mymodule :readers [!]) HySyntaxError: ... From 234c2ee59475da41620f62ed8e398133433bce12 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 26 Aug 2023 10:50:11 -0400 Subject: [PATCH 288/342] Document things that don't work with local macros --- hy/compiler.py | 4 +++- hy/core/macros.hy | 6 ++++-- hy/macros.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index f93ed0607..97f4c6572 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -751,7 +751,9 @@ def hy_eval_user(model, globals = None, locals = None, module = None): (hy.eval '(my-test-mac)) ; => 3 (import hyrule) (hy.eval '(my-test-mac) :module hyrule) ; NameError - (hy.eval '(list-n 3 1) :module hyrule) ; => [1 1 1]""" + (hy.eval '(list-n 3 1) :module hyrule) ; => [1 1 1] + + N.B. Local macros are invisible to ``hy.eval``.""" if locals is None: locals = globals diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 9746aaaab..e02dd7e0f 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -117,7 +117,8 @@ (defmacro doc [symbol] "macro documentation - Gets help for a macro function available in this module. + Gets help for a macro function available in this module (not a local + macro). Use ``require`` to make other macros available. Use ``(help foo)`` instead for help with runtime objects." @@ -168,7 +169,8 @@ (defmacro delmacro [#* names] - #[[Delete a macro(s) from the current module + #[[Delete a macro(s) from the current module. This doesn't work on a + local macro. :: => (require a-module [some-macro]) diff --git a/hy/macros.py b/hy/macros.py index 9841b6d15..40c816378 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -338,7 +338,7 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): """Expand the toplevel macros for the given Hy AST tree. Load the macros from the given `module`, then expand the (top-level) macros - in `tree` until we no longer can. + in `tree` until we no longer can. This doesn't work on local macros. `Expression` resulting from macro expansions are assigned the module in which the macro function is defined (determined using `inspect.getmodule`). From 92e129d4fcf52ac8140be9cee397cafd9469ec38 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 26 Aug 2023 10:51:42 -0400 Subject: [PATCH 289/342] Update NEWS --- NEWS.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 2eae76998..83af73db5 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,13 @@ Unreleased Breaking Changes ------------------------------ + +* `defmacro` and `require` can now define macros locally instead of + only module-wide. + + * `hy.eval`, `hy.macroexpand`, `doc`, and `delmacro` don't work with + local macros (yet). + * When a macro is `require`\d from another module, that module is no longer implicitly included when checking for further macros in the expansion. From d5ccb39c4537c532f58d5e93368043d49c361b5e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 28 Aug 2023 12:57:27 -0400 Subject: [PATCH 290/342] Test local macros appearing in other expansions --- tests/native_tests/macros_local.hy | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/native_tests/macros_local.hy b/tests/native_tests/macros_local.hy index af18772aa..26c39e7e6 100644 --- a/tests/native_tests/macros_local.hy +++ b/tests/native_tests/macros_local.hy @@ -54,6 +54,19 @@ (assert (= (shadowable) "global version"))) +(defmacro one-plus-two [] + '(+ 1 2)) + +(defn test-local-macro-in-expansion-of-nonlocal [] + (defn f [] + (defmacro + [a b] + "Shadow the core macro `+`. #yolo" + `f"zomg! {~a} {~b}") + (one-plus-two)) + (assert (= (f) "zomg! 1 2")) + (assert (= (one-plus-two) 3))) + + (defmacro local-require-test [arg] `(do (defmacro wiz [] "local wiz") From 1f486002b538c5d917e583b51e74e0abce3ee3d7 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 5 Sep 2023 17:40:08 -0400 Subject: [PATCH 291/342] Refine the documentation of `return` --- docs/api.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 2cb36ba2c..847c77038 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1033,17 +1033,17 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (return n)))) (f 3.9) ; => 4 - Note that in Hy, ``return`` is necessary much less often than in Python, - since the last form of a function is returned automatically. Hence, an + Note that in Hy, ``return`` is necessary much less often than in Python. + The last form of a function is returned automatically, so an explicit ``return`` is only necessary to exit a function early. To force Python's behavior of returning ``None`` when execution reaches the end of a - function, you can put ``None`` there yourself:: + function, just put ``None`` there yourself:: - (defn f [x] - (setv y 10) - (print (+ x y)) + (defn f [] + (setv d (dict :a 1 :b 2)) + (.pop d "b") None) - (print (f 4)) ; Prints "14" and then "None" + (print (f)) ; Prints "None", not "2" .. hy:macro:: (raise [exception :from other]) From 36adbdcc921b8b3baafa189df2da77c7ad5d24f3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 5 Sep 2023 17:40:27 -0400 Subject: [PATCH 292/342] Refine the documentation of `gensym` --- hy/core/util.hy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/core/util.hy b/hy/core/util.hy index 1b4e71f3c..fe939e477 100644 --- a/hy/core/util.hy +++ b/hy/core/util.hy @@ -39,7 +39,7 @@ (defn gensym [[g ""]] #[[Generate a symbol with a unique name. The argument will be included in the - generated symbol, as an aid to debugging. Typically one calls ``hy.gensym`` + generated symbol name, as an aid to debugging. Typically one calls ``hy.gensym`` without an argument. .. seealso:: From 500f002886bd018f47e3acb334865e5258443fcc Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 5 Sep 2023 17:18:35 -0400 Subject: [PATCH 293/342] Overhaul warnings for shadowing core macros - Warnings can now be produced by `require` as well as `defmacro`. - Warnings can now be produced by local macro definitions. - Warnings are produced only at compile-time, not also at run-time. - Warnings can be disabled with a pragma. --- hy/compiler.py | 44 ++++++++++++++++++++------ hy/core/result_macros.py | 51 +++++++++++++++++++----------- hy/macros.py | 24 ++++---------- tests/compilers/test_ast.py | 5 +-- tests/native_tests/macros.hy | 24 +++++++++++--- tests/native_tests/macros_local.hy | 9 ++++++ 6 files changed, 106 insertions(+), 51 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 97f4c6572..d9f2b2dbf 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1,9 +1,11 @@ import ast +import builtins import copy import importlib import inspect import traceback import types +import warnings from contextlib import contextmanager from funcparserlib.parser import NoParseError, many @@ -336,10 +338,12 @@ def __init__(self, module, filename=None, source=None): """ self.anon_var_count = 0 self.temp_if = None - self.local_macro_stack = [] - # A list of dictionaries that map mangled names to local - # macros. The last element is considered the top of the - # stack. + + # Make a list of dictionaries with local compiler settings, + # such as the definitions of local macros. The last element is + # considered the top of the stack. + self.local_state_stack = [] + self.new_local_state() if not inspect.ismodule(module): self.module = importlib.import_module(module) @@ -364,6 +368,30 @@ def __init__(self, module, filename=None, source=None): self.scope = ScopeGlobal(self) + def new_local_state(self): + 'Add a new local state to the top of the stack.' + self.local_state_stack.append(dict(macros = {})) + + def is_in_local_state(self): + return len(self.local_state_stack) > 1 + + def get_local_option(self, key, default): + 'Get the topmost available value of a local-state setting.' + return next( + (s[key] + for s in reversed(self.local_state_stack) + if key in s), + default) + + def warn_on_core_shadow(self, name): + if ( + mangle(name) in getattr(builtins, "_hy_macros", {}) and + self.get_local_option('warn_on_core_shadow', True)): + warnings.warn( + f"New macro `{name}` will shadow the core macro of the same name", + RuntimeWarning + ) + def get_anon_var(self, base="_hy_anon_var"): self.anon_var_count += 1 return f"{base}_{self.anon_var_count}" @@ -522,14 +550,12 @@ def eval(self, model): import_stdlib = False) @contextmanager - def local_macros(self): - """Make `defmacro` and `require` assign to a new element of - `self.local_macro_stack` instead of a module.""" - self.local_macro_stack.append({}) + def local_state(self): + self.new_local_state() try: yield finally: - self.local_macro_stack.pop() + self.local_state_stack.pop() @builds_model(Expression) def compile_expression(self, expr): diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 122870c3a..81c24ffec 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -738,7 +738,7 @@ def compile_comprehension(compiler, expr, root, parts, final): is_for = root == "for" ctx = nullcontext() if is_for else compiler.scope.create(ScopeGen) - mac_con = nullcontext() if is_for else compiler.local_macros() + mac_con = nullcontext() if is_for else compiler.local_state() with mac_con, ctx as scope: # Compile the parts. @@ -1438,7 +1438,7 @@ def compile_function_lambda(compiler, expr, root, tp, params, body): for param in (posonly or []) + args + kwonly + [rest, kwargs] ) args, ret = compile_lambda_list(compiler, params) - with compiler.local_macros(), compiler.scope.create(ScopeFn, args): + with compiler.local_state(), compiler.scope.create(ScopeFn, args): body = compiler._compile_branch(body) # Compile to lambda if we can @@ -1466,7 +1466,7 @@ def compile_function_def(compiler, expr, root, decorators, tp, name, params, bod ret += ret2 name = mangle(compiler._nonconst(name)) compiler.scope.define(name) - with compiler.local_macros(), compiler.scope.create(ScopeFn, args): + with compiler.local_state(), compiler.scope.create(ScopeFn, args): body = compiler._compile_branch(body) return ret + compile_function_node( @@ -1510,15 +1510,15 @@ def compile_macro_def(compiler, expr, root, name, params, body): def E(*x): return Expression(x) S = Symbol + compiler.warn_on_core_shadow(name) ret = [] fn_def = E( S("fn"), List([S("&compiler"), *expr[2]]), *body) - if compiler.local_macro_stack: - # We're in a local macro scope, so define the new macro - # locally. - compiler.local_macro_stack[-1][mangle(name)] = ( + if compiler.is_in_local_state(): + # We're in a local scope, so define the new macro locally. + compiler.local_state_stack[-1]['macros'][mangle(name)] = ( compiler.eval(fn_def.replace(expr))) return Result() # Otherwise, define the macro module-wide. @@ -1683,7 +1683,7 @@ def compile_class_expression(compiler, expr, root, decorators, tp, name, rest): name = mangle(compiler._nonconst(name)) compiler.scope.define(name) - with compiler.local_macros(), compiler.scope.create(ScopeFn): + with compiler.local_state(), compiler.scope.create(ScopeFn): e = compiler._compile_branch(body) bodyr += e + e.expr_as_stmt() @@ -1784,11 +1784,19 @@ def compile_require(compiler, expr, root, entries): # we don't want to import all macros as prefixed if we're specifically # importing readers but not macros # (require a-module :readers ["!"]) - if (rest or not readers) and compiler.local_macro_stack: - require(module_name, compiler.local_macro_stack[-1], assignments=assignments, prefix=prefix) + if (rest or not readers) and compiler.is_in_local_state(): + require( + module_name, + compiler.local_state_stack[-1]['macros'], + assignments = assignments, + prefix = prefix, + compiler = compiler) elif (rest or not readers) and require( - module_name, compiler.module, assignments=assignments, prefix=prefix - ): + module_name, + compiler.module, + assignments = assignments, + prefix = prefix, + compiler = compiler): # Actually calling `require` is necessary for macro expansions # occurring during compilation. # The `require` we're creating in AST is the same as above, but used at @@ -1950,13 +1958,20 @@ def compile_deftype(compiler, expr, root, tp, name, value): **digest_type_params(compiler, tp)) +@pattern_macro("pragma", [many(KEYWORD + FORM)]) +def compile_pragma(compiler, expr, root, kwargs): + for kw, value in kwargs: + if kw == Keyword("warn-on-core-shadow"): + compiler.local_state_stack[-1]['warn_on_core_shadow'] = ( + bool(compiler.eval(value))) + else: + raise compiler._syntax_error(kw, f"Unknown pragma `{kw}`. Perhaps it's implemented by a newer version of Hy.") + return Result() + + @pattern_macro( - "pragma unquote unquote-splice unpack-mapping except except* finally else".split(), + "unquote unquote-splice unpack-mapping except except* finally else".split(), [many(FORM)], ) def compile_placeholder(compiler, expr, root, body): - raise ValueError( - "`{}` is not allowed {}".format( - root, "in this version of Hy" if root == "pragma" else "here" - ) - ) + raise ValueError(f"`{root}` is not allowed here") diff --git a/hy/macros.py b/hy/macros.py index 40c816378..657999500 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -5,7 +5,6 @@ import re import sys import traceback -import warnings from ast import AST from funcparserlib.parser import NoParseError @@ -84,18 +83,7 @@ def wrapper(hy_compiler, *args): def install_macro(name, fn, module_of): name = mangle(name) fn = rename_function(fn, name) - macros_obj = module_of.__globals__.setdefault("_hy_macros", {}) - if name in getattr(builtins, "_hy_macros", {}): - warnings.warn( - ( - f"{name} already refers to: `{name}` in module: `builtins`," - f" being replaced by: `{module_of.__globals__.get('__name__', '(globals)')}.{name}`" - ), - RuntimeWarning, - stacklevel=3, - ) - - macros_obj[name] = fn + module_of.__globals__.setdefault("_hy_macros", {})[name] = fn return fn @@ -195,7 +183,7 @@ def enable_readers(module, reader, names): reader.reader_macros[name] = namespace["_hy_reader_macros"][name] -def require(source_module, target, assignments, prefix="", target_module_name=None): +def require(source_module, target, assignments, prefix="", target_module_name=None, compiler=None): """Load macros from a module. Return a `bool` indicating whether macros were actually transferred. @@ -263,6 +251,8 @@ def require(source_module, target, assignments, prefix="", target_module_name=No ) ): _name = mangle(name) + if compiler: + compiler.warn_on_core_shadow(prefix + alias) alias = mangle(prefix + alias) if _name in source_module._hy_macros: target_macros[alias] = source_macros[_name] @@ -379,9 +369,9 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): # Choose the first namespace with the macro. m = ((compiler and next( - (d[fn] - for d in reversed(compiler.local_macro_stack) - if fn in d), + (d['macros'][fn] + for d in reversed(compiler.local_state_stack) + if fn in d['macros']), None)) or next( (mod._hy_macros[fn] diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index fc92493e1..27cf35053 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -627,8 +627,9 @@ def test_module_prelude(): def test_pragma(): - cant_compile("(pragma)") - cant_compile("(pragma :native-code :namespaced-symbols :give-user-a-pony)") + can_compile("(pragma)") + can_compile("(pragma :warn-on-core-shadow True)") + cant_compile("(pragma :native-code True)") def test_error_with_expectation(): diff --git a/tests/native_tests/macros.hy b/tests/native_tests/macros.hy index 90a7743c8..9156d5972 100644 --- a/tests/native_tests/macros.hy +++ b/tests/native_tests/macros.hy @@ -1,4 +1,4 @@ -(import os sys +(import os sys warnings pytest hy.errors [HySyntaxError HyTypeError HyMacroExpansionError]) @@ -186,7 +186,21 @@ (with [exc (pytest.raises NameError)] (hy.eval '(parald)))) -(defn test-macro-redefinition-warning - [] - (with [(pytest.warns RuntimeWarning :match "require already refers to")] - (hy.eval '(defmacro require [] 1)))) + +(defn macro-redefinition-warning-tester [local] + (for [should-warn? [True False] head ["defmacro" "require"]] + (with [(if should-warn? + (pytest.warns RuntimeWarning :match "will shadow the core macro") + (warnings.catch-warnings))] + (when (not should-warn?) + ; Elevate any warning to an error. + (warnings.simplefilter "error")) + (hy.eval `( + ~@(if local '[defn f []] '[do]) + ~(if should-warn? None '(pragma :warn-on-core-shadow False)) + ~(if (= head "defmacro") + '(defmacro when [] 1) + '(require tests.resources.tlib [qplah :as when]))))))) + +(defn test-macro-redefinition-warning [] + (macro-redefinition-warning-tester :local False)) diff --git a/tests/native_tests/macros_local.hy b/tests/native_tests/macros_local.hy index 26c39e7e6..6126b9c29 100644 --- a/tests/native_tests/macros_local.hy +++ b/tests/native_tests/macros_local.hy @@ -1,5 +1,9 @@ "Tests of local `defmacro` and `require`." +(import + tests.native-tests.macros [macro-redefinition-warning-tester] + pytest) + (defn test-nonleaking [] (defn fun [] @@ -59,6 +63,7 @@ (defn test-local-macro-in-expansion-of-nonlocal [] (defn f [] + (pragma :warn-on-core-shadow False) (defmacro + [a b] "Shadow the core macro `+`. #yolo" `f"zomg! {~a} {~b}") @@ -86,3 +91,7 @@ (defn test-require-star [] (local-require-test *)) + + +(defn test-redefinition-warning [] + (macro-redefinition-warning-tester :local True)) From 0b07ecb5f4d513d867ba571010ae6407739d3e8b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 5 Sep 2023 17:22:07 -0400 Subject: [PATCH 294/342] Update NEWS --- NEWS.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 83af73db5..4b3e70821 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -28,6 +28,7 @@ New Features * `defn`, `defn/a`, and `defclass` now support type parameters. * `HyReader` now has an optional parameter to install existing reader macros from the calling module. +* New pragma `warn-on-core-shadow`. Misc. Improvements ------------------------------ @@ -42,6 +43,8 @@ Bug Fixes * `~@ #*` now produces a syntax error instead of a nonsensical result. * Fixed `hy.eval` failing on `defreader` or `require` forms that install a new reader. +* `require` now warns when you shadow a core macro, like `defmacro` + already did. 0.27.0 (released 2023-07-06) ============================= From 23746e2de6d933237fa74048838ba5810c4bd0a3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 5 Sep 2023 17:38:37 -0400 Subject: [PATCH 295/342] Document the new pragma --- docs/api.rst | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 847c77038..ed191ff97 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1274,11 +1274,27 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. (deftype IntOrStr (| int str)) (deftype :tp [T] ListOrSet (| (get list T) (get set T))) -.. hy:macro:: (pragma) +.. hy:macro:: (pragma [#* args]) - ``pragma`` is reserved as a core macro name for future use, especially for - allowing backwards-compatible addition of new features after the release of Hy - 1.0. Currently, trying to use ``pragma`` is an error. + ``pragma`` is used to adjust the state of the compiler. It's called for its + side-effects, and returns ``None``. The arguments are key-value pairs, like a + function call with keyword arguments:: + + (pragma :prag1 value1 :prag2 (get-value2)) + + Each key is a literal keyword giving the name of a pragma. Each value is an + arbitrary form, which is evaluated as ordinary Hy code but at compile-time. + + The effect of each pragma is locally scoped to its containing function, + class, or comprehension form (other than ``for``), if there is one. + + Only one pragma is currently implemented: + + - ``:warn-on-core-shadow``: If true (the default), :hy:func:`defmacro` and + :hy:func:`require` will raise a warning at compile-time if you define a macro + with the same name as a core macro. Shadowing a core macro in this fashion is + dangerous, because other macros may call your new macro when they meant to + refer to the core macro. .. hy:automodule:: hy.core.macros :macros: From e891aa7a6c1b0163aac3c79a880a5707d1ff458e Mon Sep 17 00:00:00 2001 From: wrobell Date: Fri, 15 Sep 2023 10:31:07 +0100 Subject: [PATCH 296/342] Fix references to Dotted Identifiers section --- docs/syntax.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/syntax.rst b/docs/syntax.rst index 6b051878d..b7f84436a 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -131,7 +131,7 @@ names, but any nonempty sequence of characters that aren't ASCII whitespace nor one of the following: ``()[]{};"'`~``. The reader will attempt to read each identifier as a :ref:`numeric literal `, then attempt to read it as a :ref:`keyword ` if that fails, then attempt to read it as a -:ref:`dotted identifier ` if that fails, then fall back on +:ref:`dotted identifier ` if that fails, then fall back on reading it as a :ref:`symbol ` if that fails. .. _numeric-literals: @@ -395,7 +395,7 @@ first element. of unknown length with a real addition expression. - If it is itself an expression of the form ``(. None …)`` (typically produced - with a :ref:`dotted identifier ` like ``.add``), it's used + with a :ref:`dotted identifier ` like ``.add``), it's used to construct a method call with the element after ``None`` as the object: thus, ``(.add my-set 5)`` is equivalent to ``((. my-set add) 5)``, which becomes ``my_set.add(5)`` in Python. From d5c5a4e4c66f10ad2de9e7ad403110b46d1fdbcc Mon Sep 17 00:00:00 2001 From: wrobell Date: Fri, 15 Sep 2023 10:37:45 +0100 Subject: [PATCH 297/342] Fix comments in the index and why Hy documents This is to avoid the following warnings when building the documentation Explicit markup ends without a blank line; unexpected unindent. --- docs/index.rst | 4 ++-- docs/whyhy.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 3255ce8dd..c33294927 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,8 +14,8 @@ Hy is a Lisp dialect that's embedded in Python. Since Hy transforms its Lisp code into Python abstract syntax tree (AST) objects, you have the whole beautiful world of Python at your fingertips, in Lisp form. -.. - Changes to this paragraph should be mirrored on Hy's homepage. +.. Changes to this paragraph should be mirrored on Hy's homepage. + To install the latest release of Hy, just use the command ``pip3 install --user hy``. Then you can start an interactive read-eval-print loop (REPL) with the command ``hy``, or run a Hy program with ``hy myprogram.hy``. diff --git a/docs/whyhy.rst b/docs/whyhy.rst index 037844b76..b4662ece9 100644 --- a/docs/whyhy.rst +++ b/docs/whyhy.rst @@ -2,8 +2,8 @@ Why Hy? ======= -.. - Changes to this paragraph should be mirrored on Hy's homepage. +.. Changes to this paragraph should be mirrored on Hy's homepage. + Hy (or "Hylang" for long; named after the insect order Hymenoptera, since Paul Tagliamonte was studying swarm behavior when he created the language) is a multi-paradigm general-purpose programming language in From c05910cdb7f1facbe0532c9faec67545a07bc9bf Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Thu, 28 Sep 2023 14:19:23 +0200 Subject: [PATCH 298/342] Optimize short-circuiting operators, fixes #854 --- hy/core/result_macros.py | 84 ++++++++++++++++++-------------- tests/compilers/test_compiler.py | 18 +++++++ 2 files changed, 66 insertions(+), 36 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 81c24ffec..fa4ff588d 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -259,45 +259,57 @@ def compile_logical_or_and_and_operator(compiler, expr, operator, args): osym = expr[0] if len(args) == 0: return asty.Constant(osym, value=default) - elif len(args) == 1: - return compiler.compile(args[0]) + ret = Result() values = list(map(compiler.compile, args)) - if any(value.stmts for value in values): - # Compile it to an if...else sequence - var = compiler.get_anon_var() - name = asty.Name(osym, id=var, ctx=ast.Store()) - expr_name = asty.Name(osym, id=var, ctx=ast.Load()) - temp_variables = [name, expr_name] - - def make_assign(value, node=None): - positioned_name = asty.Name(node or osym, id=var, ctx=ast.Store()) - temp_variables.append(positioned_name) - return asty.Assign(node or osym, targets=[positioned_name], value=value) - - current = root = [] - for i, value in enumerate(values): - if value.stmts: - node = value.stmts[0] - current.extend(value.stmts) + + + # first + ret += values[0] + cur = ret.stmts + boolop = None + + var = None + def put(node, value): + nonlocal var + if var is None: + var = compiler.get_anon_var() + name = asty.Name(node, id=var, ctx=ast.Store()) + ret.temp_variables.append(name) + return asty.Assign(node, targets=[name], value=value) + + def get(node): + if var is None: + cur.append(put(node, ret.force_expr)) + name = asty.Name(node, id=var, ctx=ast.Load()) + ret.temp_variables.append(name) + return name + + # rest + for value in values[1:]: + if value.stmts: + node = value.stmts[0] + cond = get(node) + if operator == "or": + cond = asty.UnaryOp(node, op=ast.Not(), operand=cond) + branch = asty.If(node, test=cond, body=value.stmts, orelse=[]) + cur.append(branch) + cur = branch.body + cur.append(boolop := put(node, value.force_expr)) + else: + val = value.force_expr + def enbool(expr, val): + if isinstance(expr, ast.BoolOp): + expr.values.append(val) + else: + expr = asty.BoolOp(expr, op=opnode(), values=[expr, val]) + return expr + if boolop is not None: + boolop.value = enbool(boolop.value, val) else: - node = value.expr - current.append(make_assign(value.force_expr, value.force_expr)) - if i == len(values) - 1: - # Skip a redundant 'if'. - break - if operator == "and": - cond = expr_name - elif operator == "or": - cond = asty.UnaryOp(node, op=ast.Not(), operand=expr_name) - current.append(asty.If(node, test=cond, body=[], orelse=[])) - current = current[-1].body - ret = sum(root, ret) - ret += Result(expr=expr_name, temp_variables=temp_variables) - else: - ret += asty.BoolOp( - osym, op=opnode(), values=[value.force_expr for value in values] - ) + ret.expr = enbool(ret.expr or osym, val) + if var is not None: + ret.expr = get(osym) return ret diff --git a/tests/compilers/test_compiler.py b/tests/compilers/test_compiler.py index 64fd2a01e..4269fe0d5 100644 --- a/tests/compilers/test_compiler.py +++ b/tests/compilers/test_compiler.py @@ -3,6 +3,8 @@ from hy import compiler from hy.models import Expression, Integer, List, Symbol +from hy.reader import read_many +from hy.compiler import hy_compile def make_expression(*args): @@ -14,6 +16,10 @@ def make_expression(*args): return h.replace(h) +def hy2py(s): + return ast.unparse(hy_compile(read_many(s), "test", import_stdlib=False)) + + def test_compiler_bare_names(): """ Check that the compiler doesn't drop bare names from code branches @@ -58,3 +64,15 @@ def test_compiler_yield_return(): assert isinstance(body[0].value, ast.Yield) assert isinstance(body[1], ast.Return) assert isinstance(body[1].value, ast.BinOp) + + +# https://github.com/hylang/hy/issues/854 +def test_compact_logic(): + """ + Check that we don't generate multiple unnecessary if-statements + when compiling the short-circuiting operators. + """ + py = hy2py("(and 1 2 3 (do (setv x 4) x) 5 6)") + assert py.count("if") == 1 + py = hy2py("(or 1 2 3 (do (setv x 4) x) 5 6 (do (setv y 7)))") + assert py.count("if") == 2 From 563a1228332ef6e05a9f0981decc05bc857b0831 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Oct 2023 10:55:51 -0400 Subject: [PATCH 299/342] Clean up `compile_logical_or_and_and_operator` --- hy/core/result_macros.py | 68 ++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index fa4ff588d..9ed6c65e8 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -256,60 +256,66 @@ def compile_unary_operator(compiler, expr, root, arg): def compile_logical_or_and_and_operator(compiler, expr, operator, args): ops = {"and": (ast.And, True), "or": (ast.Or, None)} opnode, default = ops[operator] - osym = expr[0] if len(args) == 0: - return asty.Constant(osym, value=default) + return asty.Constant(expr[0], value=default) - ret = Result() - values = list(map(compiler.compile, args)) - - - # first - ret += values[0] - cur = ret.stmts - boolop = None + ret = None + var = None # A temporary variable for assigning results to + assignment = None # The current assignment to `var` + stmts = None # The current statement list - var = None def put(node, value): - nonlocal var + # Save the result of the operation so far to `var`. + nonlocal var, assignment if var is None: var = compiler.get_anon_var() name = asty.Name(node, id=var, ctx=ast.Store()) ret.temp_variables.append(name) - return asty.Assign(node, targets=[name], value=value) + return (assignment := asty.Assign(node, targets=[name], value=value)) def get(node): + # Get the value of `var`, creating it if necessary. if var is None: - cur.append(put(node, ret.force_expr)) + stmts.append(put(node, ret.force_expr)) name = asty.Name(node, id=var, ctx=ast.Load()) ret.temp_variables.append(name) return name - # rest - for value in values[1:]: - if value.stmts: + for value in map(compiler.compile, args): + if ret is None: + # This is the first iteration. Don't actually introduce a + # `BoolOp` yet; the unary case doesn't need it. + ret = value + stmts = ret.stmts + elif value.stmts: + # Save the result of the statements to the temporary + # variable. Use an `if` statement to implement + # short-circuiting from this point. node = value.stmts[0] cond = get(node) if operator == "or": + # Negate the conditional. cond = asty.UnaryOp(node, op=ast.Not(), operand=cond) branch = asty.If(node, test=cond, body=value.stmts, orelse=[]) - cur.append(branch) - cur = branch.body - cur.append(boolop := put(node, value.force_expr)) + stmts.append(branch) + stmts = branch.body + stmts.append(put(node, value.force_expr)) else: - val = value.force_expr - def enbool(expr, val): + # Add this value to the current `BoolOp`, or create a new + # one if we don't have one. + value = value.force_expr + def enbool(expr): if isinstance(expr, ast.BoolOp): - expr.values.append(val) - else: - expr = asty.BoolOp(expr, op=opnode(), values=[expr, val]) - return expr - if boolop is not None: - boolop.value = enbool(boolop.value, val) + expr.values.append(value) + return expr + return asty.BoolOp(expr, op=opnode(), values=[expr, value]) + if assignment: + assignment.value = enbool(assignment.value) else: - ret.expr = enbool(ret.expr or osym, val) - if var is not None: - ret.expr = get(osym) + ret.expr = enbool(ret.expr) + + if var: + ret.expr = get(expr) return ret From 293655fde16d5919b9a16f52764416d9b688fa66 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Oct 2023 11:09:54 -0400 Subject: [PATCH 300/342] Test the avoidance of nested `BoolOp`s --- tests/compilers/test_ast.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 27cf35053..001c2e55f 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -449,6 +449,15 @@ def test_literal_newlines(): assert s("#[[\rhello\rworld]]") == "hello\nworld" +def test_linear_boolop(): + """Operations like `(and 1 2 3)` should use only one `BoolOp`, + instead of an equivalent nested sequence of `BoolOp`s.""" + for op in ("and", "or"): + node = can_compile(f'({op} 1 2.0 True "hi" 5)').body[0].value + assert len(node.values) == 5 + assert all(isinstance(v, ast.Constant) for v in node.values) + + def test_compile_error(): """Ensure we get compile error in tricky cases""" with pytest.raises(HyLanguageError) as excinfo: From 42554b2c142314db474af3fb2d52e992f52d9fa6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 1 Oct 2023 11:01:12 -0400 Subject: [PATCH 301/342] Update NEWS --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index 4b3e70821..969f8aad1 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -34,6 +34,7 @@ Misc. Improvements ------------------------------ * Some syntax errors raised by core macros now have more informative messages. +* Logical operators now compile to simpler Python code in some cases. Bug Fixes ------------------------------ From fcbfe286e214c91072a150c07a538151ad225da2 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 2 Oct 2023 14:49:46 -0400 Subject: [PATCH 302/342] Lots of small improvements to the manual --- docs/cli.rst | 8 ++++---- docs/index.rst | 3 ++- docs/interop.rst | 15 +++++---------- docs/repl.rst | 2 +- docs/semantics.rst | 4 ++-- docs/syntax.rst | 38 ++++++++++++++++---------------------- docs/tutorial.rst | 22 ++++++++++++++-------- docs/whyhy.rst | 41 +++++++++++++++++++++++++---------------- hy/models.py | 2 +- 9 files changed, 70 insertions(+), 65 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index d552c59ba..10bc1756b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -35,7 +35,7 @@ for a complete list of options and :py:ref:`Python's documentation .. cmdoption:: --repl-output-fn Set the :ref:`REPL output function `. This can be the - name of a Python builtin, mostly likely ``repr``, or a dotted name like + name of a Python builtin, most likely ``repr``, or a dotted name like ``foo.bar.baz``. In the latter case, Hy will attempt to import the named object with code like ``(import foo.bar [baz])``. @@ -45,10 +45,10 @@ for a complete list of options and :py:ref:`Python's documentation hy2py ----- -``hy2py`` is a program to convert Hy source code into Python source code. Use ``hy2py --help`` for usage instructions. It can take its input from standard input, or from a file or module name provided as a command-line argument. In the case of a module name, the current working directory should be the parent directory of that module, and the output parameter (``--output/-o``) is required. When the output parameter is provided, the output will be written into the folder or file. Otherwise, the result is written to standard output. +``hy2py`` is a program to convert Hy source code into Python source code. Use ``hy2py --help`` for usage instructions. It can take its input from standard input, or from a file or module name provided as a command-line argument. In the case of a module name, the current working directory should be the parent directory of that module, and the output parameter (``--output/-o``) is required. When the output parameter is provided, the output will be written into the given folder or file. Otherwise, the result is written to standard output. .. warning:: - ``hy2py`` can execute arbitrary code. Don't give it untrusted input. + ``hy2py`` can execute arbitrary code (via macros, :hy:func:`eval-when-compile`, etc.). Don't give it untrusted input. @@ -60,4 +60,4 @@ hyc ``hyc`` is a program to compile files of Hy code into Python bytecode. Use ``hyc --help`` for usage instructions. The generated bytecode files are named and placed according to the usual scheme of your Python executable, as indicated by :py:func:`importlib.util.cache_from_source`. .. warning:: - ``hyc`` can execute arbitrary code. Don't give it untrusted input. + ``hyc`` can execute arbitrary code (via macros, :hy:func:`eval-when-compile`, etc.). Don't give it untrusted input. diff --git a/docs/index.rst b/docs/index.rst index c33294927..b6b855378 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,7 +14,8 @@ Hy is a Lisp dialect that's embedded in Python. Since Hy transforms its Lisp code into Python abstract syntax tree (AST) objects, you have the whole beautiful world of Python at your fingertips, in Lisp form. -.. Changes to this paragraph should be mirrored on Hy's homepage. +.. Changes to the below paragraph should be mirrored on Hy's homepage + and the README. To install the latest release of Hy, just use the command ``pip3 install --user hy``. Then you can start an interactive read-eval-print loop (REPL) with diff --git a/docs/interop.rst b/docs/interop.rst index cf0d8a4a4..8c3b813f7 100644 --- a/docs/interop.rst +++ b/docs/interop.rst @@ -4,20 +4,17 @@ Python Interoperability ======================= -Despite being a Lisp, Hy aims to be fully compatible with Python. That means -every Python module or package can be imported in Hy code, and vice versa. - :ref:`Mangling ` allows variable names to be spelled differently in Hy and Python. For example, Python's ``str.format_map`` can be written ``str.format-map`` in Hy, and a Hy function named ``valid?`` would be called -``is_valid`` in Python. You can call :hy:func:`hy.mangle` and +``hyx_valid_Xquestion_markX`` in Python. You can call :hy:func:`hy.mangle` and :hy:func:`hy.unmangle` from either language. Using Python from Hy ==================== -To use a Python module from Hy, just :hy:func:`import` it. No additional -ceremony is required. +To use a Python module from Hy, just :hy:func:`import` it. In most cases, no +additional ceremony is required. You can embed Python code directly into a Hy program with the macros :hy:func:`py ` and :hy:func:`pys `, and you can use standard Python @@ -43,7 +40,7 @@ still import ``hy``, and thus require Hy to be installed in order to run; see :ref:`implicit-names` for details and workarounds. To execute Hy code from a string, use :hy:func:`hy.read-many` to convert it to -:ref:`models ` and then :hy:func:`hy.eval` to evaluate it: +:ref:`models ` and :hy:func:`hy.eval` to evaluate it: .. code-block:: python @@ -59,9 +56,7 @@ You can use :meth:`hy.REPL.run` to launch the Hy REPL from Python, as in Libraries that expect Python ============================ -There are various means by which Hy may interact poorly with a Python library - -because the library doesn't account for the possibility of Hy. For example, +There are various means by which Hy may interact poorly with a Python library because the library doesn't account for the possibility of Hy. For example, when you run :ref:`hy-cli`, ``sys.executable`` will be set to this program rather than the original Python binary. This is helpful more often than not, but will lead to trouble if e.g. the library tries to call diff --git a/docs/repl.rst b/docs/repl.rst index 838f169fd..67af1540b 100644 --- a/docs/repl.rst +++ b/docs/repl.rst @@ -26,7 +26,7 @@ Output functions By default, the return value of each REPL input is printed with :hy:func:`hy.repr`. To change this, you can set the REPL output function with e.g. the command-line argument ``--repl-output-fn``. Use :func:`repr` to get -Python representations like Python's own REPL. +Python representations, like Python's own REPL. Regardless of the output function, no output is produced when the value is ``None``, as in Python. diff --git a/docs/semantics.rst b/docs/semantics.rst index f47cca68b..626c091d9 100644 --- a/docs/semantics.rst +++ b/docs/semantics.rst @@ -49,10 +49,10 @@ called first, call it before ``f``. When bytecode is regenerated ---------------------------- -The first time Hy is asked to execute a file, it will produce a bytecode file +The first time Hy is asked to execute a file, whether directly or indirectly (as in the case of an import), it will produce a bytecode file (unless :std:envvar:`PYTHONDONTWRITEBYTECODE` is set). Subsequently, if the source file hasn't changed, Hy will load the bytecode instead of recompiling -the source. Python behaves similarly, but the difference between recompilation +the source. Python also makes bytecode files, but the difference between recompilation and loading bytecode is more consequential in Hy because of how Hy lets you run and generate code at compile-time with things like macros, reader macros, and :hy:func:`eval-and-compile`. You may be surprised by behavior like the diff --git a/docs/syntax.rst b/docs/syntax.rst index b7f84436a..1a55eb7a2 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -26,12 +26,12 @@ somewhat different (such as :class:`Set `, which is ordered, unlike actual :class:`set`\s). All models inherit from :class:`Object `, which stores textual position information, so tracebacks can point to the right place in the code. The compiler takes whatever models -are left over after parsing and macro expansion and translates them into Python +are left over after parsing and macro-expansion and translates them into Python :mod:`ast` nodes (e.g., :class:`Integer ` becomes :class:`ast.Constant`), which can then be evaluated or rendered as Python code. Macros (that is, regular macros, as opposed to reader macros) operate on the model level, taking some models as arguments and returning more models for -compilation or further macro expansion; they're free to do quite different +compilation or further macro-expansion; they're free to do quite different things with a given model than the compiler does, if it pleases them to, like using an :class:`Integer ` to construct a :class:`Symbol `. @@ -177,8 +177,8 @@ Literal keywords are most often used for their special treatment in as values. For example, ``(f :foo 3)`` calls the function ``f`` with the parameter ``foo`` set to ``3``. The keyword is also :ref:`mangled ` at compile-time. To prevent a literal keyword from being treated specially in -an expression, you can :hy:func:`quote` the keyword, or you can use it itself -as a keyword argument, as in ``(f :foo :bar)``. +an expression, you can :hy:func:`quote` the keyword, or you can use it as the +value for another keyword argument, as in ``(f :foo :bar)``. Otherwise, keywords are simple model objects that evaluate to themselves. Users of other Lisps should note that it's often a better idea to use a string than a @@ -312,7 +312,7 @@ Hy allows double-quoted strings (e.g., ``"hello"``), but not single-quoted strings like Python. The single-quote character ``'`` is reserved for preventing the evaluation of a form, (e.g., ``'(+ 1 1)``), as in most Lisps (see :ref:`more-sugar`). Python's so-called triple-quoted strings (e.g., -``'''hello'''`` and ``"""hello"""``) aren't supported. However, in Hy, unlike +``'''hello'''`` and ``"""hello"""``) aren't supported, either. However, in Hy, unlike Python, any string literal can contain newlines; furthermore, Hy has :ref:`bracket strings `. For consistency with Python's triple-quoted strings, all literal newlines in literal strings are read as in @@ -324,8 +324,8 @@ Unrecognized escape sequences are a syntax error. To create a "raw string" that interprets all backslashes literally, prefix the string with ``r``, as in ``r"slash\not"``. -Like Python, Hy treats all string literals as sequences of Unicode characters -by default. The result is the model type :class:`String `. +By default, all string literals are regarded as sequences of Unicode characters. +The result is the model type :class:`String `. You may prefix a string literal with ``b`` to treat it as a sequence of bytes, producing :class:`Bytes ` instead. @@ -351,10 +351,10 @@ like the here-documents of other languages. A bracket string begins with begins with ``f-``, the bracket string is interpreted as an :ref:`f-string `.) For example:: - => (print #[["That's very kind of yuo [sic]" Tom wrote back.]]) - "That's very kind of yuo [sic]" Tom wrote back. - => (print #[==[1 + 1 = 2]==]) - 1 + 1 = 2 + (print #[["That's very kind of yuo [sic]" Tom wrote back.]]) + ; "That's very kind of yuo [sic]" Tom wrote back. + (print #[==[1 + 1 = 2]==]) + ; 1 + 1 = 2 Bracket strings are always raw Unicode strings, and don't allow the ``r`` or ``b`` prefixes. @@ -451,12 +451,9 @@ A format string (or "f-string", or "formatted string literal") is a string literal with embedded code, possibly accompanied by formatting commands. The result is an :class:`FString `, Hy f-strings work much like :ref:`Python f-strings ` except that the embedded code is in Hy -rather than Python. +rather than Python. :: -:: - - => (print f"The sum is {(+ 1 1)}.") - The sum is 2. + (print f"The sum is {(+ 1 1)}.") ; => The sum is 2. Since ``=``, ``!``, and ``:`` are identifier characters in Hy, Hy decides where the code in a replacement field ends (and any debugging ``=``, conversion @@ -464,12 +461,9 @@ specifier, or format specifier begins) by parsing exactly one form. You can use ``do`` to combine several forms into one, as usual. Whitespace may be necessary to terminate the form:: - => (setv foo "a") - => (print f"{foo:x<5}") - … - NameError: name 'hyx_fooXcolonXxXlessHthan_signX5' is not defined - => (print f"{foo :x<5}") - axxxx + (setv foo "a") + (print f"{foo:x<5}") ; => NameError: name 'hyx_fooXcolonXxXlessHthan_signX5' is not defined + (print f"{foo :x<5}") ; => axxxx Unlike Python, whitespace is allowed between a conversion and a format specifier. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index b20e69b98..929304731 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -65,11 +65,13 @@ the **form**. ``92``, ``*``, and ``(* 92 2)`` are all forms. A Lisp program consists of a sequence of forms nested within forms. Forms are typically separated from each other by whitespace, but some forms, such as string literals (``"Hy, world!"``), can contain whitespace themselves. An -**expression** is a form enclosed in parentheses; its first child form, called -the **head**, determines what the expression does, and should generally be a -function or macro. Functions are the most ordinary sort of head, whereas macros -(described in more detail below) are functions executed at compile-time instead -and return code to be executed at run-time. +:ref:`expression ` is a form enclosed in parentheses; its first +child form, called the **head**, determines what the expression does, and +should generally be a function or macro. :py:term:`Functions `, the +most ordinary sort of head, constitute reusable pieces of code that can take in +arguments and return a value. Macros (described in more detail :ref:`below +`) are a special kind of function that's executed at +compile-time and returns code to be executed at run-time. Comments start with a ``;`` character and continue till the end of the line. A comment is functionally equivalent to whitespace. :: @@ -260,6 +262,8 @@ Python can import a Hy module like any other module so long as Hy itself has been imported first, which, of course, must have already happened if you're running a Hy program. +.. _tutorial-macros: + Macros ====== @@ -277,7 +281,9 @@ program. Here's a simple example:: (print "Value:" (m)) (print "Done executing") -If you run this program twice in a row, you'll see this:: +If you run this program twice in a row, you'll see this: + +.. code-block:: text $ hy example.hy Now for a slow computation @@ -300,8 +306,8 @@ simply:: (print "Value:" 1) (print "Done executing") -Our macro ``m`` has an especially simple return value, an integer, which at -compile-time is converted to an integer literal. In general, macros can return +Our macro ``m`` has an especially simple return value, an integer (:py:class:`int`), which at +compile-time is converted to an integer model (:class:`hy.models.Integer`). In general, macros can return arbitrary Hy forms to be executed as code. There are several helper macros that make it easy to construct forms programmatically, such as :hy:func:`quote` (``'``), :hy:func:`quasiquote` (`````), :hy:func:`unquote` (``~``), diff --git a/docs/whyhy.rst b/docs/whyhy.rst index b4662ece9..e2c8f50da 100644 --- a/docs/whyhy.rst +++ b/docs/whyhy.rst @@ -2,7 +2,7 @@ Why Hy? ======= -.. Changes to this paragraph should be mirrored on Hy's homepage. +.. Changes to the below paragraph should be mirrored on Hy's homepage. Hy (or "Hylang" for long; named after the insect order Hymenoptera, since Paul Tagliamonte was studying swarm behavior when he created the @@ -22,19 +22,27 @@ Hy versus Python The first thing a Python programmer will notice about Hy is that it has Lisp's traditional parenthesis-heavy prefix syntax in place of Python's C-like infix -syntax. For example, ``print("The answer is", 2 + object.method(arg))`` could -be written ``(print "The answer is" (+ 2 (.method object arg)))`` in Hy. -Consequently, Hy is free-form: structure is indicated by punctuation rather +syntax. For example, + +.. code-block:: python + + print("The answer is", 2 + object.method(arg)) + +could be written :: + + (print "The answer is" (+ 2 (.method object arg))) + +in Hy. Consequently, Hy is free-form: structure is indicated by punctuation rather than whitespace, making it convenient for command-line use. As in other Lisps, the value of a simplistic syntax is that it facilitates Lisp's signature feature: `metaprogramming -`_ through macros, which are -functions that manipulate code objects at compile time to produce new code -objects, which are then executed as if they had been part of the original code. -In fact, Hy allows arbitrary computation at compile-time. For example, here's a -simple macro that implements a C-style do-while loop, which executes its body -for as long as the condition is true, but at least once. +`_ through :doc:`macros +`, which are functions that manipulate code objects at compile time to +produce new code objects, which are then executed as if they had been part of +the original code. In fact, Hy allows arbitrary computation at compile-time. For +example, here's a simple macro that implements a C-style do-while loop, which +executes its body for as long as the condition is true, but at least once. .. _do-while: @@ -52,7 +60,7 @@ for as long as the condition is true, but at least once. Hy also removes Python's restrictions on mixing expressions and statements, allowing for more direct and functional code. For example, Python doesn't allow -:ref:`with ` blocks, which close a resource once you're done using it, +:keyword:`with` blocks, which close a resource once you're done using it, to return values. They can only execute a set of statements: .. code-block:: python @@ -70,7 +78,7 @@ it like an ordinary function call:: (len (with [o (open "foo")] (.read o))) (len (with [o (open "bar")] (.read o))))) -To be even more concise, you can put a ``with`` form in a :hy:func:`gfor `:: +To be even more concise, you can put a ``with`` form in a :hy:func:`gfor`:: (print (sum (gfor filename ["foo" "bar"] @@ -81,9 +89,9 @@ Operators can be given more than two arguments (e.g., ``(+ 1 2 3)``), including augmented assignment operators (e.g., ``(+= x 1 2 3)``). They are also provided as ordinary first-class functions of the same name, allowing them to be passed to higher-order functions: ``(sum xs)`` could be written ``(reduce + xs)``, -after importing the function ``+`` from the module ``hy.pyops``. +after importing the function ``+`` from the module :hy:mod:`hy.pyops`. -The Hy compiler works by reading Hy source code into Hy model objects and +The Hy compiler works by reading Hy source code into Hy :ref:`model objects ` and compiling the Hy model objects into Python abstract syntax tree (:py:mod:`ast`) objects. Python AST objects can then be compiled and run by Python itself, byte-compiled for faster execution later, or rendered into Python source code. @@ -131,8 +139,9 @@ in a set literal. However, models can be concatenated and indexed just like plain lists, and you can return ordinary Python types from a macro or give them to :hy:func:`hy.eval` and Hy will automatically promote them to models. -Hy takes much of its semantics from Python. For example, Hy is a Lisp-1 because -Python functions use the same namespace as objects that aren't functions. In +Hy takes much of its semantics from Python. For example, functions use the same +namespace as objects that aren't functions, so a variable named ``globals`` +can shadow the Python built-in function :py:func:`globals`. In general, any Python code should be possible to literally translate to Hy. At the same time, Hy goes to some lengths to allow you to do typical Lisp things that aren't straightforward in Python. For example, Hy provides the diff --git a/hy/models.py b/hy/models.py index 63f8a529c..3ce3a74ed 100644 --- a/hy/models.py +++ b/hy/models.py @@ -537,7 +537,7 @@ class Dict(Sequence): Represents a literal :class:`dict`. ``keys``, ``values``, and ``items`` methods are provided, each returning a list, although this model type does none of the normalization of a real :class:`dict`. In the case of an odd number of child models, - ``keys`` returns the last child whereas ``values`` and ``items`` ignores it. + ``keys`` returns the last child whereas ``values`` and ``items`` ignore it. """ def _pretty_str(self): From 9998e516038ec8a1d9a43e944556a7c89ff5024a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 30 Sep 2023 10:23:57 -0400 Subject: [PATCH 303/342] Test some operations on `_hy_macros` --- tests/native_tests/macros_first_class.hy | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/native_tests/macros_first_class.hy diff --git a/tests/native_tests/macros_first_class.hy b/tests/native_tests/macros_first_class.hy new file mode 100644 index 000000000..d16f9d32c --- /dev/null +++ b/tests/native_tests/macros_first_class.hy @@ -0,0 +1,60 @@ +"Tests of using macros as first-class objects: listing, creating, and +deleting them, and retrieving their docstrings." + +(import + builtins) + + +(defn test-builtins [] + (assert (in "when" (.keys builtins._hy_macros))) + (assert (not-in "global1" (.keys builtins._hy_macros))) + (assert (not-in "nonexistent" (.keys builtins._hy_macros))) + + (setv s (. builtins _hy_macros ["when"] __doc__)) + (assert s) + (assert (is (type s) str))) + + +; There are three ways to define a global macro: +; 1. `defmacro` in global scope +(defmacro global1 [] + "global1 docstring" + "from global1") +; 2. `require` in global scope +(require tests.resources.tlib [qplah :as global2]) +; 3. Manually updating `_hy_macros` +(eval-and-compile (setv (get _hy_macros "global3") (fn [&compiler] + "from global3"))) +(eval-and-compile (setv (get _hy_macros (hy.mangle "global☘")) (fn [&compiler] + "global☘ docstring" + "from global☘"))) + +(defn test-globals [] + (assert (not-in "when" (.keys _hy_macros))) + (assert (not-in "nonexistent" (.keys _hy_macros))) + (assert (all (gfor + k ["global1" "global2" "global3" "global☘"] + (in (hy.mangle k) (.keys _hy_macros))))) + (assert (= (global3) "from global3")) + (assert (= (global☘) "from global☘")) + (assert (= + (. _hy_macros ["global1"] __doc__) + "global1 docstring")) + ; https://github.com/hylang/hy/issues/1946 + (assert (= + (. _hy_macros [(hy.mangle "global☘")] __doc__) + "global☘ docstring"))) + +; Try creating and and then deleting a global macro. +(defn global4 [] + "from global4 function") +(setv global4-f1 (global4)) ; Calls the function +(eval-and-compile (setv (get _hy_macros "global4") (fn [&compiler] + "from global4 macro"))) +(setv global4-m (global4)) ; Calls the macro +(eval-and-compile (del (get _hy_macros "global4"))) +(setv global4-f2 (global4)) ; Calls the function again + +(defn test-global-delete [] + (assert (= (global4) global4-f1 global4-f2 "from global4 function")) + (assert (= global4-m "from global4 macro"))) From bbbc365880d84fbd974b3ee514dece780a1e0caf Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 30 Sep 2023 09:56:38 -0400 Subject: [PATCH 304/342] Remove `delmacro` --- NEWS.rst | 7 ++++++- hy/core/macros.hy | 28 ---------------------------- tests/native_tests/macros.hy | 19 ------------------- 3 files changed, 6 insertions(+), 48 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 969f8aad1..bff1a9e65 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,13 +3,18 @@ Unreleased ============================= +Removals +------------------------------ +* `delmacro` has been removed. Use `(del (get _hy_macros (hy.mangle + …)))` instead. + Breaking Changes ------------------------------ * `defmacro` and `require` can now define macros locally instead of only module-wide. - * `hy.eval`, `hy.macroexpand`, `doc`, and `delmacro` don't work with + * `hy.eval`, `hy.macroexpand`, and `doc` don't work with local macros (yet). * When a macro is `require`\d from another module, that module is no diff --git a/hy/core/macros.hy b/hy/core/macros.hy index e02dd7e0f..2e5ba90da 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -166,31 +166,3 @@ (if (isinstance a hy.models.List) (lfor x a (hy.models.String x)) (raise (TypeError "arguments must be keywords or lists of symbols")))))))) - -(defmacro delmacro - [#* names] - #[[Delete a macro(s) from the current module. This doesn't work on a - local macro. - :: - - => (require a-module [some-macro]) - => (some-macro) - 1 - - => (delmacro some-macro) - => (some-macro) - Traceback (most recent call last): - File "", line 1, in - (some-macro) - NameError: name 'some_macro' is not defined - - => (delmacro some-macro) - Traceback (most recent call last): - File "", line 1, in - (delmacro some-macro) - NameError: macro 'some-macro' is not defined - ]] - (let [sym (hy.gensym)] - `(eval-and-compile - (for [~sym ~(lfor name names (hy.mangle name))] - (when (in ~sym _hy_macros) (del (get _hy_macros ~sym))))))) diff --git a/tests/native_tests/macros.hy b/tests/native_tests/macros.hy index 9156d5972..6490cb707 100644 --- a/tests/native_tests/macros.hy +++ b/tests/native_tests/macros.hy @@ -168,25 +168,6 @@ (assert (in "HyWrapperError" (str excinfo.value)))) -(defmacro delete-me [] "world") - -(defn test-delmacro - [] - ;; test deletion of user defined macro - (delmacro delete-me) - (with [exc (pytest.raises NameError)] - (delete-me)) - ;; test deletion of required macros - (require tests.resources.tlib [qplah parald]) - (assert (and (qplah 1) (parald 1))) - - (delmacro qplah parald) - (with [exc (pytest.raises NameError)] - (hy.eval '(qplah))) - (with [exc (pytest.raises NameError)] - (hy.eval '(parald)))) - - (defn macro-redefinition-warning-tester [local] (for [should-warn? [True False] head ["defmacro" "require"]] (with [(if should-warn? From 12351e69f89f35d12421fd8b3481ef579fc5075d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 30 Sep 2023 11:15:51 -0400 Subject: [PATCH 305/342] Remove `hy.reserved` --- NEWS.rst | 2 ++ docs/api.rst | 6 ------ hy/reserved.hy | 33 --------------------------------- tests/native_tests/reserved.hy | 18 ------------------ 4 files changed, 2 insertions(+), 57 deletions(-) delete mode 100644 hy/reserved.hy delete mode 100644 tests/native_tests/reserved.hy diff --git a/NEWS.rst b/NEWS.rst index bff1a9e65..c978e459d 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -7,6 +7,8 @@ Removals ------------------------------ * `delmacro` has been removed. Use `(del (get _hy_macros (hy.mangle …)))` instead. +* `hy.reserved` has been removed. Use `(.keys (builtins._hy_macros))` + or Python's built-in `keyword` module instead. Breaking Changes ------------------------------ diff --git a/docs/api.rst b/docs/api.rst index ed191ff97..00530d98a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1405,9 +1405,3 @@ Python Operators .. hy:automodule:: hy.pyops :members: - -Reserved --------- - -.. hy:automodule:: hy.reserved - :members: diff --git a/hy/reserved.hy b/hy/reserved.hy deleted file mode 100644 index 6a6d4d4ff..000000000 --- a/hy/reserved.hy +++ /dev/null @@ -1,33 +0,0 @@ -;;; Get a frozenset of Hy reserved words - -(import sys keyword) - -(setv _cache None) - -(defn macros [] - "Return a frozenset of Hy's core macro names." - (frozenset (map hy.unmangle (+ - (list (.keys hy.core.result_macros._hy_macros)) - (list (.keys hy.core.macros._hy_macros)))))) - -(defn names [] - "Return a frozenset of reserved symbol names. - - The result of the first call is cached. - - The output includes all of Hy's core functions and macros, plus all - Python reserved words. All names are in unmangled form (e.g., - ``not-in`` rather than ``not_in``). - - Examples: - :: - - => (import hy.extra.reserved) - => (in \"defclass\" (hy.extra.reserved.names)) - True - " - (global _cache) - (when (is _cache None) - (setv _cache (| (macros) (frozenset (map hy.unmangle - keyword.kwlist))))) - _cache) diff --git a/tests/native_tests/reserved.hy b/tests/native_tests/reserved.hy deleted file mode 100644 index 3243b7e39..000000000 --- a/tests/native_tests/reserved.hy +++ /dev/null @@ -1,18 +0,0 @@ -(import hy.reserved [macros names]) - -(defn test-reserved-macros [] - (assert (is (type (macros)) frozenset)) - (assert (in "and" (macros))) - (assert (not-in "False" (macros))) - (assert (not-in "pass" (macros)))) - -(defn test-reserved-names [] - (assert (is (type (names)) frozenset)) - (assert (in "and" (names))) - (assert (in "False" (names))) - (assert (in "pass" (names))) - (assert (in "class" (names))) - (assert (in "defclass" (names))) - (assert (in "defmacro" (names))) - (assert (not-in "foo" (names))) - (assert (not-in "hy" (names)))) From 2b30aa0f767cc8eb28e3dd236f4c8fe7d2013afa Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 30 Sep 2023 11:16:05 -0400 Subject: [PATCH 306/342] Add some documentation for operations on macros --- docs/macros.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/macros.rst b/docs/macros.rst index 73e160aef..93db2ff25 100644 --- a/docs/macros.rst +++ b/docs/macros.rst @@ -2,6 +2,23 @@ Macros ====== +Operations on macros +-------------------- + +The currently defined global macros can be accessed as a dictionary ``_hy_macros`` in each module. (Use ``bulitins._hy_macros``, attached to Python's usual :py:mod:`builtins` module, to see core macros.) The keys are mangled macro names and the values are the function objects that implement the macros. You can operate on this dictionary to list, add, delete, or get help on macros, but be sure to use :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile` when you need the effect to happen at compile-time. :: + + (defmacro m [] + "This is a docstring." + `(print "Hello, world.")) + (print (in "m" _hy_macros)) ; => True + (help (get _hy_macros "m")) + (m) ; => "Hello, world." + (eval-and-compile + (del (get _hy_macros "m"))) + (m) ; => NameError + +``_hy_reader_macros`` is a similar dictionary for reader macros, but here, the keys aren't mangled. + .. _using-gensym: Using gensym for Safer Macros From 401e0e86b01a2fdf9f29c406949e6018668d8147 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 4 Oct 2023 13:58:57 -0400 Subject: [PATCH 307/342] Move the helper function `_cond` into `cond` --- hy/core/macros.hy | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 2e5ba90da..1e6818fdf 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -29,17 +29,16 @@ With no arguments, ``cond`` returns ``None``. With an odd number of arguments, ``cond`` raises an error." + (defn _cond [args] + (if args + `(if ~(get args 0) + ~(get args 1) + ~(_cond (cut args 2 None))) + 'None)) (if (% (len args) 2) (raise (TypeError "`cond` needs an even number of arguments")) (_cond args))) -(defn _cond [args] - (if args - `(if ~(get args 0) - ~(get args 1) - ~(_cond (cut args 2 None))) - 'None)) - (defmacro when [test #* body] #[[Shorthand for ``(if test (do …) None)``. See :hy:func:`if`. For a logically negated version, see Hyrule's :hy:func:`unless `. From 195c81edbce517638bc2e6b6a34a7d49ec3f9617 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Fri, 22 Sep 2023 15:16:40 +0200 Subject: [PATCH 308/342] automatically promote `nonlocal` to `global` as necessary --- hy/compiler.py | 4 ++- hy/core/result_macros.py | 11 ++++--- hy/scoping.py | 65 +++++++++++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 15 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index d9f2b2dbf..54bbb5215 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -35,7 +35,7 @@ is_unpack, ) from hy.reader import mangle, HyReader -from hy.scoping import ScopeGlobal +from hy.scoping import ResolveOuterVars, ScopeGlobal hy_ast_compile_flags = 0 @@ -860,6 +860,8 @@ def hy_compile( if not get_expr: result += result.expr_as_stmt() + result.stmts = list(map(ResolveOuterVars().visit, result.stmts)) + body = [] if issubclass(root, ast.Module): diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 9ed6c65e8..671bb3658 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -58,7 +58,7 @@ is_unpack, ) from hy.reader import mangle -from hy.scoping import ScopeFn, ScopeGen, ScopeLet, is_inside_function_scope +from hy.scoping import OuterVar, ScopeFn, ScopeGen, ScopeLet, is_inside_function_scope # ------------------------------------------------ # * Helpers @@ -526,15 +526,18 @@ def compile_global_or_nonlocal(compiler, expr, root, syms): if not syms: return asty.Pass(expr) - node = asty.Global if root == "global" else asty.Nonlocal - ret = node(expr, names=[mangle(s) for s in syms]) + names = [mangle(s) for s in syms] + if root == "global": + ret = asty.Global(expr, names=names) + else: + ret = OuterVar(expr, compiler.scope, names) try: compiler.scope.define_nonlocal(ret, root) except SyntaxError as e: raise compiler._syntax_error(expr, e.msg) - return ret if ret.names else Result() + return ret if syms else Result() @pattern_macro("del", [many(FORM)]) diff --git a/hy/scoping.py b/hy/scoping.py index 488087043..07552f137 100644 --- a/hy/scoping.py +++ b/hy/scoping.py @@ -31,6 +31,45 @@ def nearest_python_scope(scope): return cur +class OuterVar(ast.stmt): + "Custom AST node that can compile to either Nonlocal or Global." + def __init__(self, expr, scope, names): + from hy.compiler import asty + super().__init__() + self.__dict__.update(asty._get_pos(expr)) + self._scope = scope + self.names = names + + +class ResolveOuterVars(ast.NodeTransformer): + "Find all OuterVar nodes and replace with Nonlocal or Global as necessary." + def visit_OuterVar(self, node): + from hy.compiler import asty + scope = node._scope + defined = set() + undefined = list(node.names) # keep order, so can't use set + while undefined and scope.parent: + scope = scope.parent + has = set() + if isinstance(scope, ScopeFn): + has = scope.defined + elif isinstance(scope, ScopeLet): + has = set(scope.bindings.keys()) + elif isinstance(scope, ScopeGlobal): + res = [] + if not scope.defined.issuperset(undefined): + # emit nonlocal, let python raise the error + break + if undefined: + res.append(asty.Global(node, names=list(undefined))) + if defined: + res.append(asty.Nonlocal(node, names=list(defined))) + return res + defined.update(has.intersection(undefined)) + undefined = [name for name in undefined if name not in has] + return [asty.Nonlocal(node, names=node.names)] if node.names else [] + + class NodeRef: """ Wrapper for AST nodes that have symbol names, so that we can rename them if @@ -44,6 +83,7 @@ class NodeRef: ast.Name: "id", ast.Global: "names", ast.Nonlocal: "names", + OuterVar: "names", } if hy._compat.PY3_10: ACCESSOR.update( @@ -96,19 +136,23 @@ class ScopeBase(ABC): def __init__(self, compiler): self.parent = None self.compiler = compiler + self.children = [] def create(self, scope_type, *args): "Create new scope from this one." return scope_type(self.compiler, *args) def __enter__(self): - self.parent = self.compiler.scope + if self.compiler.scope is not self: + self.parent = self.compiler.scope + if self not in self.parent.children: + self.parent.children.append(self) self.compiler.scope = self return self def __exit__(self, *args): - self.compiler.scope = self.parent - self.parent = None + if self.parent: + self.compiler.scope = self.parent return False # Scope interface @@ -254,7 +298,7 @@ def __init__(self, compiler, args=None): "set: of all vars defined in this scope" self.seen = [] "list: of all vars accessedto in this scope" - self.nonlocal_vars = set() + self.nonlocal_vars = {} "set: of all `nonlocal`'s defined in this scope" self.is_fn = args is not None """ @@ -270,8 +314,9 @@ def __init__(self, compiler, args=None): self.define(arg.arg) def __exit__(self, *args): + self.defined.difference_update(self.nonlocal_vars.keys()) for node in self.seen: - if node.name not in self.defined or node.name in self.nonlocal_vars: + if node.name not in self.defined: # pass unbound/nonlocal names up to parent scope self.parent.access(node) return super().__exit__(*args) @@ -291,11 +336,11 @@ def define(self, name): self.defined.add(name) def define_nonlocal(self, node, root): - ( - (self.nonlocal_vars if root == "nonlocal" else self.defined).update( - node.names - ) - ) + if root == "nonlocal": + self.nonlocal_vars.update({name: node for name in node.names}) + else: + self.defined.update(node.names) + for n in self.seen: if n.name in node.names: raise SyntaxError( From 585bec31cdb39c0f24523d752bbc0c3a21fe316e Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Sat, 30 Sep 2023 21:09:50 +0200 Subject: [PATCH 309/342] test nonlocal promotion --- tests/native_tests/let.hy | 14 ++++++++++ tests/native_tests/nonlocal.hy | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 tests/native_tests/nonlocal.hy diff --git a/tests/native_tests/let.hy b/tests/native_tests/let.hy index 591612179..db7890b72 100644 --- a/tests/native_tests/let.hy +++ b/tests/native_tests/let.hy @@ -290,6 +290,20 @@ (assert (= fox 42)))) +(defn test-top-level-let-nonlocal [] + (hy.eval '(do + (let [my-fuel 50] + (defn propulse-me [distance] + (nonlocal my-fuel) + (-= my-fuel distance)) + (defn check-fuel [] + my-fuel)) + (assert (= (check-fuel) 50)) + (propulse-me 3) + (assert (= (check-fuel) 47))) + :globals {})) + + (defn test-let-nested-nonlocal [] (let [fox 42] (defn bar [] diff --git a/tests/native_tests/nonlocal.hy b/tests/native_tests/nonlocal.hy new file mode 100644 index 000000000..4bef411a5 --- /dev/null +++ b/tests/native_tests/nonlocal.hy @@ -0,0 +1,48 @@ +(import pytest) + + +(defn test-nonlocal-promotion [] + (setv G {}) + (hy.eval '(do + (setv home "earth") + (defn blastoff [] + (nonlocal home) + (setv home "saturn")) + (blastoff)) + :globals G) + (assert (= (get G "home") "saturn")) + + (setv health + (hy.eval '(do + (defn make-ration-log [days intensity] + (setv health 20 + ration-log + (list (map (fn [_] + ;; only `rations` should be upgraded + (nonlocal rations health) + (-= rations intensity) + (+= health (* 0.5 intensity)) + rations) + (range days)))) + health) + ;; "late" global binding should still work + (setv rations 100) + (make-ration-log 43 1.5)) + :globals G)) + (assert (= health (+ 20 (* 43 0.5 1.5)))) + (assert (= (get G "rations") (- 100 (* 43 1.5))))) + + +(defn test-nonlocal-must-have-def [] + (with [err (pytest.raises SyntaxError)] + (hy.eval '(do + (defn make-ration-log [days intensity] + (list (map (fn [_] + (nonlocal rations) + (-= rations intensity) + rations) + (range days)))) + ;; oops! I forgot to pack my rations! + (make-ration-log 43 1.5)) + :globals {})) + (assert (in "no binding for nonlocal 'rations'" err.value.msg))) From 754b4944a527a293e301b4ad4c0408b50e57a98c Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Sat, 30 Sep 2023 21:38:26 +0200 Subject: [PATCH 310/342] update NEWS and docs --- NEWS.rst | 2 ++ docs/api.rst | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index c978e459d..5667d9eb8 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -36,6 +36,7 @@ New Features * `HyReader` now has an optional parameter to install existing reader macros from the calling module. * New pragma `warn-on-core-shadow`. +* `nonlocal` now also works for globally defined names. Misc. Improvements ------------------------------ @@ -53,6 +54,7 @@ Bug Fixes install a new reader. * `require` now warns when you shadow a core macro, like `defmacro` already did. +* `nonlocal` now works for top-level `let`-bound names. 0.27.0 (released 2023-07-06) ============================= diff --git a/docs/api.rst b/docs/api.rst index 00530d98a..4b55ad311 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -797,7 +797,22 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. .. hy:macro:: (nonlocal [#* syms]) - As :hy:func:`global`, but the result is a :py:keyword:`nonlocal` statement. + Similar to :hy:func:`global`, but names can be declared in any enclosing + scope. ``nonlocal`` compiles to a :py:keyword:`global` statement for any + names originally defined in the global scope, and a :py:keyword:`nonlocal` + statement for all other names. :: + + (setv a 1 b 1) + (defn f [] + (setv c 10 d 10) + (defn g [] + (nonlocal a c) + (setv a 2 b 2 + c 20 d 20)) + (print a b c d) ; => 1 1 10 10 + (g) + (print a b c d)) ; => 2 1 20 10 + (f) .. hy:macro:: (py [string]) From 6607694657ee03ab6ccb84e21ca3e1a5409931ee Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Sat, 7 Oct 2023 22:51:23 +0200 Subject: [PATCH 311/342] Fix overzealous boolean compaction --- hy/core/result_macros.py | 15 ++++++++++----- tests/compilers/test_compiler.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 671bb3658..fde1d433d 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -260,17 +260,19 @@ def compile_logical_or_and_and_operator(compiler, expr, operator, args): return asty.Constant(expr[0], value=default) ret = None - var = None # A temporary variable for assigning results to - assignment = None # The current assignment to `var` - stmts = None # The current statement list + var = None # A temporary variable for assigning results to + assignment = None # The current assignment to `var` + stmts = None # The current statement list + can_append = False # Whether the current expression is the compiled boolop def put(node, value): # Save the result of the operation so far to `var`. - nonlocal var, assignment + nonlocal var, assignment, can_append if var is None: var = compiler.get_anon_var() name = asty.Name(node, id=var, ctx=ast.Store()) ret.temp_variables.append(name) + can_append = False return (assignment := asty.Assign(node, targets=[name], value=value)) def get(node): @@ -287,6 +289,7 @@ def get(node): # `BoolOp` yet; the unary case doesn't need it. ret = value stmts = ret.stmts + can_append = False elif value.stmts: # Save the result of the statements to the temporary # variable. Use an `if` statement to implement @@ -305,9 +308,11 @@ def get(node): # one if we don't have one. value = value.force_expr def enbool(expr): - if isinstance(expr, ast.BoolOp): + nonlocal can_append + if can_append: expr.values.append(value) return expr + can_append = True return asty.BoolOp(expr, op=opnode(), values=[expr, value]) if assignment: assignment.value = enbool(assignment.value) diff --git a/tests/compilers/test_compiler.py b/tests/compilers/test_compiler.py index 4269fe0d5..38b904417 100644 --- a/tests/compilers/test_compiler.py +++ b/tests/compilers/test_compiler.py @@ -76,3 +76,15 @@ def test_compact_logic(): assert py.count("if") == 1 py = hy2py("(or 1 2 3 (do (setv x 4) x) 5 6 (do (setv y 7)))") assert py.count("if") == 2 + +def test_correct_logic(): + """ + Check that we're not overzealous when compacting boolean operators. + """ + py = hy2py("(or (and 1 2) (and 3 4))") + assert py.count("and") == 2 + assert py.count("or") == 1 + py = hy2py("(and (do (setv x 4) (or x 3)) 5 6)") + assert py.count("x = 4") == 1 + assert py.count("x or 3") == 1 + assert py.count("and") == 2 From 8aa7ce6fa781ab449c57cf4247f9d39f6192c4d8 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Sat, 7 Oct 2023 21:14:52 +0200 Subject: [PATCH 312/342] Make `hy -i ` skip shebang and properly set sys.argv --- NEWS.rst | 1 + hy/cmdline.py | 4 +++- hy/repl.py | 5 ++++- tests/test_bin.py | 15 +++++++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 5667d9eb8..61d9a3fb1 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -55,6 +55,7 @@ Bug Fixes * `require` now warns when you shadow a core macro, like `defmacro` already did. * `nonlocal` now works for top-level `let`-bound names. +* `hy -i` with a filename now skips shebang lines. 0.27.0 (released 2023-07-06) ============================= diff --git a/hy/cmdline.py b/hy/cmdline.py index 1fa129582..a7d748008 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -261,12 +261,14 @@ def proc_opt(opt, arg=None, item=None, i=None): runpy.run_module(hy.mangle(action_arg), run_name="__main__", alter_sys=True) return 0 elif action == "run_script_stdin": + sys.argv = argv if repl: source = sys.stdin filename = 'stdin' else: return run_command(sys.stdin.read(), filename="") elif action == "run_script_file": + sys.argv = argv filename = Path(action_arg) set_path(filename) # Ensure __file__ is set correctly in the code we're about @@ -278,9 +280,9 @@ def proc_opt(opt, arg=None, item=None, i=None): filename = os.path.normpath(filename) if repl: source = Path(filename).read_text() + repl.compile.compiler.skip_next_shebang = True else: try: - sys.argv = argv with filtered_hy_exceptions(): runhy.run_path(str(filename), run_name="__main__") return 0 diff --git a/hy/repl.py b/hy/repl.py index bde468727..47aa14c44 100644 --- a/hy/repl.py +++ b/hy/repl.py @@ -118,6 +118,7 @@ def __init__( self.ast_callback = ast_callback self.hy_compiler = hy_compiler self.reader = HyReader() + self.skip_next_shebang = False super().__init__() @@ -165,8 +166,10 @@ def __call__(self, source, filename="", symbol="single"): self.hy_compiler.filename = name self.hy_compiler.source = source hy_ast = read_many( - source, filename=name, reader=self.reader + source, filename=name, reader=self.reader, + skip_shebang=self.skip_next_shebang, ) + self.skip_next_shebang = False exec_ast, eval_ast = hy_compile( hy_ast, self.module, diff --git a/tests/test_bin.py b/tests/test_bin.py index 6a732c40d..f35177a6a 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -202,6 +202,13 @@ def test_icmd_file(): assert "CUTTLEFISH" in output +def test_icmd_shebang(tmp_path): + (tmp_file := tmp_path / 'icmd_with_shebang.hy').write_text('#!/usr/bin/env hy\n(setv order "Sepiida")') + output, error = run_cmd(["hy", "-i", tmp_file], '(.upper order)') + assert "#!/usr/bin/env" not in error + assert "SEPIIDA" in output + + def test_icmd_and_spy(): output, _ = run_cmd('hy --spy -i -c "(+ [] [])"', "(+ 1 1)") assert "[] + []" in output @@ -227,6 +234,14 @@ def test_file_with_args(): assert "foo" in run_cmd(f"{cmd} -i foo -c bar")[0] +def test_ifile_with_args(): + cmd = "hy -i tests/resources/argparse_ex.hy" + assert "usage" in run_cmd(f"{cmd} -h")[0] + assert "got c" in run_cmd(f"{cmd} -c bar")[0] + assert "foo" in run_cmd(f"{cmd} -i foo")[0] + assert "foo" in run_cmd(f"{cmd} -i foo -c bar")[0] + + def test_hyc(): output, _ = run_cmd("hyc -h") assert "usage" in output From 070f0d50f9f9498b500adca32ce2798331f0b91b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 16 Oct 2023 10:35:14 -0400 Subject: [PATCH 313/342] Fix parsing of `nanJ` etc. --- NEWS.rst | 2 ++ docs/syntax.rst | 3 ++- hy/models.py | 2 +- tests/test_reader.py | 11 +++++++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 61d9a3fb1..d167fe402 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -50,6 +50,8 @@ Bug Fixes * Fixed incomplete recognition of macro calls with a unary dotted head like `((. defn) f [])`. * `~@ #*` now produces a syntax error instead of a nonsensical result. +* Fixed parsing of infinite and NaN imaginary literals with an + uppercase "J". * Fixed `hy.eval` failing on `defreader` or `require` forms that install a new reader. * `require` now warns when you shadow a core macro, like `defmacro` diff --git a/docs/syntax.rst b/docs/syntax.rst index 1a55eb7a2..54188fda5 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -152,7 +152,8 @@ few extensions: interpreted in octal like they do in C. For octal, use the prefix ``0o``, as in Python. - ``NaN``, ``Inf``, and ``-Inf`` are understood as literals. Each produces a - :class:`Float `. + :class:`Float `. These are case-sensitive, unlike other uses + of letters in numeric literals (``1E2``, ``0XFF``, ``5J``, etc.). - Hy allows complex literals as understood by the constructor for :class:`complex`, such as ``5+4j``. (This is also legal Python, but Hy reads it as a single :class:`Complex `, and doesn't otherwise diff --git a/hy/models.py b/hy/models.py index 3ce3a74ed..23ccb6c56 100644 --- a/hy/models.py +++ b/hy/models.py @@ -361,7 +361,7 @@ def __new__(cls, real, imag=0, *args, **kwargs): if isinstance(real, str): value = super().__new__(cls, strip_digit_separators(real)) p1, _, p2 = real.lstrip("+-").replace("-", "+").partition("+") - check_inf_nan_cap(p1, value.imag if "j" in p1 else value.real) + check_inf_nan_cap(p1, value.imag if "j" in p1.lower() else value.real) if p2: check_inf_nan_cap(p2, value.imag) return value diff --git a/tests/test_reader.py b/tests/test_reader.py index 6127a11cc..3ae60daf9 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -225,9 +225,13 @@ def t(x): def f(x): return [Expression([Symbol("foo"), x])] + assert t("2j") == f(Complex(2.0j)) + assert t("2J") == f(Complex(2.0j)) assert t("2.j") == f(Complex(2.0j)) + assert t("2.J") == f(Complex(2.0j)) assert t("-0.5j") == f(Complex(-0.5j)) assert t("1.e7j") == f(Complex(1e7j)) + assert t("1.e7J") == f(Complex(1e7j)) assert t("j") == f(Symbol("j")) assert t("J") == f(Symbol("J")) assert isnan(t("NaNj")[0][1].imag) @@ -236,6 +240,13 @@ def f(x): assert t("Inf-Infj") == f(Complex(complex(float("inf"), float("-inf")))) assert t("Inf-INFj") == f(Symbol("Inf-INFj")) + # https://github.com/hylang/hy/issues/2521 + assert isnan(t("NaNJ")[0][1].imag) + assert t("nanJ") == f(Symbol("nanJ")) + assert t("InfJ") == f(Complex(complex(0, float("inf")))) + assert t("iNfJ") == f(Symbol("iNfJ")) + assert t("Inf-INFJ") == f(Symbol("Inf-INFJ")) + def test_lex_digit_separators(): From d564d7feded77ed5b9dd6e33e8cd2dccfb1299ca Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 17 Oct 2023 11:39:42 -0400 Subject: [PATCH 314/342] Add `get-macro` --- docs/macros.rst | 4 ++-- hy/core/macros.hy | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/macros.rst b/docs/macros.rst index 93db2ff25..3bad50ba9 100644 --- a/docs/macros.rst +++ b/docs/macros.rst @@ -5,13 +5,13 @@ Macros Operations on macros -------------------- -The currently defined global macros can be accessed as a dictionary ``_hy_macros`` in each module. (Use ``bulitins._hy_macros``, attached to Python's usual :py:mod:`builtins` module, to see core macros.) The keys are mangled macro names and the values are the function objects that implement the macros. You can operate on this dictionary to list, add, delete, or get help on macros, but be sure to use :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile` when you need the effect to happen at compile-time. :: +The currently defined global macros can be accessed as a dictionary ``_hy_macros`` in each module. (Use ``bulitins._hy_macros``, attached to Python's usual :py:mod:`builtins` module, to see core macros.) The keys are mangled macro names and the values are the function objects that implement the macros. You can operate on this dictionary to list, add, delete, or get help on macros, but be sure to use :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile` when you need the effect to happen at compile-time. The core macro :hy:func:`get-macro ` provides some syntactic sugar. :: (defmacro m [] "This is a docstring." `(print "Hello, world.")) (print (in "m" _hy_macros)) ; => True - (help (get _hy_macros "m")) + (help (get-macro m)) (m) ; => "Hello, world." (eval-and-compile (del (get _hy_macros "m"))) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 1e6818fdf..ce9db191c 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -135,6 +135,33 @@ (raise (NameError f"macro {~symbol !r} is not defined")))))) +(defmacro get-macro [arg1 [arg2 None]] + "Get the function object used to implement a macro. This works for core macros, global (i.e., module-level) macros, and reader macros, but not local macros (yet). For regular macros, ``get-macro`` is called with one argument, a symbol or string literal, which can be premangled or not according to taste. For reader macros, this argument must be preceded by the literal keyword ``:reader`` (and note that the hash mark, ``#``, is not included in the name of the reader macro). :: + + (get-macro my-macro) + (get-macro :reader my-reader-macro) + + ``get-macro`` expands to a :hy:func:`get ` form on the appropriate object, such as ``_hy_macros``, selected at the time of expanding ``get-macro``. This means you can say ``(del (get-macro …))``, perhaps wrapped in :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile`, to delete a macro, but it's easy to get confused by the order of evaluation and number of evaluations. For more predictable results in complex situations, use ``(del (get …))`` directly instead of ``(del (get-macro …))``." + + (import builtins) + (setv [name namespace] (cond + (= arg1 ':reader) + [(str arg2) "_hy_reader_macros"] + (isinstance arg1 hy.models.Expression) + [(hy.mangle (.join "." (cut arg1 1 None))) "_hy_macros"] + True + [(hy.mangle arg1) "_hy_macros"])) + (cond + (in name (getattr &compiler.module namespace {})) + `(get ~(hy.models.Symbol namespace) ~name) + (in name (getattr builtins namespace {})) + `(get (. hy.M.builtins ~(hy.models.Symbol namespace)) ~name) + True + (raise (NameError (.format "no such {}macro: {!r}" + (if (= namespace "_hy_reader_macros") "reader " "") + name))))) + + (defmacro export [#* args] "A convenience macro for defining ``__all__`` and ``_hy_export_macros``, which control which Python objects and macros (respectively) are collected by ``*`` From 7d632a8ccc9736d2dd41685df40135280345848a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 15 Oct 2023 10:26:56 -0400 Subject: [PATCH 315/342] Expand the testing of first-class macros --- tests/native_tests/import.hy | 3 ++ tests/native_tests/macros_first_class.hy | 49 +++++++++++++++++------- tests/native_tests/reader_macros.hy | 10 +++++ 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/tests/native_tests/import.hy b/tests/native_tests/import.hy index c3644cfde..e6846432b 100644 --- a/tests/native_tests/import.hy +++ b/tests/native_tests/import.hy @@ -73,6 +73,9 @@ (assert (= (tests.resources.tlib.parald 1 2 3) [9 1 2 3])) (assert (= (tests.resources.tlib.✈ "silly") "plane silly")) (assert (= (tests.resources.tlib.hyx_XairplaneX "foolish") "plane foolish")) + (assert (is + (get-macro tests.resources.tlib.✈) + (get _hy_macros (hy.mangle "tests.resources.tlib.✈")))) (assert (= (TL.parald 1 2 3) [9 1 2 3])) (assert (= (TL.✈ "silly") "plane silly")) diff --git a/tests/native_tests/macros_first_class.hy b/tests/native_tests/macros_first_class.hy index d16f9d32c..255edb140 100644 --- a/tests/native_tests/macros_first_class.hy +++ b/tests/native_tests/macros_first_class.hy @@ -1,19 +1,29 @@ "Tests of using macros as first-class objects: listing, creating, and -deleting them, and retrieving their docstrings." +deleting them, and retrieving their docstrings. We also test `get-macro` +(with regular, non-reader macros)." (import builtins) +;; * Core macros -(defn test-builtins [] +(defn test-core [] (assert (in "when" (.keys builtins._hy_macros))) (assert (not-in "global1" (.keys builtins._hy_macros))) (assert (not-in "nonexistent" (.keys builtins._hy_macros))) - (setv s (. builtins _hy_macros ["when"] __doc__)) + (assert (is + (get-macro when) + (get-macro "when") + (get builtins._hy_macros "when"))) + + (setv s (. (get-macro when) __doc__)) (assert s) (assert (is (type s) str))) +;; * Global macros + +;; ** Creation ; There are three ways to define a global macro: ; 1. `defmacro` in global scope @@ -37,24 +47,37 @@ deleting them, and retrieving their docstrings." (in (hy.mangle k) (.keys _hy_macros))))) (assert (= (global3) "from global3")) (assert (= (global☘) "from global☘")) - (assert (= - (. _hy_macros ["global1"] __doc__) - "global1 docstring")) + (assert (= (. (get-macro global1) __doc__) "global1 docstring")) ; https://github.com/hylang/hy/issues/1946 - (assert (= - (. _hy_macros [(hy.mangle "global☘")] __doc__) - "global☘ docstring"))) + (assert (= (. (get-macro global☘) __doc__) "global☘ docstring")) + (assert (= (. (get-macro hyx_globalXshamrockX) __doc__) "global☘ docstring"))) + +;; ** Deletion +; Try creating and then deleting a global macro. -; Try creating and and then deleting a global macro. (defn global4 [] "from global4 function") (setv global4-f1 (global4)) ; Calls the function -(eval-and-compile (setv (get _hy_macros "global4") (fn [&compiler] - "from global4 macro"))) +(defmacro global4 [] + "from global4 macro") (setv global4-m (global4)) ; Calls the macro -(eval-and-compile (del (get _hy_macros "global4"))) +(eval-when-compile (del (get-macro global4))) (setv global4-f2 (global4)) ; Calls the function again (defn test-global-delete [] (assert (= (global4) global4-f1 global4-f2 "from global4 function")) (assert (= global4-m "from global4 macro"))) + +;; ** Shadowing a core macro +; Try overriding a core macro, then deleting the override. + +(pragma :warn-on-core-shadow False) +(defmacro / [a b] + f"{(int a)}/{(int b)}") +(setv div1 (/ 1 2)) +(eval-when-compile (del (get-macro /))) +(setv div2 (/ 1 2)) + +(defn test-global-shadowing-builtin [] + (assert (= div1 "1/2")) + (assert (= div2 0.5))) diff --git a/tests/native_tests/reader_macros.hy b/tests/native_tests/reader_macros.hy index 5f613d12d..37f5bfe3b 100644 --- a/tests/native_tests/reader_macros.hy +++ b/tests/native_tests/reader_macros.hy @@ -58,6 +58,16 @@ (with [(pytest.raises PrematureEndOfInput)] (eval-module "# _ 3"))) +(defn test-get-macro [] + (assert (eval-module #[[ + (defreader rm1 + 11) + (defreader rm☘ + 22) + (and + (is (get-macro :reader rm1) (get _hy_reader_macros "rm1")) + (is (get-macro :reader rm☘) (get _hy_reader_macros "rm☘")))]]))) + (defn test-require-readers [] (with [module (temp-module "")] (setv it (hy.read-many #[[(require tests.resources.tlib :readers [upper!]) From 552e6df61aad1fa94248b4bb2eb278fed8448136 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 15 Oct 2023 10:28:08 -0400 Subject: [PATCH 316/342] Remove `doc` A test of #1946 remains in `tests/native_tests/macros_first_class.hy`, added by 9998e516038ec8a1d9a43e944556a7c89ff5024a. There is no longer a test for #1970, but this bug should no longer be applicable. --- NEWS.rst | 4 +++- hy/core/macros.hy | 22 ------------------ tests/native_tests/doc.hy | 36 ----------------------------- tests/native_tests/reader_macros.hy | 9 ++++++++ 4 files changed, 12 insertions(+), 59 deletions(-) delete mode 100644 tests/native_tests/doc.hy diff --git a/NEWS.rst b/NEWS.rst index d167fe402..2a9d55a7a 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,8 @@ Removals …)))` instead. * `hy.reserved` has been removed. Use `(.keys (builtins._hy_macros))` or Python's built-in `keyword` module instead. +* `doc` has been removed. Use `(help (get-macro foo))` or `(help + (get-macro :reader foo))` instead. Breaking Changes ------------------------------ @@ -16,7 +18,7 @@ Breaking Changes * `defmacro` and `require` can now define macros locally instead of only module-wide. - * `hy.eval`, `hy.macroexpand`, and `doc` don't work with + * `hy.eval` and `hy.macroexpand` don't work with local macros (yet). * When a macro is `require`\d from another module, that module is no diff --git a/hy/core/macros.hy b/hy/core/macros.hy index ce9db191c..1c2cbc4a4 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -113,28 +113,6 @@ (get _hy_reader_macros ~dispatch-key))))) -(defmacro doc [symbol] - "macro documentation - - Gets help for a macro function available in this module (not a local - macro). - Use ``require`` to make other macros available. - - Use ``(help foo)`` instead for help with runtime objects." - (setv symbol (str symbol)) - (setv namespace - (if (= (cut symbol 1) "#") - (do (setv symbol (cut symbol 1 None)) - '_hy_reader_macros) - (do (setv symbol (hy.mangle symbol)) - '_hy_macros))) - (setv builtins (hy.gensym "builtins")) - `(do (import builtins :as ~builtins) - (help (or (.get ~namespace ~symbol) - (.get (. ~builtins ~namespace) ~symbol) - (raise (NameError f"macro {~symbol !r} is not defined")))))) - - (defmacro get-macro [arg1 [arg2 None]] "Get the function object used to implement a macro. This works for core macros, global (i.e., module-level) macros, and reader macros, but not local macros (yet). For regular macros, ``get-macro`` is called with one argument, a symbol or string literal, which can be premangled or not according to taste. For reader macros, this argument must be preceded by the literal keyword ``:reader`` (and note that the hash mark, ``#``, is not included in the name of the reader macro). :: diff --git a/tests/native_tests/doc.hy b/tests/native_tests/doc.hy deleted file mode 100644 index 85afc50f0..000000000 --- a/tests/native_tests/doc.hy +++ /dev/null @@ -1,36 +0,0 @@ -(import - pytest) - - -(defreader some-tag - "Some tag macro" - '1) - -(defmacro <-mangle-> [] - "a fancy docstring" - '(+ 2 2)) - -(defn test-doc [capsys] - ;; https://github.com/hylang/hy/issues/1970 - ;; Let's first make sure we can doc the builtin macros - ;; before we create the user macros. - (doc doc) - (setv [out err] (.readouterr capsys)) - (assert (in "Gets help for a macro function" out)) - - (doc "#some-tag") - (setv [out err] (.readouterr capsys)) - (assert (in "Some tag macro" out)) - - (doc <-mangle->) - (setv [out err] (.readouterr capsys)) - ;; https://github.com/hylang/hy/issues/1946 - (assert (.startswith (.strip out) - f"Help on function {(hy.mangle '<-mangle->)} in module ")) - (assert (in "a fancy docstring" out)) - (assert (not err)) - - ;; make sure doc raises an error instead of - ;; presenting a default value help screen - (with [(pytest.raises NameError)] - (doc does-not-exist))) diff --git a/tests/native_tests/reader_macros.hy b/tests/native_tests/reader_macros.hy index 37f5bfe3b..cabd39dd9 100644 --- a/tests/native_tests/reader_macros.hy +++ b/tests/native_tests/reader_macros.hy @@ -68,6 +68,15 @@ (is (get-macro :reader rm1) (get _hy_reader_macros "rm1")) (is (get-macro :reader rm☘) (get _hy_reader_macros "rm☘")))]]))) +(defn test-docstring [] + (assert (= + (eval-module #[[ + (defreader foo + "docstring of foo" + 15) + #(#foo (. (get-macro :reader foo) __doc__))]]) + #(15 "docstring of foo")))) + (defn test-require-readers [] (with [module (temp-module "")] (setv it (hy.read-many #[[(require tests.resources.tlib :readers [upper!]) From 878a85b6e7f34247ff3303b952aef7b0c5a2e392 Mon Sep 17 00:00:00 2001 From: Sunjay Cauligi Date: Thu, 26 Oct 2023 14:44:04 +0200 Subject: [PATCH 317/342] Async generators no longer implicitly return final expressions. --- NEWS.rst | 1 + hy/core/result_macros.py | 31 +++++++++++++++++++++---------- hy/scoping.py | 8 +++++++- tests/native_tests/functions.hy | 13 +++++++++++++ 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 2a9d55a7a..4a826e7c4 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -60,6 +60,7 @@ Bug Fixes already did. * `nonlocal` now works for top-level `let`-bound names. * `hy -i` with a filename now skips shebang lines. +* Async generators no longer try to return final expressions. 0.27.0 (released 2023-07-06) ============================= diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index fde1d433d..cdc425d63 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -58,7 +58,7 @@ is_unpack, ) from hy.reader import mangle -from hy.scoping import OuterVar, ScopeFn, ScopeGen, ScopeLet, is_inside_function_scope +from hy.scoping import OuterVar, ScopeFn, ScopeGen, ScopeLet, is_function_scope, is_inside_function_scope, nearest_python_scope # ------------------------------------------------ # * Helpers @@ -1457,6 +1457,7 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo @pattern_macro(["fn", "fn/a"], [maybe(type_params), maybe_annotated(lambda_list), many(FORM)]) def compile_function_lambda(compiler, expr, root, tp, params, body): + is_async = root == "fn/a" params, returns = params posonly, args, rest, kwonly, kwargs = params has_annotations = returns is not None or any( @@ -1464,17 +1465,19 @@ def compile_function_lambda(compiler, expr, root, tp, params, body): for param in (posonly or []) + args + kwonly + [rest, kwargs] ) args, ret = compile_lambda_list(compiler, params) - with compiler.local_state(), compiler.scope.create(ScopeFn, args): + with compiler.local_state(), compiler.scope.create(ScopeFn, args, is_async) as scope: body = compiler._compile_branch(body) # Compile to lambda if we can - if not (has_annotations or tp or body.stmts or root == "fn/a"): + if not (has_annotations or tp or body.stmts or is_async): return ret + asty.Lambda(expr, args=args, body=body.force_expr) # Otherwise create a standard function - node = asty.AsyncFunctionDef if root == "fn/a" else asty.FunctionDef + node = asty.AsyncFunctionDef if is_async else asty.FunctionDef name = compiler.get_anon_var() - ret += compile_function_node(compiler, expr, node, [], tp, name, args, returns, body) + ret += compile_function_node( + compiler, expr, node, [], tp, name, args, returns, body, scope + ) # return its name as the final expr return ret + Result(expr=ret.temp_variables[0]) @@ -1485,26 +1488,30 @@ def compile_function_lambda(compiler, expr, root, tp, params, body): [maybe(brackets(many(FORM))), maybe(type_params), maybe_annotated(SYM), lambda_list, many(FORM)], ) def compile_function_def(compiler, expr, root, decorators, tp, name, params, body): + is_async = root == "defn/a" name, returns = name - node = asty.FunctionDef if root == "defn" else asty.AsyncFunctionDef + node = asty.AsyncFunctionDef if is_async else asty.FunctionDef decorators, ret, _ = compiler._compile_collect(decorators[0] if decorators else []) args, ret2 = compile_lambda_list(compiler, params) ret += ret2 name = mangle(compiler._nonconst(name)) compiler.scope.define(name) - with compiler.local_state(), compiler.scope.create(ScopeFn, args): + with compiler.local_state(), compiler.scope.create(ScopeFn, args, is_async) as scope: body = compiler._compile_branch(body) return ret + compile_function_node( - compiler, expr, node, decorators, tp, name, args, returns, body + compiler, expr, node, decorators, tp, name, args, returns, body, scope ) -def compile_function_node(compiler, expr, node, decorators, tp, name, args, returns, body): +def compile_function_node(compiler, expr, node, decorators, tp, name, args, returns, body, scope): ret = Result() if body.expr: - body += asty.Return(body.expr, value=body.expr) + # implicitly return final expression, + # except for async generators + enode = asty.Expr if scope.is_async and scope.has_yield else asty.Return + body += enode(body.expr, value=body.expr) ret += node( expr, @@ -1665,6 +1672,8 @@ def compile_return(compiler, expr, root, arg): @pattern_macro("yield", [maybe(FORM)]) def compile_yield_expression(compiler, expr, root, arg): + if is_inside_function_scope(compiler.scope): + nearest_python_scope(compiler.scope).has_yield = True ret = Result() if arg is not None: ret += compiler.compile(arg) @@ -1673,6 +1682,8 @@ def compile_yield_expression(compiler, expr, root, arg): @pattern_macro(["yield-from", "await"], [FORM]) def compile_yield_from_or_await_expression(compiler, expr, root, arg): + if root == "yield-from" and is_inside_function_scope(compiler.scope): + nearest_python_scope(compiler.scope).has_yield = True ret = Result() + compiler.compile(arg) node = asty.YieldFrom if root == "yield-from" else asty.Await return ret + node(expr, value=ret.force_expr) diff --git a/hy/scoping.py b/hy/scoping.py index 07552f137..fbfa4e6b4 100644 --- a/hy/scoping.py +++ b/hy/scoping.py @@ -292,7 +292,7 @@ def add(self, target, new_name=None): class ScopeFn(ScopeBase): """Scope that corresponds to Python's own function or class scopes.""" - def __init__(self, compiler, args=None): + def __init__(self, compiler, args=None, is_async=False): super().__init__(compiler) self.defined = set() "set: of all vars defined in this scope" @@ -305,6 +305,12 @@ def __init__(self, compiler, args=None): bool: `True` if this scope is being used to track a python function `False` for classes """ + self.is_async = is_async + """bool: `True` if this scope is for an async function, + which may need special handling during compilation""" + self.has_yield = False + """bool: `True` if this scope is tracking a function that has `yield` + statements, as generator functions may need special handling""" if args: for arg in itertools.chain( diff --git a/tests/native_tests/functions.hy b/tests/native_tests/functions.hy index a2fd36d26..fd1d31b61 100644 --- a/tests/native_tests/functions.hy +++ b/tests/native_tests/functions.hy @@ -189,6 +189,19 @@ (assert (= (asyncio.run (coro-test)) [1 2 3]))) +(defn [async-test] test-no-async-gen-return [] + ; https://github.com/hylang/hy/issues/2523 + (defn/a runner [gen] + (setv vals []) + (for [:async val (gen)] + (.append vals val)) + vals) + (defn/a naysayer [] + (yield "nope")) + (assert (= (asyncio.run (runner naysayer)) ["nope"])) + (assert (= (asyncio.run (runner (fn/a [] (yield "dope!")) ["dope!"]))))) + + (defn test-root-set-correctly [] ; https://github.com/hylang/hy/issues/2475 ((. defn) not-async [] "ok") From dd1ba62ec3990ecb8e97a3f3e8abcaa689c89759 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 26 Oct 2023 09:07:32 -0400 Subject: [PATCH 318/342] Improve docs of implicit `return` with async gen. --- NEWS.rst | 2 +- docs/api.rst | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 4a826e7c4..5dab9f901 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -60,7 +60,7 @@ Bug Fixes already did. * `nonlocal` now works for top-level `let`-bound names. * `hy -i` with a filename now skips shebang lines. -* Async generators no longer try to return final expressions. +* Implicit returns are now disabled in async generators. 0.27.0 (released 2023-07-06) ============================= diff --git a/docs/api.rst b/docs/api.rst index 4b55ad311..b37ef5aeb 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -113,7 +113,10 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. forms, and the first of them is a string literal, this string becomes the :term:`py:docstring` of the function. The final body form is implicitly returned; thus, ``(defn f [] 5)`` is equivalent to ``(defn f [] (return - 5))``. + 5))``. There is one exception: due to Python limitations, no implicit return + is added if the function is an asynchronous generator (i.e., defined with + :hy:func:`defn/a` or :hy:func:`fn/a` and containing at least one + :hy:func:`yield` or :hy:func:`yield-from`). ``defn`` accepts a few more optional arguments: a bracketed list of :term:`decorators `, a list of type parameters (see below), From e49d5e9aba5f7118aeddd87feb566cf5f0d48f8f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 4 Nov 2023 09:45:01 -0400 Subject: [PATCH 319/342] Add a belated NEWS item --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index 5dab9f901..a0c15da66 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -31,6 +31,7 @@ Breaking Changes the same `sys.path` rules as Python when parsing a module vs a standalone script. * New macro `deftype`. +* New macro `get-macro`. New Features ------------------------------ From 3be7402daf49b540a6720d4566d2db01dcdecf67 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 3 Nov 2023 13:02:25 -0400 Subject: [PATCH 320/342] Add `local-macros` --- NEWS.rst | 1 + docs/macros.rst | 4 ++-- hy/core/macros.hy | 25 ++++++++++++++++++++ hy/core/result_macros.py | 35 ++++++++++++++++++++-------- hy/macros.py | 32 ++++++++++++++++++------- tests/native_tests/macros_local.hy | 26 +++++++++++++++++++++ tests/resources/local_req_example.hy | 2 ++ 7 files changed, 105 insertions(+), 20 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index a0c15da66..1d5c2d7f6 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -32,6 +32,7 @@ Breaking Changes vs a standalone script. * New macro `deftype`. * New macro `get-macro`. +* New macro `local-macros`. New Features ------------------------------ diff --git a/docs/macros.rst b/docs/macros.rst index 3bad50ba9..5ba82f46d 100644 --- a/docs/macros.rst +++ b/docs/macros.rst @@ -5,7 +5,7 @@ Macros Operations on macros -------------------- -The currently defined global macros can be accessed as a dictionary ``_hy_macros`` in each module. (Use ``bulitins._hy_macros``, attached to Python's usual :py:mod:`builtins` module, to see core macros.) The keys are mangled macro names and the values are the function objects that implement the macros. You can operate on this dictionary to list, add, delete, or get help on macros, but be sure to use :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile` when you need the effect to happen at compile-time. The core macro :hy:func:`get-macro ` provides some syntactic sugar. :: +The currently defined global macros can be accessed as a dictionary ``_hy_macros`` in each module. (Use ``bulitins._hy_macros``, attached to Python's usual :py:mod:`builtins` module, to see core macros.) The keys are mangled macro names and the values are the function objects that implement the macros. You can operate on this dictionary to list, add, delete, or get help on macros, but be sure to use :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile` when you need the effect to happen at compile-time. You can call :hy:func:`local-macros ` to list local macros, but adding or deleting elements in this case is ineffective. The core macro :hy:func:`get-macro ` provides some syntactic sugar. :: (defmacro m [] "This is a docstring." @@ -17,7 +17,7 @@ The currently defined global macros can be accessed as a dictionary ``_hy_macros (del (get _hy_macros "m"))) (m) ; => NameError -``_hy_reader_macros`` is a similar dictionary for reader macros, but here, the keys aren't mangled. +``_hy_reader_macros`` is a dictionary like ``_hy_macros`` for reader macros, but here, the keys aren't mangled. .. _using-gensym: diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 1c2cbc4a4..84bd8a557 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -140,6 +140,31 @@ name))))) +(defmacro local-macros [] + #[[Expands to a dictionary mapping the mangled names of local macros to the function objects used to implement those macros. Thus, ``local-macros`` provides a rough local equivalent of ``_hy_macros``. :: + + (defn f [] + (defmacro m [] + "This is the docstring for the macro `m`." + 1) + (help (get (local-macros) "m"))) + (f) + + The equivalency is rough in the sense that ``local-macros`` returns a literal dictionary, not a preexisting object that Hy uses for resolving macro names. So, modifying the dictionary will have no effect. + + See also :hy:func:`get-macro `.]] + (_local-macros &compiler)) + +(defn _local_macros [&compiler] + (setv seen #{}) + (dfor + state &compiler.local_state_stack + m (get state "macros") + :if (not-in m seen) + :do (.add seen m) + m (hy.models.Symbol (hy.macros.local-macro-name m)))) + + (defmacro export [#* args] "A convenience macro for defining ``__all__`` and ``_hy_export_macros``, which control which Python objects and macros (respectively) are collected by ``*`` diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index cdc425d63..de12285f0 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -18,7 +18,7 @@ from hy._compat import PY3_11, PY3_12 from hy.compiler import Result, asty, mkexpr from hy.errors import HyEvalError, HyInternalError, HyTypeError -from hy.macros import pattern_macro, require, require_reader +from hy.macros import pattern_macro, require, require_reader, local_macro_name from hy.model_patterns import ( FORM, KEYWORD, @@ -1544,21 +1544,26 @@ def E(*x): return Expression(x) S = Symbol compiler.warn_on_core_shadow(name) - ret = [] fn_def = E( S("fn"), List([S("&compiler"), *expr[2]]), - *body) + *body).replace(expr) if compiler.is_in_local_state(): # We're in a local scope, so define the new macro locally. - compiler.local_state_stack[-1]['macros'][mangle(name)] = ( - compiler.eval(fn_def.replace(expr))) - return Result() + state = compiler.local_state_stack[-1] + # Produce code that will set the macro to a local variable. + ret = compiler.compile(E( + S("setv"), + S(local_macro_name(name)), + fn_def).replace(expr)) + # Also evaluate the macro definition now, and put it in + # state['macros']. + state['macros'][mangle(name)] = compiler.eval(fn_def) + return ret + ret.expr_as_stmt() # Otherwise, define the macro module-wide. - ret.append(E( + ret = compiler.compile(E(S("eval-and-compile"), E( E(dotted("hy.macros.macro"), str(name)), - fn_def)) - ret = compiler.compile(E(S("eval-and-compile"), *ret).replace(expr)) + fn_def)).replace(expr)) return ret + ret.expr_as_stmt() @@ -1822,12 +1827,22 @@ def compile_require(compiler, expr, root, entries): # importing readers but not macros # (require a-module :readers ["!"]) if (rest or not readers) and compiler.is_in_local_state(): - require( + reqs = require( module_name, compiler.local_state_stack[-1]['macros'], assignments = assignments, prefix = prefix, compiler = compiler) + ret += compiler.compile(Expression([ + Symbol("setv"), + List([Symbol(local_macro_name(m)) for m, _, _ in reqs]), + Expression([ + dotted("hy.macros.require_vals"), + String(module_name), + Dict(), + Keyword("assignments"), + List([(String(m), String(m)) for _, m, _ in reqs])])]).replace(expr)) + ret += ret.expr_as_stmt() elif (rest or not readers) and require( module_name, compiler.module, diff --git a/hy/macros.py b/hy/macros.py index 657999500..fcdcb0e49 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -184,8 +184,9 @@ def enable_readers(module, reader, names): def require(source_module, target, assignments, prefix="", target_module_name=None, compiler=None): - """Load macros from a module. Return a `bool` indicating whether - macros were actually transferred. + """Load macros from a module. Return a list of (new name, source + name, macro object) tuples, including only macros that were + actually transferred. - `target` can be a a string (naming a module), a module object, a dictionary, or `None` (meaning the calling module). @@ -204,7 +205,7 @@ def require(source_module, target, assignments, prefix="", target_module_name=No # We use the module's underlying filename for this (when they exist), since # it's the most "fixed" attribute. if _same_modules(source_module, target_module): - return False + return [] if not inspect.ismodule(source_module): source_module = import_module_from_string(source_module, @@ -219,27 +220,29 @@ def require(source_module, target, assignments, prefix="", target_module_name=No if not source_module._hy_macros: if assignments in ("ALL", "EXPORTS"): - return False + return [] + out = [] for name, alias in assignments: try: - require( + out.extend(require( f"{source_module.__name__}.{mangle(name)}", target_module or target, "ALL", prefix=alias, - ) + )) except HyRequireError as e: raise HyRequireError( f"Cannot import name '{name}'" f" from '{source_module.__name__}'" f" ({source_module.__file__})" ) - return True + return out target_macros = target_namespace.setdefault("_hy_macros", {}) if target_module else target if prefix: prefix += "." + out = [] for name, alias in ( assignments @@ -256,12 +259,25 @@ def require(source_module, target, assignments, prefix="", target_module_name=No alias = mangle(prefix + alias) if _name in source_module._hy_macros: target_macros[alias] = source_macros[_name] + out.append((alias, _name, source_macros[_name])) else: raise HyRequireError( "Could not require name {} from {}".format(_name, source_module) ) - return True + return out + + +def require_vals(*args, **kwargs): + return [value for _, _, value in require(*args, **kwargs)] + + +def local_macro_name(original): + return '_hy_local_macro__' + (mangle(original) + # We have to do a bit of extra mangling beyond `mangle` itself to + # handle names with periods. + .replace('D', 'DN') + .replace('.', 'DD')) def load_macros(module): diff --git a/tests/native_tests/macros_local.hy b/tests/native_tests/macros_local.hy index 6126b9c29..f8639fba0 100644 --- a/tests/native_tests/macros_local.hy +++ b/tests/native_tests/macros_local.hy @@ -95,3 +95,29 @@ (defn test-redefinition-warning [] (macro-redefinition-warning-tester :local True)) + + +(defn test-get-local-macros [] + "Test the core macro `local-macros`." + + (defn check [ms inner?] + (assert (= (set (.keys ms)) + #{"m1" "m2" "rwiz" #* (if inner? ["m3" "rhelp"] [])})) + (assert (= (. ms ["m1"] __doc__) "m1 doc")) + (assert (= (. ms ["m2"] __doc__) (if inner? "m2, inner" "m2, outer"))) + (assert (= (. ms ["rwiz"] __doc__) "remote wiz doc")) + (when inner? + (assert (= (. ms ["m3"] __doc__) "m3 doc")) + (assert (= (. ms ["rhelp"] __doc__) "remote helper doc")))) + + (defmacro m1 [] "m1 doc" 1) + (defmacro m2 [] "m2, outer" 2) + (require tests.resources.local-req-example [wiz :as rwiz]) + (check (local-macros) False) + (defn inner-f [] + (defmacro m2 [] "m2, inner" 2) + (defmacro m3 [] "m3 doc" 3) + (require tests.resources.local-req-example [helper :as rhelp]) + (check (local-macros) True)) + (inner-f) + (check (local-macros) False)) diff --git a/tests/resources/local_req_example.hy b/tests/resources/local_req_example.hy index 82467fc1a..c9c2bc80f 100644 --- a/tests/resources/local_req_example.hy +++ b/tests/resources/local_req_example.hy @@ -1,6 +1,8 @@ (defmacro wiz [] + "remote wiz doc" "remote wiz") (defmacro get-wiz [] (wiz)) (defmacro helper [] + "remote helper doc" "remote helper macro") From ab5ab1610be0041d7b79ed0d8cc77d5358b2d0e9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 3 Nov 2023 12:52:14 -0400 Subject: [PATCH 321/342] Allow `get-macro` to get local macros --- hy/core/macros.hy | 17 ++++++++++------- tests/native_tests/macros_first_class.hy | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 84bd8a557..878221f83 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -114,29 +114,32 @@ (defmacro get-macro [arg1 [arg2 None]] - "Get the function object used to implement a macro. This works for core macros, global (i.e., module-level) macros, and reader macros, but not local macros (yet). For regular macros, ``get-macro`` is called with one argument, a symbol or string literal, which can be premangled or not according to taste. For reader macros, this argument must be preceded by the literal keyword ``:reader`` (and note that the hash mark, ``#``, is not included in the name of the reader macro). :: + "Get the function object used to implement a macro. This works for all sorts of macros: core macros, global (i.e., module-level) macros, local macros, and reader macros. For regular (non-reader) macros, ``get-macro`` is called with one argument, a symbol or string literal, which can be premangled or not according to taste. For reader macros, this argument must be preceded by the literal keyword ``:reader`` (and note that the hash mark, ``#``, is not included in the name of the reader macro). :: (get-macro my-macro) (get-macro :reader my-reader-macro) - ``get-macro`` expands to a :hy:func:`get ` form on the appropriate object, such as ``_hy_macros``, selected at the time of expanding ``get-macro``. This means you can say ``(del (get-macro …))``, perhaps wrapped in :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile`, to delete a macro, but it's easy to get confused by the order of evaluation and number of evaluations. For more predictable results in complex situations, use ``(del (get …))`` directly instead of ``(del (get-macro …))``." + Except when retrieving a local macro, ``get-macro`` expands to a :hy:func:`get ` form on the appropriate object, such as ``_hy_macros``, selected at the time of expanding ``get-macro``. This means you can say ``(del (get-macro …))``, perhaps wrapped in :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile`, to delete a macro, but it's easy to get confused by the order of evaluation and number of evaluations. For more predictable results in complex situations, use ``(del (get …))`` directly instead of ``(del (get-macro …))``." (import builtins) - (setv [name namespace] (cond + (setv [name reader?] (cond (= arg1 ':reader) - [(str arg2) "_hy_reader_macros"] + [(str arg2) True] (isinstance arg1 hy.models.Expression) - [(hy.mangle (.join "." (cut arg1 1 None))) "_hy_macros"] + [(hy.mangle (.join "." (cut arg1 1 None))) False] True - [(hy.mangle arg1) "_hy_macros"])) + [(hy.mangle arg1) False])) + (setv namespace (if reader? "_hy_reader_macros" "_hy_macros")) (cond + (and (not reader?) (setx local (.get (_local-macros &compiler) name))) + local (in name (getattr &compiler.module namespace {})) `(get ~(hy.models.Symbol namespace) ~name) (in name (getattr builtins namespace {})) `(get (. hy.M.builtins ~(hy.models.Symbol namespace)) ~name) True (raise (NameError (.format "no such {}macro: {!r}" - (if (= namespace "_hy_reader_macros") "reader " "") + (if reader? "reader " "") name))))) diff --git a/tests/native_tests/macros_first_class.hy b/tests/native_tests/macros_first_class.hy index 255edb140..7245a183c 100644 --- a/tests/native_tests/macros_first_class.hy +++ b/tests/native_tests/macros_first_class.hy @@ -81,3 +81,23 @@ deleting them, and retrieving their docstrings. We also test `get-macro` (defn test-global-shadowing-builtin [] (assert (= div1 "1/2")) (assert (= div2 0.5))) + +;; * Local macros + +(defn test-local-get [] + (defmacro local1 [] "local1 doc" 1) + (defmacro local2 [] "local2 outer" 2) + (require tests.resources.local-req-example :as LRE) + + (assert (= (. (get-macro local1) __doc__) "local1 doc")) + (assert (= (. (get-macro local2) __doc__) "local2 outer")) + (assert (= (. (get-macro LRE.wiz) __doc__) "remote wiz doc")) + + (defn inner [] + (defmacro local2 [] "local2 inner" 2) + (defmacro local3 [] "local3 doc" 2) + (assert (= (. (get-macro local2) __doc__) "local2 inner")) + (assert (= (. (get-macro local3) __doc__) "local3 doc")) + (assert (= (. (get-macro LRE.wiz) __doc__) "remote wiz doc"))) + + (inner)) From 43114b08c0f6530c06ee2154f790895796174cec Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 4 Nov 2023 09:41:41 -0400 Subject: [PATCH 322/342] Add a `macros` parameter to `hy.eval` --- NEWS.rst | 6 +++--- hy/compiler.py | 34 +++++++++++++++++++++++++------ hy/macros.py | 8 +++++--- tests/native_tests/hy_eval.hy | 38 +++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 12 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 1d5c2d7f6..319600a18 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -18,13 +18,13 @@ Breaking Changes * `defmacro` and `require` can now define macros locally instead of only module-wide. - * `hy.eval` and `hy.macroexpand` don't work with - local macros (yet). + * `hy.macroexpand` doesn't work with local macros (yet). * When a macro is `require`\d from another module, that module is no longer implicitly included when checking for further macros in the expansion. -* `hy.eval` has been overhauled to be more like Python's `eval`. +* `hy.eval` has been overhauled to be more like Python's `eval`. It + also has a new parameter `macros`. * `hy` now only implicitly launches a REPL if standard input is a TTY. * `hy -i` has been overhauled to work as a flag like `python3 -i`. * `hy2py` now requires `-m` to specify modules, and uses diff --git a/hy/compiler.py b/hy/compiler.py index 54bbb5215..a72af4204 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -326,7 +326,7 @@ def is_annotate_expression(model): class HyASTCompiler: """A Hy-to-Python AST compiler""" - def __init__(self, module, filename=None, source=None): + def __init__(self, module, filename=None, source=None, extra_macros=None): """ Args: module (Union[str, types.ModuleType]): Module name or object in which the Hy tree is evaluated. @@ -335,9 +335,12 @@ def __init__(self, module, filename=None, source=None): debugging. source (Optional[str]): The source for the file, if any, being compiled. This is optional information for informative error messages and debugging. + extra_macros (Optional[dict]): More macros to use during lookup. They take precedence + over macros in `module`. """ self.anon_var_count = 0 self.temp_if = None + self.extra_macros = extra_macros or {} # Make a list of dictionaries with local compiler settings, # such as the definitions of local macros. The last element is @@ -736,6 +739,7 @@ def hy_eval( source=None, import_stdlib=True, globals=None, + extra_macros=None, ): module = get_compiler_module(module, compiler, True) @@ -752,6 +756,7 @@ def hy_eval( filename=filename, source=source, import_stdlib=import_stdlib, + extra_macros=extra_macros, ) if globals is None: @@ -764,14 +769,16 @@ def hy_eval( return eval(ast_compile(expr, filename, "eval"), globals, locals) -def hy_eval_user(model, globals = None, locals = None, module = None): +def hy_eval_user(model, globals = None, locals = None, module = None, macros = None): # This function is advertised as `hy.eval`. """An equivalent of Python's :func:`eval` for evaluating Hy code. The chief difference is that the first argument should be a :ref:`model ` rather than source text. If you have a string of source text you want to evaluate, convert it to a model first with :hy:func:`hy.read` or :hy:func:`hy.read-many`:: (hy.eval '(+ 1 1)) ; => 2 (hy.eval (hy.read "(+ 1 1)")) ; => 2 - The optional arguments ``globals`` and ``locals`` work as in the case of ``eval``. There's one more optional argument, ``module``, which can be a module object or a string naming a module. The module's ``__dict__`` attribute can fill in for ``globals`` (and hence also for ``locals``) if ``module`` is provided but ``globals`` isn't, but the primary purpose of ``module`` is to control where macro calls are looked up. Without this argument, the calling module of ``hy.eval`` is used instead. :: + The optional arguments ``globals`` and ``locals`` work as in the case of :func:`eval`. + + Another optional argument, ``module``, can be a module object or a string naming a module. The module's ``__dict__`` attribute can fill in for ``globals`` (and hence also for ``locals``) if ``module`` is provided but ``globals`` isn't, but the primary purpose of ``module`` is to control where macro calls are looked up. Without this argument, the calling module of ``hy.eval`` is used instead. :: (defmacro my-test-mac [] 3) (hy.eval '(my-test-mac)) ; => 3 @@ -779,7 +786,15 @@ def hy_eval_user(model, globals = None, locals = None, module = None): (hy.eval '(my-test-mac) :module hyrule) ; NameError (hy.eval '(list-n 3 1) :module hyrule) ; => [1 1 1] - N.B. Local macros are invisible to ``hy.eval``.""" + Finally, finer control of macro lookup can be achieved by passing in a dictionary of macros as the ``macros`` argument. The keys of this dictionary should be mangled macro names, and the values should be function objects to implement those macros. This is the same structure as is produced by :hy:func:`local-macros`, and in fact, ``(hy.eval … :macros (local-macros))`` is useful to make local macros visible to ``hy.eval``, which otherwise doesn't see them. :: + + (defn f [] + (defmacro lmac [] 1) + (hy.eval '(lmac)) ; NameError + (print (hy.eval '(lmac) :macros (local-macros)))) ; => 1 + (f) + + In any case, macros provided in this dictionary will shadow macros of the same name that are associated with the provided or implicit module. You can shadow a core macro, too, so be careful: there's no warning for this as there is in the case of :hy:func:`defmacro`.""" if locals is None: locals = globals @@ -793,7 +808,8 @@ def hy_eval_user(model, globals = None, locals = None, module = None): locals = (inspect.getargvalues(inspect.stack()[1][0]).locals if locals is None and module is None else locals), - module = get_compiler_module(module, None, True)) + module = get_compiler_module(module, None, True), + extra_macros = macros) finally: if locals is not None: if hy_was: @@ -815,6 +831,7 @@ def hy_compile( filename=None, source=None, import_stdlib=True, + extra_macros=None, ): """Compile a hy.models.Object tree into a Python AST Module. @@ -835,6 +852,7 @@ def hy_compile( if `None`, an attempt will be made to obtain it from the module given by `module`. When `compiler` is given, its `source` field value is always used. + extra_macros (Optional[dict]): Passed through to `HyASTCompiler`, if it's called. Returns: ast.AST: A Python AST tree @@ -851,7 +869,11 @@ def hy_compile( "`tree` must be a hy.models.Object or capable of " "being promoted to one" ) - compiler = compiler or HyASTCompiler(module, filename=filename, source=source) + compiler = compiler or HyASTCompiler( + module, + filename = filename, + source = source, + extra_macros = extra_macros) with HyReader.using_reader(reader, create=False), compiler.scope: result = compiler.compile(tree) diff --git a/hy/macros.py b/hy/macros.py index fcdcb0e49..a701c28b1 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -385,9 +385,11 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): # Choose the first namespace with the macro. m = ((compiler and next( - (d['macros'][fn] - for d in reversed(compiler.local_state_stack) - if fn in d['macros']), + (d[fn] + for d in [ + compiler.extra_macros, + *(s['macros'] for s in reversed(compiler.local_state_stack))] + if fn in d), None)) or next( (mod._hy_macros[fn] diff --git a/tests/native_tests/hy_eval.hy b/tests/native_tests/hy_eval.hy index f758e8b85..479b5907a 100644 --- a/tests/native_tests/hy_eval.hy +++ b/tests/native_tests/hy_eval.hy @@ -206,6 +206,44 @@ (assert (= (hy.eval '(qplah 1)) [8 1]))) +(defn test-extra-macros [] + (setv ab 15) + + (assert (= + (hy.eval '(chippy a b) :macros (dict + :chippy (fn [&compiler arg1 arg2] + (hy.models.Symbol (+ (str arg1) (str arg2)))))) + 15)) + + ; By default, `hy.eval` can't see local macros. + (defmacro oh-hungee [arg1 arg2] + (hy.models.Symbol (+ (str arg1) (str arg2)))) + (with [(pytest.raises NameError)] + (hy.eval '(oh-hungee a b))) + ; But you can pass them in with the `macros` argument. + (assert (= + (hy.eval '(oh-hungee a b) :macros (local-macros)) + 15)) + (assert (= + (hy.eval '(oh-hungee a b) :macros {"oh_hungee" (get-macro oh-hungee)} + 15))) + + ; You can shadow a global macro. + (assert (= + (hy.eval '(cheese)) + "gorgonzola")) + (assert (= + (hy.eval '(cheese) :macros {"cheese" (fn [&compiler] "cheddar")}) + "cheddar")) + + ; Or even a core macro, and with no warning. + (assert (= + (hy.eval '(+ 1 1) :macros + {(hy.mangle "+") (fn [&compiler #* args] + (.join "" (gfor x args (str (int x)))))}) + "11"))) + + (defn test-filename [] (setv m (hy.read "(/ 1 0)" :filename "bad_math.hy")) (with [e (pytest.raises ZeroDivisionError)] From 235244cc60e0fdc3be1bb62fca55bc0b71b340d5 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 4 Nov 2023 11:18:08 -0400 Subject: [PATCH 323/342] Overhaul `hy.macroexpand` and `hy.macroexpand-1` --- NEWS.rst | 7 +-- hy/core/util.hy | 66 ++++++++++++++++------------ hy/macros.py | 6 --- tests/macros/test_macro_processor.py | 4 +- tests/native_tests/hy_misc.hy | 28 ++++++++++-- 5 files changed, 67 insertions(+), 44 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 319600a18..8f111d4b1 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -17,14 +17,13 @@ Breaking Changes * `defmacro` and `require` can now define macros locally instead of only module-wide. - - * `hy.macroexpand` doesn't work with local macros (yet). - * When a macro is `require`\d from another module, that module is no longer implicitly included when checking for further macros in the expansion. * `hy.eval` has been overhauled to be more like Python's `eval`. It also has a new parameter `macros`. +* `hy.macroexpand` and `hy.macroexpand-1` have been overhauled and + generalized to include more of the features of `hy.eval`. * `hy` now only implicitly launches a REPL if standard input is a TTY. * `hy -i` has been overhauled to work as a flag like `python3 -i`. * `hy2py` now requires `-m` to specify modules, and uses @@ -63,6 +62,8 @@ Bug Fixes * `nonlocal` now works for top-level `let`-bound names. * `hy -i` with a filename now skips shebang lines. * Implicit returns are now disabled in async generators. +* The parameter `result-ok` that was mistakenly included in the + signature of `hy.macroexpand` is now gone. 0.27.0 (released 2023-07-06) ============================= diff --git a/hy/core/util.hy b/hy/core/util.hy index fe939e477..217b6366b 100644 --- a/hy/core/util.hy +++ b/hy/core/util.hy @@ -84,34 +84,42 @@ (setv f (get (.stack inspect) (+ n 1) 0)) (get f.f_globals "__name__")) -(defn macroexpand [form [result-ok False]] - "Return the full macro expansion of `form`. - - Examples: - :: - - => (require hyrule [->]) - => (hy.macroexpand '(-> (a b) (x y))) - '(x (a b) y) - => (hy.macroexpand '(-> (a b) (-> (c d) (e f)))) - '(e (c (a b) d) f) - " - (import hy.macros) - (setv module (calling-module)) - (hy.macros.macroexpand form module (HyASTCompiler module) :result-ok result-ok)) - -(defn macroexpand-1 [form] - "Return the single step macro expansion of `form`. - - Examples: - :: - - => (require hyrule [->]) - => (hy.macroexpand-1 '(-> (a b) (-> (c d) (e f)))) - '(-> (a b) (c d) (e f)) - " - (import hy.macros) - (setv module (calling-module)) - (hy.macros.macroexpand-1 form module (HyASTCompiler module))) +(defn _macroexpand [model module macros #** kwargs] + (if (and (isinstance model hy.models.Expression) model) + (hy.macros.macroexpand + :tree model + :module module + :compiler (HyASTCompiler module :extra-macros macros) + :result-ok False + #** kwargs) + model)) + +(defn macroexpand [model [module None] [macros None]] + "As :hy:func:`hy.macroexpand-1`, but the expansion process is repeated until it has no effect. :: + + (defmacro m [x] + (and (int x) `(m ~(- x 1)))) + (print (hy.repr (hy.macroexpand-1 '(m 5)))) + ; => '(m 4) + (print (hy.repr (hy.macroexpand '(m 5)))) + ; => '0 + + Note that in general, macro calls in the arguments of the expression still won't expanded. To expand these, too, try Hyrule's :hy:func:`macroexpand-all `." + (_macroexpand model (or module (calling-module)) macros)) + +(defn macroexpand-1 [model [module None] [macros None]] + "Check if ``model`` is an :class:`Expression ` specifying a macro call. If so, expand the macro and return the expansion; otherwise, return ``model`` unchanged. :: + + (defmacro m [x] + `(do ~x ~x ~x)) + (print (hy.repr (hy.macroexpand-1 '(m (+= n 1))))) + ; => '(do (+= n 1) (+= n 1) (+= n 1)) + + An exceptional case is if the macro is a core macro that returns one of Hy's internal compiler result objects instead of a real model. Then, you just get the original back, as if the macro hadn't been expanded. + + The optional arguments ``module`` and ``macros`` can be provided to control where macros are looked up, as with :hy:func:`hy.eval`. + + See also :hy:func:`hy.macroexpand`." + (_macroexpand model (or module (calling-module)) macros :once True)) (setv __all__ []) diff --git a/hy/macros.py b/hy/macros.py index a701c28b1..90dae35b6 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -415,12 +415,6 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): return tree -def macroexpand_1(tree, module, compiler=None): - """Expand the toplevel macro from `tree` once, in the context of - `compiler`.""" - return macroexpand(tree, module, compiler, once=True) - - def rename_function(f, new_name): """Create a copy of a function, but with a new name.""" f = type(f)( diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py index 464dcca81..1e4c452e9 100644 --- a/tests/macros/test_macro_processor.py +++ b/tests/macros/test_macro_processor.py @@ -2,7 +2,7 @@ from hy.compiler import HyASTCompiler from hy.errors import HyMacroExpansionError -from hy.macros import macro, macroexpand, macroexpand_1 +from hy.macros import macro, macroexpand from hy.models import Expression, Float, List, String, Symbol from hy.reader import read @@ -58,6 +58,6 @@ def test_macroexpand_source_data(): ast = Expression([Symbol("when"), String("a")]) ast.start_line = 3 ast.start_column = 5 - bad = macroexpand_1(ast, "hy.core.macros") + bad = macroexpand(ast, "hy.core.macros", once = True) assert bad.start_line == 3 assert bad.start_column == 5 diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index 96a818bd4..07a03dff4 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -22,10 +22,30 @@ (defn test-macroexpand [] - (assert (= (hy.macroexpand '(mac (a b) (x y))) - '(x y (a b)))) - (assert (= (hy.macroexpand '(mac (a b) (mac 5))) - '(a b 5)))) + (assert (= + (hy.macroexpand '(mac (a b) (x y))) + '(x y (a b)))) + (assert (= + (hy.macroexpand '(mac (a b) (mac 5))) + '(a b 5))) + (assert (= + (hy.macroexpand '(qplah "phooey") :module hy.M.tests.resources.tlib) + '[8 "phooey"])) + (assert (= + (hy.macroexpand '(chippy 1) :macros + {"chippy" (fn [&compiler x] `[~x ~x])}) + '[1 1])) + ; Non-Expressions just get returned as-is. + (defn f []) + (assert (is + (hy.macroexpand f) + f)) + ; If the macro expands to a `Result`, the user gets the original + ; back instead of the `Result`. + (setv model '(+ 1 1)) + (assert (is + (hy.macroexpand model) + model))) (defmacro m-with-named-import [] From 6d146a5d4a972759b0e33ae944575dbeb48502a4 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 5 Nov 2023 10:22:57 -0500 Subject: [PATCH 324/342] Revert the autoformatting of `fastentrypoints` This file is meant to be a copy of an upstream file. --- fastentrypoints.py | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/fastentrypoints.py b/fastentrypoints.py index 4369fbecd..9707f74a3 100644 --- a/fastentrypoints.py +++ b/fastentrypoints.py @@ -24,7 +24,7 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" +''' Monkey patch setuptools to write faster console_scripts with this format: import sys @@ -35,12 +35,10 @@ (c) 2016, Aaron Christianson http://github.com/ninjaaron/fast-entry_points -""" -import re - +''' from setuptools.command import easy_install - -TEMPLATE = """\ +import re +TEMPLATE = '''\ # -*- coding: utf-8 -*- # EASY-INSTALL-ENTRY-SCRIPT: '{3}','{4}','{5}' __requires__ = '{3}' @@ -51,7 +49,7 @@ if __name__ == '__main__': sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit({2}())""" + sys.exit({2}())''' @classmethod @@ -64,15 +62,15 @@ def get_args(cls, dist, header=None): # noqa: D205,D400 # pylint: disable=E1101 header = cls.get_header() spec = str(dist.as_requirement()) - for type_ in "console", "gui": - group = type_ + "_scripts" + for type_ in 'console', 'gui': + group = type_ + '_scripts' for name, ep in dist.get_entry_map(group).items(): # ensure_safe_name - if re.search(r"[\\/]", name): + if re.search(r'[\\/]', name): raise ValueError("Path separators not allowed in script names") script_text = TEMPLATE.format( - ep.module_name, ep.attrs[0], ".".join(ep.attrs), spec, group, name - ) + ep.module_name, ep.attrs[0], '.'.join(ep.attrs), + spec, group, name) # pylint: disable=E1101 args = cls._get_script_args(type_, name, header, script_text) for res in args: @@ -88,29 +86,27 @@ def main(): import re import shutil import sys - - dests = sys.argv[1:] or ["."] - filename = re.sub("\.pyc$", ".py", __file__) + dests = sys.argv[1:] or ['.'] + filename = re.sub('\.pyc$', '.py', __file__) for dst in dests: shutil.copy(filename, dst) - manifest_path = os.path.join(dst, "MANIFEST.in") - setup_path = os.path.join(dst, "setup.py") + manifest_path = os.path.join(dst, 'MANIFEST.in') + setup_path = os.path.join(dst, 'setup.py') # Insert the include statement to MANIFEST.in if not present - with open(manifest_path, "a+") as manifest: + with open(manifest_path, 'a+') as manifest: manifest.seek(0) manifest_content = manifest.read() - if "include fastentrypoints.py" not in manifest_content: - manifest.write( - ("\n" if manifest_content else "") + "include fastentrypoints.py" - ) + if 'include fastentrypoints.py' not in manifest_content: + manifest.write(('\n' if manifest_content else '') + + 'include fastentrypoints.py') # Insert the import statement to setup.py if not present - with open(setup_path, "a+") as setup: + with open(setup_path, 'a+') as setup: setup.seek(0) setup_content = setup.read() - if "import fastentrypoints" not in setup_content: + if 'import fastentrypoints' not in setup_content: setup.seek(0) setup.truncate() - setup.write("import fastentrypoints\n" + setup_content) + setup.write('import fastentrypoints\n' + setup_content) From 03f7a6fddba31e23654dbf5a663faa26c30070eb Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 5 Nov 2023 10:30:00 -0500 Subject: [PATCH 325/342] Update `fastentrypoints` to a3a26f320c7ae2191fde71b79d4f4bf325d162f3 The changes are trivial code-style tweaks. Despite what its README might suggest, `fastentrypoints` still provides an apparent speedup with Python 3.11.4 and `setuptools` 65.5.1, even if the speedup is smaller than it used to be. On my laptop, I get hy -c 1 0.05s user 0.00s system 99% cpu 0.050 total hy -c 1 0.04s user 0.01s system 99% cpu 0.049 total hy -c 1 0.04s user 0.01s system 99% cpu 0.051 total with it, versus hy -c 1 0.06s user 0.01s system 99% cpu 0.063 total hy -c 1 0.06s user 0.00s system 99% cpu 0.065 total hy -c 1 0.05s user 0.01s system 99% cpu 0.063 total without. --- fastentrypoints.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fastentrypoints.py b/fastentrypoints.py index 9707f74a3..c26a4ff64 100644 --- a/fastentrypoints.py +++ b/fastentrypoints.py @@ -38,7 +38,7 @@ ''' from setuptools.command import easy_install import re -TEMPLATE = '''\ +TEMPLATE = r''' # -*- coding: utf-8 -*- # EASY-INSTALL-ENTRY-SCRIPT: '{3}','{4}','{5}' __requires__ = '{3}' @@ -49,7 +49,8 @@ if __name__ == '__main__': sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit({2}())''' + sys.exit({2}()) +'''.lstrip() @classmethod @@ -83,11 +84,10 @@ def get_args(cls, dist, header=None): # noqa: D205,D400 def main(): import os - import re import shutil import sys dests = sys.argv[1:] or ['.'] - filename = re.sub('\.pyc$', '.py', __file__) + filename = re.sub(r'\.pyc$', '.py', __file__) for dst in dests: shutil.copy(filename, dst) From 130bb3bd9fc8c3828dec4e95c3bd9f537fd08a97 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 5 Nov 2023 09:36:18 -0500 Subject: [PATCH 326/342] Update `pytest_collect_file` Don't use the deprecated parameter `path`. --- conftest.py | 12 ++++++------ requirements-dev.txt | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conftest.py b/conftest.py index a88904842..9cb5a2916 100644 --- a/conftest.py +++ b/conftest.py @@ -4,16 +4,16 @@ import pytest -NATIVE_TESTS = os.path.join("", "tests", "native_tests", "") +NATIVE_TESTS = Path.cwd() / "tests/native_tests" # https://github.com/hylang/hy/issues/2029 os.environ.pop("HYSTARTUP", None) -def pytest_collect_file(parent, path): +def pytest_collect_file(file_path, parent): if ( - path.ext == ".hy" - and NATIVE_TESTS in path.dirname + os.sep - and path.basename != "__init__.hy" + file_path.suffix == ".hy" + and NATIVE_TESTS in file_path.parents + and file_path.name != "__init__.hy" ): - return pytest.Module.from_parent(parent, path=Path(path)) + return pytest.Module.from_parent(parent, path=file_path) diff --git a/requirements-dev.txt b/requirements-dev.txt index 05f75721e..a28ba3224 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -pytest >= 6 +pytest >= 7 # documentation Pygments == 2.15.1 From 75f80b21592be6888c9149f6b38995f0edf070d3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 7 Nov 2023 16:18:36 -0500 Subject: [PATCH 327/342] Clean up imports in `conftest.py` --- conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index 9cb5a2916..815d86d89 100644 --- a/conftest.py +++ b/conftest.py @@ -1,8 +1,7 @@ -import importlib import os from pathlib import Path -import pytest +import hy, pytest NATIVE_TESTS = Path.cwd() / "tests/native_tests" From 2d825aabf2e71906bc352cde2806a2bc6cc9fab7 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 7 Nov 2023 16:21:56 -0500 Subject: [PATCH 328/342] On GitHub Actions, use Python 3.12, not 3.12-dev --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 13619bea9..f1b2a7008 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: matrix: name-prefix: [''] os: [ubuntu-latest] - python: [3.8, 3.9, '3.10', 3.11, 3.12-dev, pypy-3.10, pyodide] + python: [3.8, 3.9, '3.10', 3.11, 3.12, pypy-3.10, pyodide] include: # To keep the overall number of runs low, we test Windows and MacOS # only on the latest CPython. From e94f6c7a22192f9b434483c39351017f860890b4 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 6 Nov 2023 14:07:40 -0500 Subject: [PATCH 329/342] =?UTF-8?q?Mass=20replacement:=20`hy.M`=20?= =?UTF-8?q?=E2=86=92=20`hy.I`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/api.rst | 2 +- docs/conf.py | 2 +- hy/__init__.py | 8 ++++---- hy/core/macros.hy | 2 +- tests/native_tests/comprehensions.hy | 2 +- tests/native_tests/deftype.hy | 2 +- tests/native_tests/hy_eval.hy | 2 +- tests/native_tests/hy_misc.hy | 30 ++++++++++++++-------------- tests/native_tests/let.hy | 2 +- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index b37ef5aeb..69344d4d0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1369,7 +1369,7 @@ the following methods .. hy:autofunction:: hy.as-model -.. hy:autoclass:: hy.M +.. hy:autoclass:: hy.I .. _reader-macros: diff --git a/docs/conf.py b/docs/conf.py index 3d6be0873..e12278d77 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -73,7 +73,7 @@ ) import hy -hy.M = type(hy.M) # A trick to enable `hy:autoclass:: hy.M` +hy.I = type(hy.I) # A trick to enable `hy:autoclass:: hy.I` # ** Generate Cheatsheet import json diff --git a/hy/__init__.py b/hy/__init__.py index 03fc2e2d5..3a38caec5 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -16,10 +16,10 @@ def _initialize_env_var(env_var, default_val): # we import for side-effects. -class M: - """``hy.M`` is an object that provides syntactic sugar for imports. It allows syntax like ``(hy.M.math.sqrt 2)`` to mean ``(import math) (math.sqrt 2)``, except without bringing ``math`` or ``math.sqrt`` into scope. This is useful in macros to avoid namespace pollution. To refer to a module with dots in its name, use slashes instead: ``hy.M.os/path.basename`` gets the function ``basename`` from the module ``os.path``. +class I: + """``hy.I`` is an object that provides syntactic sugar for imports. It allows syntax like ``(hy.I.math.sqrt 2)`` to mean ``(import math) (math.sqrt 2)``, except without bringing ``math`` or ``math.sqrt`` into scope. This is useful in macros to avoid namespace pollution. To refer to a module with dots in its name, use slashes instead: ``hy.I.os/path.basename`` gets the function ``basename`` from the module ``os.path``. - You can also call ``hy.M`` like a function, as in ``(hy.M "math")``, which is useful when the module name isn't known until run-time. This interface just calls :py:func:`importlib.import_module`, avoiding (1) mangling due to attribute lookup, and (2) the translation of ``/`` to ``.`` in the module name. The advantage of ``(hy.M modname)`` over ``importlib.import_module(modname)`` is merely that it avoids bringing ``importlib`` itself into scope.""" + You can also call ``hy.I`` like a function, as in ``(hy.I "math")``, which is useful when the module name isn't known until run-time. This interface just calls :py:func:`importlib.import_module`, avoiding (1) mangling due to attribute lookup, and (2) the translation of ``/`` to ``.`` in the module name. The advantage of ``(hy.I modname)`` over ``importlib.import_module(modname)`` is merely that it avoids bringing ``importlib`` itself into scope.""" def __call__(self, module_name): import importlib return importlib.import_module(module_name) @@ -29,7 +29,7 @@ def __getattr__(self, s): r'/(-*)', lambda m: '.' + '_' * len(m.group(1)), hy.unmangle(s)))) -M = M() +I = I() # Import some names on demand so that the dependent modules don't have diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 878221f83..4eab02e17 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -136,7 +136,7 @@ (in name (getattr &compiler.module namespace {})) `(get ~(hy.models.Symbol namespace) ~name) (in name (getattr builtins namespace {})) - `(get (. hy.M.builtins ~(hy.models.Symbol namespace)) ~name) + `(get (. hy.I.builtins ~(hy.models.Symbol namespace)) ~name) True (raise (NameError (.format "no such {}macro: {!r}" (if reader? "reader " "") diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy index bc7166759..bc354a4cb 100644 --- a/tests/native_tests/comprehensions.hy +++ b/tests/native_tests/comprehensions.hy @@ -264,7 +264,7 @@ (defmacro eval-isolated [#* body] - `(hy.eval '(do ~@body) :module (hy.M.types.ModuleType "") :locals {})) + `(hy.eval '(do ~@body) :module (hy.I.types.ModuleType "") :locals {})) (defn test-lfor-nonlocal [] diff --git a/tests/native_tests/deftype.hy b/tests/native_tests/deftype.hy index 8be8a9e3b..3fb6e0e25 100644 --- a/tests/native_tests/deftype.hy +++ b/tests/native_tests/deftype.hy @@ -10,7 +10,7 @@ (assert (= Foo.__value__) int) (deftype Foo (| int bool)) - (assert (is (type Foo.__value__ hy.M.types.UnionType))) + (assert (is (type Foo.__value__ hy.I.types.UnionType))) (deftype :tp [#^ int A #** B] Foo int) (assert (= (ttp.show Foo) [ diff --git a/tests/native_tests/hy_eval.hy b/tests/native_tests/hy_eval.hy index 479b5907a..3abf52f35 100644 --- a/tests/native_tests/hy_eval.hy +++ b/tests/native_tests/hy_eval.hy @@ -248,7 +248,7 @@ (setv m (hy.read "(/ 1 0)" :filename "bad_math.hy")) (with [e (pytest.raises ZeroDivisionError)] (hy.eval m)) - (assert (in "bad_math.hy" (get (hy.M.traceback.format-tb e.tb) -1)))) + (assert (in "bad_math.hy" (get (hy.I.traceback.format-tb e.tb) -1)))) (defn test-eval-failure [] diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index 07a03dff4..37992d42f 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -1,5 +1,5 @@ ;; Tests of `hy.gensym`, `hy.macroexpand`, `hy.macroexpand-1`, -;; `hy.disassemble`, `hy.read`, and `hy.M` +;; `hy.disassemble`, `hy.read`, and `hy.I` (import pytest) @@ -29,7 +29,7 @@ (hy.macroexpand '(mac (a b) (mac 5))) '(a b 5))) (assert (= - (hy.macroexpand '(qplah "phooey") :module hy.M.tests.resources.tlib) + (hy.macroexpand '(qplah "phooey") :module hy.I.tests.resources.tlib) '[8 "phooey"])) (assert (= (hy.macroexpand '(chippy 1) :macros @@ -109,44 +109,44 @@ (assert (is (type (hy.read "0")) (type '0)))) -(defn test-hyM [] +(defn test-hyI [] (defmacro no-name [name] `(with [(pytest.raises NameError)] ~name)) - ; `hy.M` doesn't bring the imported stuff into scope. - (assert (= (hy.M.math.sqrt 4) 2)) - (assert (= (.sqrt (hy.M "math") 4) 2)) + ; `hy.I` doesn't bring the imported stuff into scope. + (assert (= (hy.I.math.sqrt 4) 2)) + (assert (= (.sqrt (hy.I "math") 4) 2)) (no-name math) (no-name sqrt) ; It's independent of bindings to such names. (setv math (type "Dummy" #() {"sqrt" "hello"})) - (assert (= (hy.M.math.sqrt 4) 2)) + (assert (= (hy.I.math.sqrt 4) 2)) (assert (= math.sqrt "hello")) ; It still works in a macro expansion. (defmacro frac [a b] - `(hy.M.fractions.Fraction ~a ~b)) + `(hy.I.fractions.Fraction ~a ~b)) (assert (= (* 6 (frac 1 3)) 2)) (no-name fractions) (no-name Fraction) ; You can use `/` for dotted module names. - (assert (= (hy.M.os/path.basename "foo/bar") "bar")) + (assert (= (hy.I.os/path.basename "foo/bar") "bar")) (no-name os) (no-name path) - ; `hy.M.__getattr__` attempts to cope with mangling. + ; `hy.I.__getattr__` attempts to cope with mangling. (with [e (pytest.raises ModuleNotFoundError)] - (hy.M.a-b☘c-d/e.z)) + (hy.I.a-b☘c-d/e.z)) (assert (= e.value.name (hy.mangle "a-b☘c-d"))) - ; `hy.M.__call__` doesn't. + ; `hy.I.__call__` doesn't. (with [e (pytest.raises ModuleNotFoundError)] - (hy.M "a-b☘c-d/e.z")) + (hy.I "a-b☘c-d/e.z")) (assert (= e.value.name "a-b☘c-d/e"))) -(defn test-hyM-mangle-chain [tmp-path monkeypatch] +(defn test-hyI-mangle-chain [tmp-path monkeypatch] ; We can get an object from a submodule with various kinds of ; mangling in the name chain. @@ -162,4 +162,4 @@ ; don't reload it explicitly. (import foo) (import importlib) (importlib.reload foo) - (assert (= hy.M.foo/foo?/_foo/☘foo☘/foo.foo 5))) + (assert (= hy.I.foo/foo?/_foo/☘foo☘/foo.foo 5))) diff --git a/tests/native_tests/let.hy b/tests/native_tests/let.hy index db7890b72..94f0b1122 100644 --- a/tests/native_tests/let.hy +++ b/tests/native_tests/let.hy @@ -520,7 +520,7 @@ (defmacro eval-isolated [#* body] - `(hy.eval '(do ~@body) :module (hy.M.types.ModuleType "") :locals {})) + `(hy.eval '(do ~@body) :module (hy.I.types.ModuleType "") :locals {})) (defn test-let-bound-nonlocal [] From d67299d3ca9c95a05e9311c64c6e762f8c5dd91e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 6 Nov 2023 14:11:52 -0500 Subject: [PATCH 330/342] Update NEWS --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index 8f111d4b1..2ecf76765 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -20,6 +20,7 @@ Breaking Changes * When a macro is `require`\d from another module, that module is no longer implicitly included when checking for further macros in the expansion. +* `hy.M` has been renamed to `hy.I`. * `hy.eval` has been overhauled to be more like Python's `eval`. It also has a new parameter `macros`. * `hy.macroexpand` and `hy.macroexpand-1` have been overhauled and From f4729a28cea76a617e5bd95982aafd9922f8744d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 6 Nov 2023 15:19:10 -0500 Subject: [PATCH 331/342] Spin out part of `hy.I` into `slashes2dots` --- hy/__init__.py | 7 ++----- hy/reader/mangling.py | 8 ++++++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/hy/__init__.py b/hy/__init__.py index 3a38caec5..dcd95c94c 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -24,11 +24,8 @@ def __call__(self, module_name): import importlib return importlib.import_module(module_name) def __getattr__(self, s): - import re - return self(hy.mangle(re.sub( - r'/(-*)', - lambda m: '.' + '_' * len(m.group(1)), - hy.unmangle(s)))) + from hy.reader.mangling import slashes2dots + return self(slashes2dots(s)) I = I() diff --git a/hy/reader/mangling.py b/hy/reader/mangling.py index 74b2c932b..538c38c4f 100644 --- a/hy/reader/mangling.py +++ b/hy/reader/mangling.py @@ -106,3 +106,11 @@ def unmangle(s): s = s.replace("_", "-") return prefix + s + suffix + + +def slashes2dots(s): + 'Interpret forward slashes as a substitute for periods.' + return mangle(re.sub( + r'/(-*)', + lambda m: '.' + '_' * len(m.group(1)), + unmangle(s))) From 9645421fe87b9772c9cfc1268fe290752eb97077 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 8 Nov 2023 09:06:12 -0500 Subject: [PATCH 332/342] Add `hy.R` --- NEWS.rst | 2 ++ docs/syntax.rst | 5 +++++ hy/__init__.py | 2 +- hy/macros.py | 42 ++++++++++++++++++++++------------- tests/native_tests/hy_misc.hy | 15 ++++++++++++- 5 files changed, 49 insertions(+), 17 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 2ecf76765..f35350399 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -39,6 +39,8 @@ New Features * `defn`, `defn/a`, and `defclass` now support type parameters. * `HyReader` now has an optional parameter to install existing reader macros from the calling module. +* New syntax `(hy.R.aaa/bbb.m …)` for calling the macro `m` from the + module `aaa.bbb` without bringing `m` or `aaa.bbb` into scope. * New pragma `warn-on-core-shadow`. * `nonlocal` now also works for globally defined names. diff --git a/docs/syntax.rst b/docs/syntax.rst index 54188fda5..ed4a347aa 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -400,6 +400,11 @@ first element. to construct a method call with the element after ``None`` as the object: thus, ``(.add my-set 5)`` is equivalent to ``((. my-set add) 5)``, which becomes ``my_set.add(5)`` in Python. + + .. _hy.R: + + - Exception: expressions like ``((. hy R module-name macro-name) …)``, or equivalently ``(hy.R.module-name.macro-name …)``, get special treatment. They import the module ``module-name`` and call its macro ``macro-name``, so ``(hy.R.foo.bar 1)`` is equivalent to ``(require foo) (foo.bar 1)``, but without bringing ``foo`` or ``foo.bar`` into scope. Thus ``hy.R`` is convenient syntactic sugar for macros you'll only call once in a file, or for macros that you want to appear in the expansion of other macros without having to call :hy:func:`require` in the expansion. As with :hy:class:`hy.I`, dots in the module name must be replaced with slashes. + - Otherwise, the expression is compiled into a Python-level call, with the first element being the calling object. (So, you can call a function that has the same name as a macro with an expression like ``((do setv) …)``.) The diff --git a/hy/__init__.py b/hy/__init__.py index dcd95c94c..afa43dcd4 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -17,7 +17,7 @@ def _initialize_env_var(env_var, default_val): class I: - """``hy.I`` is an object that provides syntactic sugar for imports. It allows syntax like ``(hy.I.math.sqrt 2)`` to mean ``(import math) (math.sqrt 2)``, except without bringing ``math`` or ``math.sqrt`` into scope. This is useful in macros to avoid namespace pollution. To refer to a module with dots in its name, use slashes instead: ``hy.I.os/path.basename`` gets the function ``basename`` from the module ``os.path``. + """``hy.I`` is an object that provides syntactic sugar for imports. It allows syntax like ``(hy.I.math.sqrt 2)`` to mean ``(import math) (math.sqrt 2)``, except without bringing ``math`` or ``math.sqrt`` into scope. (See :ref:`hy.R ` for a version that requires a macro instead of importing a Python object.) This is useful in macros to avoid namespace pollution. To refer to a module with dots in its name, use slashes instead: ``hy.I.os/path.basename`` gets the function ``basename`` from the module ``os.path``. You can also call ``hy.I`` like a function, as in ``(hy.I "math")``, which is useful when the module name isn't known until run-time. This interface just calls :py:func:`importlib.import_module`, avoiding (1) mangling due to attribute lookup, and (2) the translation of ``/`` to ``.`` in the module name. The advantage of ``(hy.I modname)`` over ``importlib.import_module(modname)`` is merely that it avoids bringing ``importlib`` itself into scope.""" def __call__(self, module_name): diff --git a/hy/macros.py b/hy/macros.py index 90dae35b6..0c028ccc1 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -20,6 +20,7 @@ from hy.model_patterns import whole from hy.models import Expression, Symbol, as_model, is_unpack, replace_hy_obj from hy.reader import mangle +from hy.reader.mangling import slashes2dots EXTRA_MACROS = ["hy.core.result_macros", "hy.core.macros"] @@ -383,21 +384,32 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): else: break - # Choose the first namespace with the macro. - m = ((compiler and next( - (d[fn] - for d in [ - compiler.extra_macros, - *(s['macros'] for s in reversed(compiler.local_state_stack))] - if fn in d), - None)) or - next( - (mod._hy_macros[fn] - for mod in (module, builtins) - if fn in getattr(mod, "_hy_macros", ())), - None)) - if not m: - break + if fn.startswith('hy.R.'): + # Special syntax for a one-shot `require`. + req_from, _, fn = fn[len('hy.R.'):].partition('.') + req_from = slashes2dots(req_from) + try: + m = importlib.import_module(req_from)._hy_macros[fn] + except ImportError as e: + raise HyRequireError(e.args[0]).with_traceback(None) + except (AttributeError, KeyError): + raise HyRequireError(f'Could not require name {fn} from {req_from}') + else: + # Choose the first namespace with the macro. + m = ((compiler and next( + (d[fn] + for d in [ + compiler.extra_macros, + *(s['macros'] for s in reversed(compiler.local_state_stack))] + if fn in d), + None)) or + next( + (mod._hy_macros[fn] + for mod in (module, builtins) + if fn in getattr(mod, "_hy_macros", ())), + None)) + if not m: + break with MacroExceptions(module, tree, compiler): if compiler: diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index 37992d42f..8109aa065 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -1,5 +1,5 @@ ;; Tests of `hy.gensym`, `hy.macroexpand`, `hy.macroexpand-1`, -;; `hy.disassemble`, `hy.read`, and `hy.I` +;; `hy.disassemble`, `hy.read`, `hy.I`, and `hy.R` (import pytest) @@ -163,3 +163,16 @@ (import foo) (import importlib) (importlib.reload foo) (assert (= hy.I.foo/foo?/_foo/☘foo☘/foo.foo 5))) + + +(defn test-hyR [] + (assert (= (hy.R.tests/resources/tlib.qplah "x") [8 "x"])) + (assert (= (hy.R.tests/resources/tlib.✈ "x") "plane x")) + (with [(pytest.raises NameError)] + (hy.eval '(tests.resources.tlib.qplah "x"))) + (with [(pytest.raises NameError)] + (hy.eval '(qplah "x"))) + (with [(pytest.raises hy.errors.HyRequireError)] + (hy.eval '(hy.R.tests/resources/tlib.nonexistent-macro "x"))) + (with [(pytest.raises hy.errors.HyRequireError)] + (hy.eval '(hy.R.nonexistent-module.qplah "x")))) From 1c7e0dd44c5607c05a07341e350352ca3b685545 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 9 Nov 2023 13:16:26 -0500 Subject: [PATCH 333/342] Overhaul debugging environment variables `HY_DEBUG` and `HY_FILTER_INTERNAL_ERRORS` were redundant in intention, but `HY_DEBUG` worked more reliably because `hy_exc_handler` is called explicitly in a few places. The new environment variable `HY_SHOW_INTERNAL_ERRORS` uses a more specific name while avoiding the awkwardness of how `HY_FILTER_INTERNAL_ERRORS` had to be set to an empty string to have an effect. The new environment variable is still untested, because it's only for internal use, and is documented merely for completeness. --- NEWS.rst | 2 ++ docs/env_var.rst | 9 ++------- hy/errors.py | 24 ++++++++++-------------- tests/test_bin.py | 3 --- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index f35350399..72f2b470a 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -11,6 +11,8 @@ Removals or Python's built-in `keyword` module instead. * `doc` has been removed. Use `(help (get-macro foo))` or `(help (get-macro :reader foo))` instead. +* The environment variables `HY_DEBUG` and `HY_FILTER_INTERNAL_ERRORS` + have been replaced with `HY_SHOW_INTERNAL_ERRORS`. Breaking Changes ------------------------------ diff --git a/docs/env_var.rst b/docs/env_var.rst index 8053efbaf..202911153 100644 --- a/docs/env_var.rst +++ b/docs/env_var.rst @@ -11,14 +11,9 @@ set to anything else. (Default: nothing) Path to a file containing Hy source code to execute when starting the REPL. -.. envvar:: HY_DEBUG +.. envvar:: HY_SHOW_INTERNAL_ERRORS - (Default: false) Does something mysterious that's probably similar to - ``HY_FILTER_INTERNAL_ERRORS``. - -.. envvar:: HY_FILTER_INTERNAL_ERRORS - - (Default: true) Whether to hide some parts of tracebacks that point to + (Default: false) Whether to show some parts of tracebacks that point to internal Hy code and won't be helpful to the typical Hy user. .. envvar:: HY_HISTORY diff --git a/hy/errors.py b/hy/errors.py index c594e1dcd..3bba6f1f3 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -8,7 +8,7 @@ from hy import _initialize_env_var from hy._compat import PYPY -_hy_filter_internal_errors = _initialize_env_var("HY_FILTER_INTERNAL_ERRORS", True) +_hy_show_internal_errors = _initialize_env_var("HY_SHOW_INTERNAL_ERRORS", False) class HyError(Exception): @@ -270,11 +270,11 @@ def hy_exc_filter(exc_type, exc_value, exc_traceback): def hy_exc_handler(exc_type, exc_value, exc_traceback): """A `sys.excepthook` handler that uses `hy_exc_filter` to - remove internal Hy frames from a traceback print-out. + remove internal Hy frames from a traceback print-out, so long + as `_hy_show_internal_errors` is false. """ - if os.environ.get("HY_DEBUG", False): + if _hy_show_internal_errors: return sys.__excepthook__(exc_type, exc_value, exc_traceback) - try: output = hy_exc_filter(exc_type, exc_value, exc_traceback) sys.stderr.write(output) @@ -289,14 +289,10 @@ def filtered_hy_exceptions(): from tracebacks. Filtering can be controlled by the variable - `hy.errors._hy_filter_internal_errors` and environment variable - `HY_FILTER_INTERNAL_ERRORS`. + `hy.errors._hy_show_internal_errors` and environment variable + `HY_SHOW_INTERNAL_ERRORS`. """ - global _hy_filter_internal_errors - if _hy_filter_internal_errors: - current_hook = sys.excepthook - sys.excepthook = hy_exc_handler - yield - sys.excepthook = current_hook - else: - yield + current_hook = sys.excepthook + sys.excepthook = hy_exc_handler + yield + sys.excepthook = current_hook diff --git a/tests/test_bin.py b/tests/test_bin.py index f35177a6a..364c2cf27 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -410,9 +410,6 @@ def test_macro_require(): def test_tracebacks(): """Make sure the printed tracebacks are correct.""" - # We want the filtered tracebacks. - os.environ["HY_DEBUG"] = "" - def req_err(x): assert x == "hy.errors.HyRequireError: No module named 'not_a_real_module'" From 606f98a8608e8ef7c3b3a54e8d1e1b32ddb023bc Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 10 Nov 2023 10:40:49 -0500 Subject: [PATCH 334/342] Make the magic parameter of macros optional A way to refer back to the compiler is only needed for Hy's internal use. This change relieves the need for the user to include this parameter when defining macros dynamically with e.g. `fn`. --- hy/macros.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hy/macros.py b/hy/macros.py index 0c028ccc1..e32566163 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -414,7 +414,13 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): with MacroExceptions(module, tree, compiler): if compiler: compiler.this = tree - obj = m(compiler, *tree[1:]) + obj = m( + *([compiler] + if m.__code__.co_varnames and m.__code__.co_varnames[0] in ( + 'compiler', 'hy_compiler', 'hyx_XampersandXcompiler', + 'ETname') + else []), + *tree[1:]) if isinstance(obj, (hy.compiler.Result, AST)): return obj if result_ok else tree From e9af9769b7c1b5851edf317b83311c3433e9824d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 10 Nov 2023 09:46:45 -0500 Subject: [PATCH 335/342] Remove some now-unnecessary params in tests --- tests/macros/test_macro_processor.py | 2 +- tests/native_tests/hy_eval.hy | 6 +++--- tests/native_tests/hy_misc.hy | 2 +- tests/native_tests/macros_first_class.hy | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py index 1e4c452e9..d870a4d24 100644 --- a/tests/macros/test_macro_processor.py +++ b/tests/macros/test_macro_processor.py @@ -8,7 +8,7 @@ @macro("test") -def tmac(ETname, *tree): +def tmac(*tree): """Turn an expression into a list""" return List(tree) diff --git a/tests/native_tests/hy_eval.hy b/tests/native_tests/hy_eval.hy index 3abf52f35..c69267432 100644 --- a/tests/native_tests/hy_eval.hy +++ b/tests/native_tests/hy_eval.hy @@ -211,7 +211,7 @@ (assert (= (hy.eval '(chippy a b) :macros (dict - :chippy (fn [&compiler arg1 arg2] + :chippy (fn [arg1 arg2] (hy.models.Symbol (+ (str arg1) (str arg2)))))) 15)) @@ -233,13 +233,13 @@ (hy.eval '(cheese)) "gorgonzola")) (assert (= - (hy.eval '(cheese) :macros {"cheese" (fn [&compiler] "cheddar")}) + (hy.eval '(cheese) :macros {"cheese" (fn [] "cheddar")}) "cheddar")) ; Or even a core macro, and with no warning. (assert (= (hy.eval '(+ 1 1) :macros - {(hy.mangle "+") (fn [&compiler #* args] + {(hy.mangle "+") (fn [#* args] (.join "" (gfor x args (str (int x)))))}) "11"))) diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index 8109aa065..4b230aa91 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -33,7 +33,7 @@ '[8 "phooey"])) (assert (= (hy.macroexpand '(chippy 1) :macros - {"chippy" (fn [&compiler x] `[~x ~x])}) + {"chippy" (fn [x] `[~x ~x])}) '[1 1])) ; Non-Expressions just get returned as-is. (defn f []) diff --git a/tests/native_tests/macros_first_class.hy b/tests/native_tests/macros_first_class.hy index 7245a183c..aaf3aa345 100644 --- a/tests/native_tests/macros_first_class.hy +++ b/tests/native_tests/macros_first_class.hy @@ -33,9 +33,9 @@ deleting them, and retrieving their docstrings. We also test `get-macro` ; 2. `require` in global scope (require tests.resources.tlib [qplah :as global2]) ; 3. Manually updating `_hy_macros` -(eval-and-compile (setv (get _hy_macros "global3") (fn [&compiler] +(eval-and-compile (setv (get _hy_macros "global3") (fn [] "from global3"))) -(eval-and-compile (setv (get _hy_macros (hy.mangle "global☘")) (fn [&compiler] +(eval-and-compile (setv (get _hy_macros (hy.mangle "global☘")) (fn [] "global☘ docstring" "from global☘"))) From fdf7d83d79860647c3f2f683fe0d9739217f104c Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 10 Nov 2023 09:51:00 -0500 Subject: [PATCH 336/342] In `defmacro`, make the magic parameter explicit --- hy/core/macros.hy | 6 +++--- hy/core/result_macros.py | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 4eab02e17..8b2a456ed 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -50,7 +50,7 @@ (return panic))]] `(if ~test (do ~@body) None)) -(defmacro defreader [key #* body] +(defmacro defreader [&compiler key #* body] "Define a new reader macro. Reader macros are expanded at read time and allow you to modify the behavior @@ -113,7 +113,7 @@ (get _hy_reader_macros ~dispatch-key))))) -(defmacro get-macro [arg1 [arg2 None]] +(defmacro get-macro [&compiler arg1 [arg2 None]] "Get the function object used to implement a macro. This works for all sorts of macros: core macros, global (i.e., module-level) macros, local macros, and reader macros. For regular (non-reader) macros, ``get-macro`` is called with one argument, a symbol or string literal, which can be premangled or not according to taste. For reader macros, this argument must be preceded by the literal keyword ``:reader`` (and note that the hash mark, ``#``, is not included in the name of the reader macro). :: (get-macro my-macro) @@ -143,7 +143,7 @@ name))))) -(defmacro local-macros [] +(defmacro local-macros [&compiler] #[[Expands to a dictionary mapping the mangled names of local macros to the function objects used to implement those macros. Thus, ``local-macros`` provides a rough local equivalent of ``_hy_macros``. :: (defn f [] diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index de12285f0..994ee5978 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -1544,10 +1544,7 @@ def E(*x): return Expression(x) S = Symbol compiler.warn_on_core_shadow(name) - fn_def = E( - S("fn"), - List([S("&compiler"), *expr[2]]), - *body).replace(expr) + fn_def = E(S("fn"), List(expr[2]), *body).replace(expr) if compiler.is_in_local_state(): # We're in a local scope, so define the new macro locally. state = compiler.local_state_stack[-1] From cefe45c89ea6f3ef2e13f71132ec8235492ee1cf Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 10 Nov 2023 10:35:11 -0500 Subject: [PATCH 337/342] Name the magic macro param `_hy_compiler` --- hy/core/macros.hy | 22 +++++++++++----------- hy/macros.py | 19 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 8b2a456ed..3732f65d5 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -50,7 +50,7 @@ (return panic))]] `(if ~test (do ~@body) None)) -(defmacro defreader [&compiler key #* body] +(defmacro defreader [_hy_compiler key #* body] "Define a new reader macro. Reader macros are expanded at read time and allow you to modify the behavior @@ -89,9 +89,9 @@ See the :ref:`reader macros docs ` for more detailed information on how reader macros work and are defined. " - (when (not (isinstance &compiler.scope hy.scoping.ScopeGlobal)) - (raise (&compiler._syntax-error - &compiler.this + (when (not (isinstance _hy_compiler.scope hy.scoping.ScopeGlobal)) + (raise (_hy_compiler._syntax-error + _hy_compiler.this f"Cannot define reader macro outside of global scope."))) (when (not (isinstance key hy.models.Symbol)) @@ -113,7 +113,7 @@ (get _hy_reader_macros ~dispatch-key))))) -(defmacro get-macro [&compiler arg1 [arg2 None]] +(defmacro get-macro [_hy_compiler arg1 [arg2 None]] "Get the function object used to implement a macro. This works for all sorts of macros: core macros, global (i.e., module-level) macros, local macros, and reader macros. For regular (non-reader) macros, ``get-macro`` is called with one argument, a symbol or string literal, which can be premangled or not according to taste. For reader macros, this argument must be preceded by the literal keyword ``:reader`` (and note that the hash mark, ``#``, is not included in the name of the reader macro). :: (get-macro my-macro) @@ -131,9 +131,9 @@ [(hy.mangle arg1) False])) (setv namespace (if reader? "_hy_reader_macros" "_hy_macros")) (cond - (and (not reader?) (setx local (.get (_local-macros &compiler) name))) + (and (not reader?) (setx local (.get (_local-macros _hy_compiler) name))) local - (in name (getattr &compiler.module namespace {})) + (in name (getattr _hy_compiler.module namespace {})) `(get ~(hy.models.Symbol namespace) ~name) (in name (getattr builtins namespace {})) `(get (. hy.I.builtins ~(hy.models.Symbol namespace)) ~name) @@ -143,7 +143,7 @@ name))))) -(defmacro local-macros [&compiler] +(defmacro local-macros [_hy_compiler] #[[Expands to a dictionary mapping the mangled names of local macros to the function objects used to implement those macros. Thus, ``local-macros`` provides a rough local equivalent of ``_hy_macros``. :: (defn f [] @@ -156,12 +156,12 @@ The equivalency is rough in the sense that ``local-macros`` returns a literal dictionary, not a preexisting object that Hy uses for resolving macro names. So, modifying the dictionary will have no effect. See also :hy:func:`get-macro `.]] - (_local-macros &compiler)) + (_local-macros _hy_compiler)) -(defn _local_macros [&compiler] +(defn _local_macros [_hy_compiler] (setv seen #{}) (dfor - state &compiler.local_state_stack + state _hy_compiler.local_state_stack m (get state "macros") :if (not-in m seen) :do (.add seen m) diff --git a/hy/macros.py b/hy/macros.py index e32566163..9f4f77ce5 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -43,18 +43,18 @@ def pattern_macro(names, pattern, shadow=None): def dec(fn): def wrapper_maker(name): - def wrapper(hy_compiler, *args): + def wrapper(_hy_compiler, *args): if shadow and any(is_unpack("iterable", x) for x in args): # Try a shadow function call with this name instead. return Expression( [Expression(map(Symbol, [".", "hy", "pyops", name])), *args] - ).replace(hy_compiler.this) + ).replace(_hy_compiler.this) - expr = hy_compiler.this + expr = _hy_compiler.this if py_version_required and sys.version_info < py_version_required: - raise hy_compiler._syntax_error( + raise _hy_compiler._syntax_error( expr, "`{}` requires Python {} or later".format( name, ".".join(map(str, py_version_required)) @@ -64,13 +64,13 @@ def wrapper(hy_compiler, *args): try: parse_tree = pattern.parse(args) except NoParseError as e: - raise hy_compiler._syntax_error( + raise _hy_compiler._syntax_error( expr[min(e.state.pos + 1, len(expr) - 1)], "parse error for pattern macro '{}': {}".format( name, e.msg.replace("end of input", "end of macro call") ), ) - return fn(hy_compiler, expr, name, *parse_tree) + return fn(_hy_compiler, expr, name, *parse_tree) return wrapper @@ -415,10 +415,11 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): if compiler: compiler.this = tree obj = m( + # If the macro's first parameter is named + # `_hy_compiler`, pass in the current compiler object + # in its place. *([compiler] - if m.__code__.co_varnames and m.__code__.co_varnames[0] in ( - 'compiler', 'hy_compiler', 'hyx_XampersandXcompiler', - 'ETname') + if m.__code__.co_varnames[:1] == ('_hy_compiler',) else []), *tree[1:]) if isinstance(obj, (hy.compiler.Result, AST)): From 8c09116bdfe070a90d55215b4344cf159ada1df8 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 12 Nov 2023 12:55:16 -0500 Subject: [PATCH 338/342] Overhaul the manual's chapter on macros --- docs/api.rst | 41 +------- docs/macros.rst | 249 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 186 insertions(+), 104 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 69344d4d0..c67c69432 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,6 +1,7 @@ API === +.. _core-macros: Core Macros ----------- @@ -1308,6 +1309,8 @@ base names, such that ``hy.core.macros.foo`` can be called as just ``foo``. Only one pragma is currently implemented: +.. _warn-on-core-shadow: + - ``:warn-on-core-shadow``: If true (the default), :hy:func:`defmacro` and :hy:func:`require` will raise a warning at compile-time if you define a macro with the same name as a core macro. Shadowing a core macro in this fashion is @@ -1371,47 +1374,9 @@ the following methods .. hy:autoclass:: hy.I -.. _reader-macros: - Reader Macros ------------- -Reader macros allow one to hook directly into Hy's reader to customize how -different forms are parsed. Reader macros can be imported from other libraries -using :hy:func:`require`, and can be defined directly using -:hy:func:`defreader`. - -Like regular macros, reader macros should return a Hy form that will then be -passed to the compiler for execution. Reader macros access the Hy reader using -the ``&reader`` name. It gives access to all of the text- and form-parsing logic -that Hy uses to parse itself. See :py:class:`HyReader ` and -its base class :py:class:`Reader ` for details regarding -the available processing methods. - -Note that Hy reads and parses each top-level form completely before it is executed, -so the following code will throw an exception: - -.. code-block:: hylang - - => (do - ... (defreader up - ... (.slurp-space &reader) - ... (.upper (.read-one-form &reader))) - ... (print #up "hello?")) - ;; !! ERROR reader macro '#up' is not defined - -Since the entire ``do`` block is read at once, the ``defreader`` will not have -yet been evaluated when the parser encounters the call to ``#up``. However, if -the reader macro isn't used until a later top-level form, then it will work: - -.. code-block:: hylang - - => (defreader up - ... (.slurp-space &reader) - ... (.upper (.read-one-form &reader))) - => (print #up "hy there!") - HY THERE! - .. autoclass:: hy.reader.hy_reader.HyReader :members: parse, parse_one_form, parse_forms_until, read_default, fill_pos diff --git a/docs/macros.rst b/docs/macros.rst index 5ba82f46d..693bc7e98 100644 --- a/docs/macros.rst +++ b/docs/macros.rst @@ -2,94 +2,211 @@ Macros ====== -Operations on macros +Macros, and the metaprogramming they enable, are one of the characteristic features of Lisp, and one of the main advantages Hy offers over vanilla Python. Much of the material covered in this chapter will be familiar to veterans of other Lisps, but there are also a lot of Hyly specific details. + +What are macros for? -------------------- -The currently defined global macros can be accessed as a dictionary ``_hy_macros`` in each module. (Use ``bulitins._hy_macros``, attached to Python's usual :py:mod:`builtins` module, to see core macros.) The keys are mangled macro names and the values are the function objects that implement the macros. You can operate on this dictionary to list, add, delete, or get help on macros, but be sure to use :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile` when you need the effect to happen at compile-time. You can call :hy:func:`local-macros ` to list local macros, but adding or deleting elements in this case is ineffective. The core macro :hy:func:`get-macro ` provides some syntactic sugar. :: +The gist of `metaprogramming +`_ is that it allows you to program the programming language itself (hence the word). You can create new control structures, like :ref:`do-while `, or other kinds of new syntax, like a concise literal notation for your favorite data structure. You can also modify how existing syntax is understood within a region of code, as by making identifiers that start with a capital letter implicitly imported from a certain module. Finally, metaprogramming can improve performance in some cases by effectively inlining functions, or by computing something once at compile-time rather than several times at run-time. With a Lisp-like macro system, you can metaprogram in a slicker and less error-prone way than generating code as text with conventional string formatting, or with lexer-level macros like those provided by the C preprocessor. - (defmacro m [] - "This is a docstring." - `(print "Hello, world.")) - (print (in "m" _hy_macros)) ; => True - (help (get-macro m)) - (m) ; => "Hello, world." - (eval-and-compile - (del (get _hy_macros "m"))) - (m) ; => NameError +Types of macros +--------------- -``_hy_reader_macros`` is a dictionary like ``_hy_macros`` for reader macros, but here, the keys aren't mangled. +Hy offers two types of macros: regular macros and reader macros. -.. _using-gensym: +**Regular macros**, typically defined with :hy:func:`defmacro`, are the kind Lispers usually mean when they talk about "macros". Regular macros are called like a function, with an :ref:`expression ` whose head is the macro name: for example, ``(foo a b)`` could call a macro named ``foo``. A regular macro is called at compile-time, after the entire top-level form in which it appears is parsed, and receives parsed :ref:`models ` as arguments. Regular macros come in :ref:`three varieties, which vary in scope `. -Using gensym for Safer Macros ------------------------------ +**Reader macros**, typically defined with :hy:func:`defreader`, are lower-level than regular macros. They're called with the hash sign ``#``; for example, ``#foo`` calls a reader macro named ``foo``. A reader macro is called at parse-time. It doesn't receive conventional arguments. Instead, it uses an implicitly available parser object to parse the subsequent source text. When it returns, the standard Hy parser picks up where it left off. -When writing macros, one must be careful to avoid capturing external variables -or using variable names that might conflict with user code. +Related constructs +~~~~~~~~~~~~~~~~~~ -We will use an example macro ``nif`` (see http://letoverlambda.com/index.cl/guest/chap3.html#sec_5 -for a more complete description.) ``nif`` is an example, something like a numeric ``if``, -where based on the expression, one of the 3 forms is called depending on if the -expression is positive, zero or negative. +There are three other constructs that perform compile-time processing much like macros, and hence are worth mentioning here. -A first pass might be something like:: +- :hy:func:`do-mac` is essentially shorthand for defining and then immediately calling a regular macro with no arguments. +- :hy:func:`eval-when-compile` evaluates some code at compile-time, but contributes no code to the final program, like a macro that returns ``None`` in a context where the ``None`` doesn't do anything. +- :hy:func:`eval-and-compile` evaluates some code at compile-time, like :hy:func:`eval-when-compile`, but also leaves the same code to be re-evaluated at run-time. - (defmacro nif [expr pos-form zero-form neg-form] - `(do - (setv obscure-name ~expr) - (cond (> obscure-name 0) ~pos-form - (= obscure-name 0) ~zero-form - (< obscure-name 0) ~neg-form))) +When to use what +~~~~~~~~~~~~~~~~ + +The variety of options can be intimidating. In addition to all of Hy's features listed above, Python is a dynamic programming language that allows you to do a lot of things at run-time that other languages would blanch at. For example, you can dynamically define a new class by calling :class:`type`. So, watch out for cases where your first thought is to use a macro, but you don't actually need one. + +When deciding what to use, a good rule of thumb is to use the least powerful option that suffices for the syntax, semantics, and performance that you want. So first, see if Python's dynamic features are enough. If they aren't, try a macro-like construct or a regular macro. If even those aren't enough, try a reader macro. Using the least powerful applicable option will help you avoid the :ref:`macro pitfalls described below `, as well as other headaches such as wanting to use a macro where a Python API needs a function. (For the sake of providing simpler examples, much of the below discussion will ignore this advice and consider macros that could easily be written as functions.) + +The basics +---------- + +A regular macro can be defined with :hy:func:`defmacro` using a syntax similar to that of :hy:func:`defn`. Here's how you could define and call a trivial macro that takes no arguments and returns a constant:: + + (defmacro seventeen [] + 17) + + (print (seventeen)) + +To see that ``seventeen`` is expanded at compile-time, run ``hy2py`` on this script and notice that it ends with ``print(17)`` rather than ``print(seventeen())``. If you insert a ``print`` call inside the macro definition, you'll also see that the print happens when the file is compiled, but not when it's rerun (provided an up-to-date bytecode file exists). + +A more useful macro returns code. You can construct a model the long way, like this:: + + (defmacro addition [] + (hy.models.Expression [ + (hy.models.Symbol "+") + (hy.models.Integer 1) + (hy.models.Integer 1)])) + +or more concisely with :hy:func:`quote`, like this:: + + (defmacro addition [] + '(+ 1 1)) + +You don't need to always return a model because the compiler calls :hy:func:`hy.as-model` on everything before trying to compile it. Thus, the ``17`` above works fine in place of ``(hy.models.Integer 17)``. But trying to compile something that ``hy.as-model`` chokes on, like a function object, is an error. + +Arguments are always passed in as models. You can use quasiquotation (see :hy:func:`quasiquote`) to concisely define a model with partly literal and partly evaluated components:: + + (defmacro set-to-2 [variable] + `(setv ~variable 2)) + (set-to-2 foobar) + (print foobar) + +Macros don't understand keyword arguments like functions do. Rather, the :ref:`keyword objects ` themselves are passed in literally. This gives you flexibility in how to handle them. Thus, ``#** kwargs`` and ``*`` aren't allowed in the parameter list of a macro, although ``#* args`` and ``/`` are. + +On the inside, macros are functions, and obey the usual Python semantics for functions. For example, :hy:func:`setv` inside a macro will define or modify a variable local to the current macro call, and :hy:func:`return` ends macro execution and uses its argument as the expansion. + +Macros from other modules can be brought into the current scope with :hy:func:`require`. + +.. _macro-pitfalls: + +Pitfalls +-------- + +Macros are powerful, but with great power comes great potential for anguish. There are a few characteristic issues you need to guard against to write macros well, and, to a lesser extent, even to use macros well. -where ``obscure-name`` is an attempt to pick some variable name as not to -conflict with other code. But of course, while well-intentioned, -this is no guarantee. +Name games +~~~~~~~~~~ -The method :hy:func:`gensym ` is designed to generate a new, unique symbol for just -such an occasion. A much better version of ``nif`` would be:: +A lot of these issues are variations on the theme of names not referring to what you intend them to, or in other words, surprise shadowing. For example, the macro below was intended to define a new variable named ``x``, but it ends up modifying a preexisting variable. :: - (defmacro nif [expr pos-form zero-form neg-form] + (defmacro upper-twice [arg] + `(do + (setv x (.upper ~arg)) + (+ x x))) + (setv x "Okay guys, ") + (setv salutation (upper-twice "bye")) + (print (+ x salutation)) + ; Intended result: "Okay guys, BYEBYE" + ; Actual result: "BYEBYEBYE" + +If you avoid the assignment entirely, by using an argument more than once, you can cause a different problem: surprise multiple evaluation. :: + + (defmacro upper-twice [arg] + `(+ (.upper ~arg) (.upper ~arg))) + (setv items ["a" "b" "c"]) + (print (upper-twice (.pop items))) + ; Intended result: "CC" + ; Actual result: "CB" + +A better approach is to use :hy:func:`hy.gensym` to choose your variable name:: + + (defmacro upper-twice [arg] (setv g (hy.gensym)) `(do - (setv ~g ~expr) - (cond (> ~g 0) ~pos-form - (= ~g 0) ~zero-form - (< ~g 0) ~neg-form))) + (setv ~g (.upper ~arg)) + (+ ~g ~g))) + +Hyrule provides some macros that make using gensyms more convenient, like :hy:func:`defmacro! ` and :hy:func:`with-gensyms `. + +Macro subroutines +~~~~~~~~~~~~~~~~~ + +A case where you could want something to be in the scope of a macro's expansion, and then it turns out not to be, is when you want to call a function or another macro in the expansion:: + + (defmacro hypotenuse [a b] + (import math) + `(math.sqrt (+ (** ~a 2) (** ~b 2)))) + (print (hypotenuse 3 4)) + ; NameError: name 'math' is not defined + +The form ``(import math)`` here appears in the wrong context, in the macro call itself rather than the expansion. You could use ``import`` or ``require`` to bind the module name or one of its members to a gensym, but an often more convenient option is to use the one-shot import syntax :hy:class:`hy.I` or the one-shot require syntax :ref:`hy.R `:: + + (defmacro hypotenuse [a b] + `(hy.I.math.sqrt (+ (** ~a 2) (** ~b 2)))) + (hypotenuse 3 4) + +A related but distinct issue is when you want to use a function (or other ordinary Python object) in a macro's code, but it isn't available soon enough:: + + (defn subroutine [x] + (hy.models.Symbol (.upper x))) + (defmacro uppercase-symbol [x] + (subroutine x)) + (setv (uppercase-symbol foo) 1) + ; NameError: name 'subroutine' is not defined + +Here, ``subroutine`` is only defined at run-time, so ``uppercase-symbol`` can't see it when it's expanding (unless you happen to be calling ``uppercase-symbol`` from a different module). This is easily worked around by wrapping ``(defn subroutine …)`` in :hy:func:`eval-and-compile` (or :hy:func:`eval-when-compile` if you want ``subroutine`` to be invisible at run-time). + +By the way, despite the need for ``eval-and-compile``, extracting a lot of complex logic out of a macro into a function is often a good idea. Functions are typically easier to debug and to make use of in other macros. + +The important take-home big fat WARNING +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Ultimately it's wisest to use only four kinds of names in macro expansions: gensyms, core macros, objects that Python puts in scope by default (like its built-in functions), and ``hy`` and its attributes. It's possible to rebind nearly all these names, so surprise shadowing is still theoretically possible. Unfortunately, the only way to prevent these pathological rebindings from coming about is… don't do that. Don't make a new macro named ``setv`` or name a function argument ``type`` unless you're ready for every macro you call to break, the same way you wouldn't monkey-patch a built-in Python module without thinking carefully. This kind of thing is the responsibility of the macro caller; the macro writer can't do much to defend against it. There is at least a pragma :ref:`warn-on-core-shadow `, enabled by default, that causes ``defmacro`` and ``require`` to warn you if you give your new macro the same name as a core macro. + +.. _reader-macros: + +Reader macros +------------- -This is an easy case, since there is only one symbol. But if there is -a need for several gensym's there is a second macro :hy:func:`with-gensyms ` that -basically expands to a ``setv`` form:: +Reader macros allow you to hook directly into Hy's parser to customize how text is parsed into models. They're defined with :hy:func:`defreader`, or, like regular macros, brought in from other modules with :hy:func:`require`. Rather than receiving function arguments, a reader macro has access to a :py:class:`HyReader ` object named ``&reader``, which provides all the text-parsing logic that Hy uses to parse itself (see :py:class:`HyReader ` and its base class :py:class:`Reader ` for the available methods). A reader macro is called with the hash sign ``#``, and like a regular macro, it should return a model or something convertible to a model. :: - (with-gensyms [a b c] - ...) + (defreader matrix + (.slurp-space &reader) + (setv start (.getc &reader)) + (assert (= start "[")) + (setv out [[]]) + (while (not (do (.slurp-space &reader) (.peek-and-getc &reader "]"))) + (setv x (.parse-one-form &reader)) + (if (= x '|) + (.append out []) + (.append (get out -1) x))) + (if (= out [[]]) [] out)) -expands to:: + (print (hy.repr #matrix [1 2 3 | 4 5 6 | 7 8 9])) + ; => [[1 2 3] [4 5 6] [7 8 9]] + +Note that because reader macros are evaluated at parse-time, and top-level forms are completely parsed before any further compile-time execution occurs, you can't use a reader macro in the same top-level form that defines it:: (do - (setv a (hy.gensym) - b (hy.gensym) - c (hy.gensym)) - ...) + (defreader up + (.slurp-space &reader) + (.upper (.read-one-form &reader))) + (print #up "hello?")) + ; LexException: reader macro '#up' is not defined -so our re-written ``nif`` would look like:: +.. _macro-namespaces: - (defmacro nif [expr pos-form zero-form neg-form] - (with-gensyms [g] - `(do - (setv ~g ~expr) - (cond (> ~g 0) ~pos-form - (= ~g 0) ~zero-form - (< ~g 0) ~neg-form)))) +Macro namespaces and operations on macros +----------------------------------------- -Finally, though we can make a new macro that does all this for us. :hy:func:`defmacro/g! ` -will take all symbols that begin with ``g!`` and automatically call ``gensym`` with the -remainder of the symbol. So ``g!a`` would become ``(hy.gensym "a")``. +Macros don't share namespaces with ordinary Python objects. That's why something like ``(defmacro m []) (print m)`` fails with a ``NameError``, and how :hy:mod:`hy.pyops` can provide a function named ``+`` without hiding the core macro ``+``. -Our final version of ``nif``, built with ``defmacro/g!`` becomes:: +There are three scoped varieties of regular macro. First are **core macros**, which are built into Hy; :ref:`the set of core macros ` is fixed. They're available by default. You can inspect them in the dictionary ``bulitins._hy_macros``, which is attached to Python's usual :py:mod:`builtins` module. The keys are strings giving :ref:`mangled ` names and the values are the function objects implementing the macros. - (defmacro/g! nif [expr pos-form zero-form neg-form] - `(do - (setv ~g!res ~expr) - (cond (> ~g!res 0) ~pos-form - (= ~g!res 0) ~zero-form - (< ~g!res 0) ~neg-form))) +**Global macros** are associated with modules, like Python global variables. They're defined when you call ``defmacro`` or ``require`` in a global scope. You can see them in the global variable ``_hy_macros`` associated with the same module. You can manipulate ``_hy_macros`` to list, add, delete, or get help on macros, but be sure to use :hy:func:`eval-and-compile` or :hy:func:`eval-when-compile` when you need the effect to happen at compile-time, which is often. (Modifying ``bulitins._hy_macros`` is of course a risky proposition.) Here's an example, which also demonstrates the core macro :hy:func:`get-macro `. ``get-macro`` provides syntactic sugar for getting all sorts of macros as objects. :: + + (defmacro m [] + "This is a docstring." + `(print "Hello, world.")) + (print (in "m" _hy_macros)) ; => True + (help (get-macro m)) + (m) ; => "Hello, world." + (eval-and-compile + (del (get _hy_macros "m"))) + (m) ; => NameError + (eval-and-compile + (setv (get _hy_macros (hy.mangle "new-mac")) (fn [] + '(print "Goodbye, world.")))) + (new-mac) ; => "Goodbye, world." + +**Local macros** are associated with function, class, or comprehension scopes, like Python local variables. They come about when you call ``defmacro`` or ``require`` in an appropriate scope. You can call :hy:func:`local-macros ` to view local macros, but adding or deleting elements is ineffective. + +Finally, ``_hy_reader_macros`` is a per-module dictionary like ``_hy_macros`` for reader macros, but here, the keys aren't mangled. There are no local reader macros, and there's no official way to introspect on Hy's handful of core reader macros. So, of the three scoped varieties of regular macro, reader macros most resemble global macros. From b48b2b5b25e90abfd2ba6b0b3df75ab8c948aea5 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Wed, 15 Nov 2023 13:10:44 -0500 Subject: [PATCH 339/342] Adjust where `as_model` is called in `macroexpand` This produces more consistent semantics that will better match the documentation. --- hy/core/result_macros.py | 6 +++--- hy/macros.py | 37 ++++++++--------------------------- tests/native_tests/hy_misc.hy | 5 +++++ 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py index 994ee5978..7f82948c1 100644 --- a/hy/core/result_macros.py +++ b/hy/core/result_macros.py @@ -223,7 +223,7 @@ def render_quoted_form(compiler, form, level): body = [List(contents)] if isinstance(form, FString) and form.brackets is not None: - body.extend([Keyword("brackets"), form.brackets]) + body.extend([Keyword("brackets"), String(form.brackets)]) elif isinstance(form, FComponent) and form.conversion is not None: body.extend([Keyword("conversion"), String(form.conversion)]) @@ -235,7 +235,7 @@ def render_quoted_form(compiler, form, level): elif isinstance(form, String): if form.brackets is not None: - body.extend([Keyword("brackets"), form.brackets]) + body.extend([Keyword("brackets"), String(form.brackets)]) return (Expression([dotted("hy.models." + name), *body]).replace(form), False) @@ -1862,7 +1862,7 @@ def compile_require(compiler, expr, root, entries): ( String("EXPORTS") if assignments == "EXPORTS" - else [[String(k), String(v)] for k, v in assignments] + else List([List([String(k), String(v)]) for k, v in assignments]) ), Keyword("prefix"), String(prefix), diff --git a/hy/macros.py b/hy/macros.py index 9f4f77ce5..0bbe87a56 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -342,33 +342,13 @@ def __exit__(self, exc_type, exc_value, exc_traceback): def macroexpand(tree, module, compiler=None, once=False, result_ok=True): - """Expand the toplevel macros for the given Hy AST tree. - - Load the macros from the given `module`, then expand the (top-level) macros - in `tree` until we no longer can. This doesn't work on local macros. - - `Expression` resulting from macro expansions are assigned the module in - which the macro function is defined (determined using `inspect.getmodule`). - If the resulting `Expression` is itself macro expanded, then the namespace - of the assigned module is checked first for a macro corresponding to the - expression's head/car symbol. If the head/car symbol of such a `Expression` - is not found among the macros of its assigned module's namespace, the - outer-most namespace--e.g. the one given by the `module` parameter--is used - as a fallback. - - Args: - tree (Union[Object, list]): Hy AST tree. - module (Union[str, ModuleType]): Module used to determine the local - namespace for macros. - compiler (Optional[HyASTCompiler] ): The compiler object passed to - expanded macros. Defaults to None - once (bool): Only expand the first macro in `tree`. Defaults to False - result_ok (bool): Whether or not it's okay to return a compiler `Result` instance. - Defaults to True. - - Returns: - Union[Object, Result]: A mutated tree with macros expanded. - """ + '''If `tree` isn't an `Expression` that might be a macro call, + return it unchanged. Otherwise, try to expand it. Do this + repeatedly unless `once` is true. Call `as_model` after each + expansion. If the return value is a compiler `Result` object, and + `result_ok` is false, return the previous value. Otherwise, return + the final expansion.''' + if not inspect.ismodule(module): module = importlib.import_module(module) @@ -421,7 +401,7 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): *([compiler] if m.__code__.co_varnames[:1] == ('_hy_compiler',) else []), - *tree[1:]) + *map(as_model, tree[1:])) if isinstance(obj, (hy.compiler.Result, AST)): return obj if result_ok else tree @@ -430,7 +410,6 @@ def macroexpand(tree, module, compiler=None, once=False, result_ok=True): if once: break - tree = as_model(tree) return tree diff --git a/tests/native_tests/hy_misc.hy b/tests/native_tests/hy_misc.hy index 4b230aa91..a75d63c20 100644 --- a/tests/native_tests/hy_misc.hy +++ b/tests/native_tests/hy_misc.hy @@ -40,6 +40,11 @@ (assert (is (hy.macroexpand f) f)) + ; Likewise Expressions that aren't macro calls. + (setv model '(wmbatt 1 2)) + (assert (is + (hy.macroexpand model) + model)) ; If the macro expands to a `Result`, the user gets the original ; back instead of the `Result`. (setv model '(+ 1 1)) From 304cee6a40c6c48a325d91c6657e2ce53a3f5a66 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 16 Nov 2023 12:39:49 -0500 Subject: [PATCH 340/342] Add a section "What Hy is not" to the manual --- docs/whyhy.rst | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/whyhy.rst b/docs/whyhy.rst index e2c8f50da..861823292 100644 --- a/docs/whyhy.rst +++ b/docs/whyhy.rst @@ -150,8 +150,20 @@ aforementioned mixing of statements and expressions, :ref:`name mangling Python-legal identifiers, and a :hy:func:`let` macro to provide block-level scoping in place of Python's usual function-level scoping. -Overall, Hy, like Common Lisp, is intended to be an unopinionated big-tent -language that lets you do what you want. If you're interested in a more -small-and-beautiful approach to Lisp, in the style of Scheme, check out -`Hissp `_, another Lisp embedded in Python -that was created by a Hy developer. + +What Hy is not +-------------- + +Hy isn't minimal or elegant. Hy is big and ugly and proud of it; it's an +unopinionated big-tent language that lets you do what you want. It has all +of Python's least-motivated semantic features, plus more features, plus +various kinds of syntactic sugar. (The syntax isn't as complex as +Python's, but there are a lot of details beyond plain old S-expressions.) +If you're interested in a more small-and-beautiful approach to Lisp, in +the style of Scheme, check out `Hissp `_, +another Lisp embedded in Python that was created by a Hy developer. + +Also, Hy isn't a reimplementation of an older Lisp. It is its own +language. It looks kind of like Clojure and kind of like Common Lisp, but +nontrivial programs that run in one of these langauges can't be expected +to run on another unaltered. From 8a45c6a222d6d4fd48dbf3a3103a21372e91b77a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 16 Nov 2023 14:33:42 -0500 Subject: [PATCH 341/342] Complicate the example reader macro `matrix` The previous version could be implemented as a regular macro. --- docs/macros.rst | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/macros.rst b/docs/macros.rst index 693bc7e98..c4ad7e40f 100644 --- a/docs/macros.rst +++ b/docs/macros.rst @@ -156,22 +156,32 @@ Ultimately it's wisest to use only four kinds of names in macro expansions: gens Reader macros ------------- -Reader macros allow you to hook directly into Hy's parser to customize how text is parsed into models. They're defined with :hy:func:`defreader`, or, like regular macros, brought in from other modules with :hy:func:`require`. Rather than receiving function arguments, a reader macro has access to a :py:class:`HyReader ` object named ``&reader``, which provides all the text-parsing logic that Hy uses to parse itself (see :py:class:`HyReader ` and its base class :py:class:`Reader ` for the available methods). A reader macro is called with the hash sign ``#``, and like a regular macro, it should return a model or something convertible to a model. :: +Reader macros allow you to hook directly into Hy's parser to customize how text is parsed into models. They're defined with :hy:func:`defreader`, or, like regular macros, brought in from other modules with :hy:func:`require`. Rather than receiving function arguments, a reader macro has access to a :py:class:`HyReader ` object named ``&reader``, which provides all the text-parsing logic that Hy uses to parse itself (see :py:class:`HyReader ` and its base class :py:class:`Reader ` for the available methods). A reader macro is called with the hash sign ``#``, and like a regular macro, it should return a model or something convertible to a model. + +Here's a moderately complex example of a reader macro that couldn't be implemented as a regular macro. It reads in a list of lists in which the inner lists are newline-separated, but newlines are allowed inside elements. :: (defreader matrix (.slurp-space &reader) (setv start (.getc &reader)) (assert (= start "[")) + (.slurp-space &reader) (setv out [[]]) - (while (not (do (.slurp-space &reader) (.peek-and-getc &reader "]"))) - (setv x (.parse-one-form &reader)) - (if (= x '|) - (.append out []) - (.append (get out -1) x))) - (if (= out [[]]) [] out)) - - (print (hy.repr #matrix [1 2 3 | 4 5 6 | 7 8 9])) - ; => [[1 2 3] [4 5 6] [7 8 9]] + (while (not (.peek-and-getc &reader "]")) + (cond + (any (gfor c " \t" (.peek-and-getc &reader c))) + None + (.peek-and-getc &reader "\n") + (.append out []) + True + (.append (get out -1) (.parse-one-form &reader)))) + (lfor line out :if line line)) + + (print (hy.repr #matrix [ + 1 (+ 1 1) 3 + 4 ["element" "containing" + "a" "newline"] 6 + 7 8 9])) + ; => [[1 2 3] [4 ["element" "containing" "a" "newline"] 6] [7 8 9]] Note that because reader macros are evaluated at parse-time, and top-level forms are completely parsed before any further compile-time execution occurs, you can't use a reader macro in the same top-level form that defines it:: From 67a5343563f1ad5bd2fa1fb6cf52f73444a68cc4 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 8 Dec 2023 09:46:39 -0500 Subject: [PATCH 342/342] Document and test weirder uses of digit separators --- docs/syntax.rst | 6 +++++- tests/test_reader.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/syntax.rst b/docs/syntax.rst index ed4a347aa..9a3ab9a0d 100644 --- a/docs/syntax.rst +++ b/docs/syntax.rst @@ -146,7 +146,11 @@ few extensions: - Commas (``,``) can be used like underscores (``_``) to separate digits without changing the result. Thus, ``10_000_000_000`` may also be written - ``10,000,000,000``. + ``10,000,000,000``. Hy is also more permissive about the placement of + separators than Python: several may be in a row, and they may be after all + digits, after ``.``, ``e``, or ``j``, or even inside a radix prefix. Separators + before the first digit are still forbidden because e.g. ``_1`` is a legal + Python variable name, so it's a symbol in Hy rather than an integer. - Integers can begin with leading zeroes, even without a radix prefix like ``0x``. Leading zeroes don't automatically cause the literal to be interpreted in octal like they do in C. For octal, use the prefix ``0o``, as diff --git a/tests/test_reader.py b/tests/test_reader.py index 3ae60daf9..0ef34dc57 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -259,14 +259,15 @@ def test_lex_digit_separators(): assert tokenize("0x_af") == [Integer(0xAF)] assert tokenize("0x,af") == [Integer(0xAF)] + assert tokenize("0_xaf") == [Integer(0xAF)] assert tokenize("0b_010") == [Integer(0b010)] assert tokenize("0b,010") == [Integer(0b010)] assert tokenize("0o_373") == [Integer(0o373)] assert tokenize("0o,373") == [Integer(0o373)] - assert tokenize("1_2.3,4") == [Float(12.34)] - assert tokenize("1_2e3,4") == [Float(12e34)] - assert tokenize("1,0_00j") == [Complex(1000j)] + assert tokenize("1_2._3,4") == [Float(12.34)] + assert tokenize("1_2e_3,4") == [Float(12e34)] + assert tokenize("1,0_00j,") == [Complex(1000j)] assert tokenize("1,,,,___,____,,__,,2__,,,__") == [Integer(12)] assert tokenize("_1,,,,___,____,,__,,2__,,,__") == [