diff --git a/.gitignore b/.gitignore index 5f65356..c6f9a44 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1 @@ -*.DS_Store -*.jl.cov -*.jl.*.cov -*.jl.mem -Manifest.toml -docs/build -docs/site -.vscode/settings.json \ No newline at end of file +.vscode/settings.json diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..c6dcd68 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,91 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "Pull latest version of CodeTracking", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/CodeTracking", "https://github.com/timholy/CodeTracking.jl", "master", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of CSTParser", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/CSTParser", "https://github.com/julia-vscode/CSTParser.jl", "master", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of JSON", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/JSON", "https://github.com/JuliaIO/JSON.jl", "master", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of JSONRPC", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/JSONRPC", "https://github.com/julia-vscode/JSONRPC.jl", "master", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of JuliaInterpreter", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/JuliaInterpreter", "https://github.com/JuliaDebug/JuliaInterpreter.jl", "master", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of LoweredCodeUtils", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/LoweredCodeUtils", "https://github.com/JuliaDebug/LoweredCodeUtils.jl", "master", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of OrderedCollections", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/OrderedCollections", "https://github.com/JuliaCollections/OrderedCollections.jl", "master", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of Revise", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/Revise", "https://github.com/timholy/Revise.jl", "master", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of TestEnv", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/TestEnv", "https://github.com/davidanthoff/TestEnv.jl", "main", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of TestItemServer", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/TestItemServer", "https://github.com/julia-vscode/TestItemServer.jl", "main", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of Tokenize", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/Tokenize", "https://github.com/JuliaLang/Tokenize.jl", "master", "--squash"], + "problemMatcher": [] + }, + { + "label": "Pull latest version of URIParser", + "type": "process", + "command": "git", + "args": ["subtree", "pull", "--prefix", "packages/URIParser", "https://github.com/JuliaWeb/URIParser.jl", "master", "--squash"], + "problemMatcher": [] + } + ] +} diff --git a/LICENSE b/LICENSE index d4439c8..310c72a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,49 +1,21 @@ -The TestEnv.jl package is licensed under the MIT "Expat" License: +MIT License -> Copyright (c) 2018-2021: Lyndon White and Malcolm Miller. -> -> Permission is hereby granted, free of charge, to any person obtaining a copy -> of this software and associated documentation files (the "Software"), to deal -> in the Software without restriction, including without limitation the rights -> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -> copies of the Software, and to permit persons to whom the Software is -> furnished to do so, subject to the following conditions: -> -> The above copyright notice and this permission notice shall be included in all -> copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -> SOFTWARE. -> +Copyright (c) 2023 David Anthoff -Some of the code (in particular /src/runner.jl), is based on code from the Julia Base library https://github.com/JuliaLang/julia +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -> Copyright (c) 2009-2021: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, -> and other contributors: -> -> https://github.com/JuliaLang/julia/contributors -> -> Permission is hereby granted, free of charge, to any person obtaining -> a copy of this software and associated documentation files (the -> "Software"), to deal in the Software without restriction, including -> without limitation the rights to use, copy, modify, merge, publish, -> distribute, sublicense, and/or sell copies of the Software, and to -> permit persons to whom the Software is furnished to do so, subject to -> the following conditions: -> -> The above copyright notice and this permission notice shall be -> included in all copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -> NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -> LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -> OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -> WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Project.toml b/Project.toml index eb52465..62a5d11 100644 --- a/Project.toml +++ b/Project.toml @@ -1,21 +1,31 @@ -name = "TestEnv" -uuid = "1e6cf692-eddd-4d53-88a5-2d735e33781b" -version = "2.0.0" +name = "TestItemRunner2" +uuid = "e1dee961-0fd3-483e-9ab1-c4c3c701c341" +authors = ["David Anthoff "] +version = "0.1.0-DEV" [deps] -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - -[compat] -ChainRulesCore = "=1.0.2" -MCMCDiagnosticTools = "=0.1.0" -YAXArrays = "0.1.3" -julia = "1" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +JuliaWorkspaces = "e554591c-7f10-434f-9f27-2097f62a04fd" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +JSONRPC = "b9b8584e-8fd3-41f9-ad0c-7255d428e418" +Mmap = "a63ad114-7e13-5084-954f-fe012c677804" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [extras] -ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -MCMCDiagnosticTools = "be115224-59cd-429b-ad48-344e309966f0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c" + +[compat] +julia = "1" +JuliaWorkspaces = "1.1.2" +JSON = "0.21" +JSONRPC = "1.3.6" +ProgressMeter = "1.7" [targets] -test = ["ChainRulesCore", "MCMCDiagnosticTools", "Test", "YAXArrays"] +test = ["Test"] diff --git a/README.md b/README.md index f476808..e0ff2a3 100644 --- a/README.md +++ b/README.md @@ -1,76 +1,32 @@ -# TestEnv +# TestItemRunner2 -[![Code Style: Blue](https://img.shields.io/badge/code%20style-blue-4495d1.svg)](https://github.com/invenia/BlueStyle) -[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac) - - -This is a 1-function package: `TestEnv.activate`. -It lets you activate the test enviroment from a given package. -Just like `Pkg.activate` lets you activate it's main enviroment. - - -Consider for example **ChainRules.jl** has as a test-only dependency of **ChainRulesTestUtils.jl**, -not a main dependency +Example: ```julia -pkg> activate ~/.julia/dev/ChainRules +using TestItemRunner2 -julia> using TestEnv; - -julia> TestEnv.activate(); - -julia> using ChainRulesTestUtils +run_tests("/users/foo/.julia/dev/InlineStrings") ``` -Use `Pkg.activate` to re-activate the previous environment, e.g. `Pkg.activate("~/.julia/dev/ChainRules")`. - -You can also pass in the name of a package, to activate that package and it's test dependencies: -`TestEnv.activate("Javis")` for example would activate Javis.jl's test environment. +## API -Finally you can pass in a function to run in this environment. ```julia -using TestEnv, ReTest -TestEnv.activate("Example") do - retest() -end + run_tests(path; filter=nothing, verbose=false, max_workers::Int=Sys.CPU_THREADS, timeout=60*5, return_results=false, print_failed_results=true) ``` -## Where is the code? -The astute reader has probably notice that the default branch of this git repo is basically empty. -This is because we keep all the code in other branches. -One per minor release: `release-1.0`, `release-1.1` etc. -We do this because TestEnv.jl accesses a whole ton of interals of [Pkg](https://github.com/JuliaLang/Pkg.jl). -These internals change basically every single release. -Maintaining compatibility in a single branch for multiple julia versions leads to code that is a nightmare. -As such, we instead maintain 1 branch per julia minor version. -And we tag releases off that branch with major and minor versions matching the julia version supported, but with patch versions allowed to change freely. +Runs test items. This will re-use client processes from a previous call to `run_tests`, but is guaranteed to use the latest code version (through a combination of static parsing and Revise). - - [release-1.0](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.0) contains the code to support julia v1.0.x - - [release-1.1](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.1) contains the code to support julia v1.1.x - - [release-1.2](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.2) contains the code to support julia v1.2.x - - [release-1.3](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.3) contains the code to support julia v1.3.x - - [release-1.4](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.4) contains the code to support julia v1.4.x, v1.5.x, and v1.6.x - - This was a rare goldern ages where the internals of Pkg did not change for almost a year. - - [release-1.7](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.7) contains the code to support julia v1.7.x - - [release-1.8](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.8) contains the code to support julia v1.8.x +Args: +- `path`: Filesystem path to a folder. +- `filter`: A filter callback function that will be called for each identified test item. If the filter callback returns `true` that test item will be run, if `false` it will not run. Each call to the provided filter callback passes a named tuple argument with a number of fields that contain metadata about the specific test item. The provided information is `filename`, `name`, `tags` and `package_name`. +- `verbose` Not implemented right now. +- `max_workers`: Max number of child processes per identified project. +- `timeout`: Timeout in seconds. +- `return_results`: Returns all test results as a vector, includes status, error messages and logs. +- `print_failed_results`: Print error messages for all failed tests when done running all tests. +```julia + kill_test_processes() +``` -**Do not make PRs against this COVER branch.** -Except to update this README. -Instead you probably want to PR a branch for some current version of Julia. - -This is a bit weird for semver. -New features *can* be added in patch release, but they must be ported to all later branches, and patch releases must be made there also. -For the this reason: we *only* support the latest patch release of any branch. -Older ones may be yanked if they start causing issues for people. - - -## What should I put in my Project.toml `[compat]` section -If using this as a dependency of a package that supports many versions of julia you may wonder what to put in your Project.toml's [compat] section. -Do not fear, the package manager has your back. -If you put in your `[compat]` for `TestEnv=`: `1` or equivalently `1.0` or `1.0` or `1.0.0` or `^1`, or `^1.0` or `^1.0` or `^1.0.0`, -then the package manager is free to choose any compatible version `v` with `1.0.0 <= v < 2.0.0`. -It will thus chose the corret minor version of TestEnv that is compatible with the loaded version of Julia. - -### See also: - - [Discourse Release Announcement](https://discourse.julialang.org/t/ann-testenv-jl-activate-your-test-enviroment-so-you-can-use-your-test-dependencies/65739) +Terminate all active client processes that are used to run test items. diff --git a/.github/workflows/TagBot.yml b/packages/CodeTracking/.github/workflows/TagBot.yml similarity index 100% rename from .github/workflows/TagBot.yml rename to packages/CodeTracking/.github/workflows/TagBot.yml diff --git a/packages/CodeTracking/.github/workflows/ci.yml b/packages/CodeTracking/.github/workflows/ci.yml new file mode 100644 index 0000000..053acd9 --- /dev/null +++ b/packages/CodeTracking/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +name: CI +on: + pull_request: + branches: + - master + push: + branches: + - master + tags: '*' +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.6' # latest LTS + - '1' + - 'nightly' + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x64 + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v1 + - uses: julia-actions/julia-buildpkg@latest + - uses: julia-actions/julia-runtest@latest + - name: Revise tests + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + julia -e ' + using Pkg + Pkg.develop(path=".") + Pkg.add(url="https://github.com/timholy/Revise.jl") + Pkg.test("Revise") + ' + - name: Test while running Revise + if: ${{ matrix.os == 'ubuntu-latest' && matrix.version != '1.0' }} + run: | + TERM="xterm" julia --project -i --code-coverage -e ' + using InteractiveUtils, REPL, Revise, Pkg + Pkg.add("ColorTypes") + # @async(Base.run_main_repl(true, true, false, true, false)) + sleep(2) + cd("test") + include("runtests.jl") + REPL.eval_user_input(:(exit()), Base.active_repl_backend) + ' + - uses: julia-actions/julia-processcoverage@latest + - uses: codecov/codecov-action@v3 + with: + file: lcov.info diff --git a/packages/CodeTracking/.gitignore b/packages/CodeTracking/.gitignore new file mode 100644 index 0000000..e338037 --- /dev/null +++ b/packages/CodeTracking/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +/Manifest.toml diff --git a/packages/CodeTracking/LICENSE b/packages/CodeTracking/LICENSE new file mode 100644 index 0000000..2cd7369 --- /dev/null +++ b/packages/CodeTracking/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2019 Tim Holy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/CodeTracking/Project.toml b/packages/CodeTracking/Project.toml new file mode 100644 index 0000000..3f35778 --- /dev/null +++ b/packages/CodeTracking/Project.toml @@ -0,0 +1,21 @@ +name = "CodeTracking" +uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +authors = ["Tim Holy "] +version = "1.2.0" + +[deps] +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[compat] +julia = "1.6" + +[extras] +ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["ColorTypes", "LinearAlgebra", "REPL", "SparseArrays", "Test"] diff --git a/packages/CodeTracking/README.md b/packages/CodeTracking/README.md new file mode 100644 index 0000000..a3a9c81 --- /dev/null +++ b/packages/CodeTracking/README.md @@ -0,0 +1,145 @@ +# CodeTracking + +[![Build status](https://github.com/timholy/CodeTracking.jl/actions/workflows/ci.yml/badge.svg)](https://github.com/timholy/CodeTracking.jl/actions/workflows/ci.yml) +[![Coverage](https://codecov.io/gh/timholy/CodeTracking.jl/branch/master/graph/badge.svg?token=bBzCYyj19O)](https://codecov.io/gh/timholy/CodeTracking.jl) + +CodeTracking can be thought of as an extension of Julia's +[InteractiveUtils library](https://docs.julialang.org/en/v1/stdlib/InteractiveUtils/). +It provides an interface for obtaining: + +- the strings and expressions of method definitions +- the method signatures at a specific file & line number +- location information for "dynamic" code that might have moved since it was first loaded +- a list of files that comprise a particular package. + +CodeTracking is a minimal package designed to work with +[Revise.jl](https://github.com/timholy/Revise.jl) (for versions v1.1.0 and higher). +CodeTracking is a very lightweight dependency. + +## Examples + +### `@code_string` and `@code_expr` + +```julia +julia> using CodeTracking, Revise + +julia> print(@code_string sum(1:5)) +function sum(r::AbstractRange{<:Real}) + l = length(r) + # note that a little care is required to avoid overflow in l*(l-1)/2 + return l * first(r) + (iseven(l) ? (step(r) * (l-1)) * (l>>1) + : (step(r) * l) * ((l-1)>>1)) +end + +julia> @code_expr sum(1:5) +[ Info: tracking Base +quote + #= toplevel:977 =# + function sum(r::AbstractRange{<:Real}) + #= /home/tim/src/julia-1/base/range.jl:978 =# + l = length(r) + #= /home/tim/src/julia-1/base/range.jl:980 =# + return l * first(r) + if iseven(l) + (step(r) * (l - 1)) * l >> 1 + else + (step(r) * l) * (l - 1) >> 1 + end + end +end +``` + +`@code_string` succeeds in that case even if you are not using Revise, but `@code_expr` always requires Revise. +(If you must live without Revise, you can use `Meta.parse(@code_string(...))` as a fallback.) + +"Difficult" methods are handled more accurately with `@code_expr` and Revise. +Here's one that's defined via an `@eval` statement inside a loop: + +```julia +julia> @code_expr Float16(1) + Float16(2) +:(a::Float16 + b::Float16 = begin + #= /home/tim/src/julia-1/base/float.jl:398 =# + Float16(Float32(a) + Float32(b)) + end) +``` + +whereas `@code_string` cannot return a useful result: + +``` +julia> @code_string Float16(1) + Float16(2) +"# This file is a part of Julia. License is MIT: https://julialang.org/license\n\nconst IEEEFloat = Union{Float16, Float32, Float64}" +``` +Consequently it's recommended to use `@code_expr` in preference to `@code_string` wherever possible. + +`@code_expr` and `@code_string` have companion functional variants, `code_expr` and `code_string`, which accept the function and a `Tuple{T1, T2, ...}` of types. + +`@code_expr` and `@code_string` are based on the lower-level function `definition`; +you can read about it with `?definition`. + +### Location information + +```julia +julia> using CodeTracking, Revise + +julia> m = @which sum([1,2,3]) +sum(a::AbstractArray) in Base at reducedim.jl:648 + +julia> Revise.track(Base) # also edit reducedim.jl + +julia> file, line = whereis(m) +("/home/tim/src/julia-1/usr/share/julia/base/reducedim.jl", 642) + +julia> m.line +648 +``` + +In this (ficticious) example, `sum` moved because I deleted a few lines higher in the file; +these didn't affect the functionality of `sum` (so we didn't need to redefine and recompile it), +but it does change the starting line number of the file at which this method appears. +`whereis` reports the current line number, and `m.line` the old line number. (For technical reasons, it is important that `m.line` remain at the value it had when the code was lowered.) + +Other methods of `whereis` allow you to obtain the current position corresponding to a single +statement inside a method; see `?whereis` for details. + +CodeTracking can also be used to find out what files define a particular package: + +```julia +julia> using CodeTracking, Revise, ColorTypes + +julia> pkgfiles(ColorTypes) +PkgFiles(ColorTypes [3da002f7-5984-5a60-b8a6-cbb66c0b333f]): + basedir: /home/tim/.julia/packages/ColorTypes/BsAWO + files: ["src/ColorTypes.jl", "src/types.jl", "src/traits.jl", "src/conversions.jl", "src/show.jl", "src/operations.jl"] +``` + + +You can also find the method-signatures at a particular location: + +```julia +julia> signatures_at(ColorTypes, "src/traits.jl", 14) +1-element Array{Any,1}: + Tuple{typeof(red),AbstractRGB} + +julia> signatures_at("/home/tim/.julia/packages/ColorTypes/BsAWO/src/traits.jl", 14) +1-element Array{Any,1}: + Tuple{typeof(red),AbstractRGB} +``` + +CodeTracking also helps correcting for [Julia issue #26314](https://github.com/JuliaLang/julia/issues/26314): + +```julia +julia> @which uuid1() +uuid1() in UUIDs at C:\cygwin\home\Administrator\buildbot\worker\package_win64\build\usr\share\julia\stdlib\v1.1\UUIDs\src\UUIDs.jl:50 + +julia> CodeTracking.whereis(@which uuid1()) +("C:\\Users\\SomeOne\\AppData\\Local\\Julia-1.1.0\\share\\julia\\stdlib\\v1.1\\UUIDs\\src\\UUIDs.jl", 50) +``` + +## A few details + +CodeTracking has limited functionality unless the user is also running Revise, +because Revise populates CodeTracking's internal variables. +(Using `whereis` as an example, CodeTracking will just return the +file/line info in the method itself if Revise isn't running.) + +CodeTracking is perhaps best thought of as the "query" part of Revise.jl, +providing a lightweight and stable API for gaining access to information it maintains internally. diff --git a/packages/CodeTracking/src/CodeTracking.jl b/packages/CodeTracking/src/CodeTracking.jl new file mode 100644 index 0000000..b433bd8 --- /dev/null +++ b/packages/CodeTracking/src/CodeTracking.jl @@ -0,0 +1,340 @@ +""" +CodeTracking can be thought of as an extension of InteractiveUtils, and pairs well with Revise.jl. + +- `code_string`, `@code_string`: fetch the source code (as a string) for a method definition +- `code_expr`, `@code_expr`: fetch the expression for a method definition +- `definition`: a lower-level variant of the above +- `pkgfiles`: return information about the source files that define a package +- `whereis`: Return location information about methods (with Revise, it updates as you edit files) +- `signatures_at`: return the signatures of all methods whose definition spans the specified location +""" +module CodeTracking + +using Base: PkgId +using Core: LineInfoNode +using Base.Meta: isexpr +using UUIDs +using InteractiveUtils + +export code_expr, @code_expr, code_string, @code_string, whereis, definition, pkgfiles, signatures_at + +# More recent Julia versions assign the line number to the line with the function declaration, +# not the first non-comment line of the body. +const line_is_decl = VERSION >= v"1.5.0-DEV.567" + +include("pkgfiles.jl") +include("utils.jl") + +### Global storage + +# These values get populated by Revise + +# `method_info[sig]` is either: +# - `missing`, to indicate that the method cannot be located +# - a list of `(lnn,ex)` pairs. In almost all cases there will be just one of these, +# but "mistakes" in moving methods from one file to another can result in more than +# definition. The last pair in the list is the currently-active definition. +const method_info = IdDict{Type,Union{Missing,Vector{Tuple{LineNumberNode,Expr}}}}() + +const _pkgfiles = Dict{PkgId,PkgFiles}() + +# Callback for method-lookup. `lookupfunc = method_lookup_callback[]` must have the form +# ret = lookupfunc(method) +# where `ret` is either `nothing` or `(lnn, def)`. `lnn` is a LineNumberNode (or any valid +# input to `CodeTracking.fileline`) and `def` is the expression defining the method. +const method_lookup_callback = Ref{Any}(nothing) + +# Callback for `signatures_at` (lookup by file/lineno). `lookupfunc = expressions_callback[]` +# must have the form +# mod, exsigs = lookupfunc(id, relpath) +# where +# id is the PkgId of the corresponding package +# relpath is the path of the file from the basedir of `id` +# mod is the "active" module at that point in the source +# exsigs is a ex=>sigs dictionary, where `ex` is the source expression and `sigs` +# a list of method-signatures defined by that expression. +const expressions_callback = Ref{Any}(nothing) + +const juliabase = joinpath("julia", "base") +const juliastdlib = joinpath("julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)") + +### Public API + +""" + filepath, line = whereis(method::Method) + +Return the file and line of the definition of `method`. The meaning of `line` +depends on the Julia version: on Julia 1.5 and higher it is the line number of +the method declaration, otherwise it is the first line of the method's body. +""" +function whereis(method::Method) + file, line = String(method.file), method.line + startswith(file, "REPL[") && return file, line + lin = get(method_info, method.sig, nothing) + if lin === nothing + f = method_lookup_callback[] + if f !== nothing + try + Base.invokelatest(f, method) + lin = get(method_info, method.sig, nothing) + catch + end + end + end + if lin === nothing || ismissing(lin) + else + file, line = fileline(lin[end][1]) + end + file = maybe_fix_path(file) + return file, line +end + +""" + loc = whereis(sf::StackFrame) + +Return location information for a single frame of a stack trace. +If `sf` corresponds to a frame that was inlined, `loc` will be `nothing`. +Otherwise `loc` will be `(filepath, line)`. +""" +function whereis(sf::StackTraces.StackFrame) + sf.linfo === nothing && return nothing + return whereis(sf, sf.linfo.def) +end + +""" + filepath, line = whereis(lineinfo, method::Method) + +Return the file and line number associated with a specific statement in `method`. +`lineinfo.line` should contain the line number of the statement at the time `method` +was compiled. The current location is returned. +""" +whereis(lineinfo, method::Method) = whereis((lineinfo.file, lineinfo.line), method) +function whereis((curfile, curline)::NTuple{2, Any}, method::Method) + file, line1 = whereis(method) + # We could be in an expanded macro. Apply the correction only if the filename checks out. + # (We're not super-fastidious here because of symlinks and other path ambiguities) + samefile = basename(file) == basename(String(curfile)) + if !samefile + return maybe_fix_path(String(curfile)), curline + end + return file, curline - method.line + line1 +end +function whereis(lineinfo::Core.LineInfoNode, method::Method) + # With LineInfoNode we have certainty about whether we're in a macro expansion, but + # we're still not guaranteed that the lineinfo points into the method otherwise + # (e.g. from generated or programmatically defined methods) + meth = lineinfo.method + if isa(meth, WeakRef) + meth = meth.value + end + if meth === Symbol("macro expansion") + return maybe_fix_path(String(lineinfo.file)), lineinfo.line + end + return whereis((lineinfo.file, lineinfo.line), method) +end + +""" + sigs = signatures_at(filename, line) + +Return the signatures of all methods whose definition spans the specified location. +Prior to Julia 1.5, `line` must correspond to a line in the method body +(not the signature or final `end`). + +Returns `nothing` if there are no methods at that location. +""" +function signatures_at(filename::AbstractString, line::Integer) + if !startswith(filename, "REPL[") + filename = abspath(filename) + end + if occursin(juliabase, filename) + rpath = postpath(filename, juliabase) + id = PkgId(Base) + return signatures_at(id, rpath, line) + elseif occursin(juliastdlib, filename) + rpath = postpath(filename, juliastdlib) + spath = splitpath(rpath) + libname = spath[1] + project = Base.active_project() + id = getpkgid(project, libname) + return signatures_at(id, joinpath(spath[2:end]...), line) + end + if startswith(filename, "REPL[") + id = PkgId("@REPL") + return signatures_at(id, filename, line) + end + for (id, pkgfls) in _pkgfiles + if startswith(filename, basedir(pkgfls)) || id.name == "Main" + bdir = basedir(pkgfls) + rpath = isempty(bdir) ? filename : relpath(filename, bdir) + if rpath ∈ pkgfls.files + return signatures_at(id, rpath, line) + end + end + end + throw(ArgumentError("$filename not found in internal data, perhaps the package is not loaded (or not loaded with `includet`)")) +end + +""" + sigs = signatures_at(mod::Module, relativepath, line) + +For a package that defines module `mod`, return the signatures of all methods whose definition +spans the specified location. `relativepath` indicates the path of the file relative to +the packages top-level directory, e.g., `"src/utils.jl"`. +`line` must correspond to a line in the method body (not the signature or final `end`). + +Returns `nothing` if there are no methods at that location. +""" +function signatures_at(mod::Module, relpath::AbstractString, line::Integer) + id = PkgId(mod) + return signatures_at(id, relpath, line) +end + +function signatures_at(id::PkgId, relpath::AbstractString, line::Integer) + expressions = expressions_callback[] + expressions === nothing && error("cannot look up methods by line number, try `using Revise` before loading other packages") + try + for (mod, exsigs) in Base.invokelatest(expressions, id, relpath) + for (ex, sigs) in exsigs + lr = linerange(ex) + lr === nothing && continue + line ∈ lr && return sigs + end + end + catch + end + return nothing +end + +""" + src, line1 = definition(String, method::Method) + +Return a string with the code that defines `method`. Also return the first line of the +definition, including the signature (which may not be the same line number returned +by `whereis`). If the method can't be located (line number 0), then `definition` +instead returns `nothing.` + +Note this may not be terribly useful for methods that are defined inside `@eval` statements; +see [`definition(Expr, method::Method)`](@ref) instead. + +See also [`code_string`](@ref). +""" +function definition(::Type{String}, method::Method) + file, line = whereis(method) + line == 0 && return nothing + src = src_from_file_or_REPL(file) + src === nothing && return nothing + src = replace(src, "\r"=>"") + eol = isequal('\n') + linestarts = Int[] + istart = 1 + for i = 1:line-1 + push!(linestarts, istart) + istart = findnext(eol, src, istart) + 1 + end + ex, iend = Meta.parse(src, istart; raise=false) + iend = prevind(src, iend) + if isfuncexpr(ex, method.name) + iend = min(iend, lastindex(src)) + return strip(src[istart:iend], '\n'), line + end + # The function declaration was presumably on a previous line + lineindex = lastindex(linestarts) + linestop = max(0, lineindex - 20) + while !isfuncexpr(ex, method.name) && lineindex > linestop + istart = linestarts[lineindex] + try + ex, iend = Meta.parse(src, istart) + catch + end + lineindex -= 1 + line -= 1 + end + lineindex <= linestop && return nothing + return chomp(src[istart:iend-1]), line +end + +""" + ex = definition(Expr, method::Method) + ex = definition(method::Method) + +Return an expression that defines `method`. If the definition can't be found, +returns `nothing`. + +See also [`code_expr`](@ref). +""" +function definition(::Type{Expr}, method::Method) + file = String(method.file) + def = startswith(file, "REPL[") ? nothing : get(method_info, method.sig, nothing) + if def === nothing + f = method_lookup_callback[] + if f !== nothing + try + Base.invokelatest(f, method) + def = get(method_info, method.sig, nothing) + catch + end + end + end + return def === nothing || ismissing(def) ? nothing : copy(def[end][2]) +end + +definition(method::Method) = definition(Expr, method) + +""" + code_expr(f, types) + +Returns the expression for the method definition for `f` with the specified types. + +May return `nothing` if Revise isn't loaded. In such cases, calling +`Meta.parse(code_string(f, types))` can sometimes be an alternative. +""" +code_expr(f, t) = definition(Expr, which(f, t)) +macro code_expr(ex0...) + InteractiveUtils.gen_call_with_extracted_types_and_kwargs(__module__, :code_expr, ex0) +end + +""" + code_string(f, types) + +Returns the code-string for the method definition for `f` with the specified types. +""" +function code_string(f, t) + def = definition(String, which(f, t)) + return def === nothing ? nothing : def[1] +end +macro code_string(ex0...) + InteractiveUtils.gen_call_with_extracted_types_and_kwargs(__module__, :code_string, ex0) +end + +""" + info = pkgfiles(name::AbstractString) + info = pkgfiles(name::AbstractString, uuid::UUID) + +Return a [`CodeTracking.PkgFiles`](@ref) structure with information about the files that +define the package specified by `name` and `uuid`. +Returns `nothing` if this package has not been loaded. +""" +pkgfiles(name::AbstractString, uuid::UUID) = pkgfiles(PkgId(uuid, name)) +function pkgfiles(name::AbstractString) + project = Base.active_project() + # The value returned by Base.project_deps_get depends on the Julia version + id = isdefined(Base, :TOMLCache) && Base.VERSION < v"1.6.0-DEV.1180" ? Base.project_deps_get(project, name, Base.TOMLCache()) : + Base.project_deps_get(project, name) + (id == false || id === nothing) && error("no package ", name, " recognized") + return isa(id, PkgId) ? pkgfiles(id) : pkgfiles(name, id) +end +pkgfiles(id::PkgId) = get(_pkgfiles, id, nothing) + +""" + info = pkgfiles(mod::Module) + +Return a [`CodeTracking.PkgFiles`](@ref) structure with information about the files that +were loaded to define the package that defined `mod`. +""" +pkgfiles(mod::Module) = pkgfiles(PkgId(mod)) + +if ccall(:jl_generating_output, Cint, ()) == 1 + precompile(Tuple{typeof(setindex!), Dict{PkgId,PkgFiles}, PkgFiles, PkgId}) +end + +end # module diff --git a/packages/CodeTracking/src/pkgfiles.jl b/packages/CodeTracking/src/pkgfiles.jl new file mode 100644 index 0000000..4854066 --- /dev/null +++ b/packages/CodeTracking/src/pkgfiles.jl @@ -0,0 +1,39 @@ +""" +PkgFiles encodes information about the current location of a package. +Fields: +- `id`: the `PkgId` of the package +- `basedir`: the current base directory of the package +- `files`: a list of files (relative path to `basedir`) that define the package. + +Note that `basedir` may be subsequently updated by Pkg operations such as `add` and `dev`. +""" +mutable struct PkgFiles + id::PkgId + basedir::String + files::Vector{Any} +end + +PkgFiles(id::PkgId, path::AbstractString) = PkgFiles(id, path, Any[]) +PkgFiles(id::PkgId, ::Nothing) = PkgFiles(id, "") +PkgFiles(id::PkgId) = PkgFiles(id, normpath(basepath(id))) +PkgFiles(id::PkgId, files::Vector{Any}) = + PkgFiles(id, normpath(basepath(id)), files) + +# Abstraction interface +Base.PkgId(info::PkgFiles) = info.id +srcfiles(info::PkgFiles) = info.files +basedir(info::PkgFiles) = info.basedir + +function Base.show(io::IO, info::PkgFiles) + compact = get(io, :compact, false) + if compact + print(io, "PkgFiles(", info.id.name, ", ", info.basedir, ", ") + show(io, info.files) + print(io, ')') + else + println(io, "PkgFiles(", info.id, "):") + println(io, " basedir: \"", info.basedir, '"') + print(io, " files: ") + show(io, info.files) + end +end diff --git a/packages/CodeTracking/src/utils.jl b/packages/CodeTracking/src/utils.jl new file mode 100644 index 0000000..d0b2b0b --- /dev/null +++ b/packages/CodeTracking/src/utils.jl @@ -0,0 +1,158 @@ +# This should stay as the first method because it's used in a test +# (or change the test) +function checkname(fdef::Expr, name) + fproto = fdef.args[1] + (fdef.head === :where || fdef.head == :(::)) && return checkname(fproto, name) + fdef.head === :call || return false + if fproto isa Expr + fproto.head == :(::) && return last(fproto.args) == name + # A metaprogramming-generated function + fproto.head === :$ && return true # uncheckable, let's assume all is well + # Is the check below redundant? + fproto.head === :. || return false + # E.g. `function Mod.bar.foo(a, b)` + return checkname(fproto.args[end], name) + end + isa(fproto, Symbol) || isa(fproto, QuoteNode) || isa(fproto, Expr) || return false + return checkname(fproto, name) +end +checkname(fname::Symbol, name::Symbol) = begin + fname === name && return true + startswith(string(name), string('#', fname, '#')) && return true + string(name) == string(fname, "##kw") && return true + return false +end +checkname(fname::Symbol, ::Nothing) = true +checkname(fname::QuoteNode, name) = checkname(fname.value, name) + +function isfuncexpr(ex, name=nothing) + # Strip any macros that wrap the method definition + while ex isa Expr && ex.head === :macrocall && length(ex.args) == 3 + ex = ex.args[3] + end + isa(ex, Expr) || return false + if ex.head === :function || ex.head === :(=) + return checkname(ex.args[1], name) + end + return false +end + +function linerange(def::Expr) + start, haslinestart = findline(def, identity) + stop, haslinestop = findline(def, Iterators.reverse) + (haslinestart & haslinestop) && return start:stop + return nothing +end +linerange(arg) = linerange(convert(Expr, arg)) # Handle Revise's RelocatableExpr + +function findline(ex, order) + ex.head === :line && return ex.args[1], true + for a in order(ex.args) + a isa LineNumberNode && return a.line, true + if a isa Expr + ln, hasline = findline(a, order) + hasline && return ln, true + end + end + return 0, false +end + +fileline(lin::LineInfoNode) = String(lin.file), lin.line +fileline(lnn::LineNumberNode) = String(lnn.file), lnn.line + +# This is piracy, but it's not ambiguous in terms of what it should do +Base.convert(::Type{LineNumberNode}, lin::LineInfoNode) = LineNumberNode(lin.line, lin.file) + +# This regex matches the pseudo-file name of a REPL history entry. +const rREPL = r"^REPL\[(\d+)\]$" + +""" + src = src_from_file_or_REPL(origin::AbstractString, repl = Base.active_repl) + +Read the source for a function from `origin`, which is either the name of a file +or "REPL[\$i]", where `i` is an integer specifying the particular history entry. +Methods defined at the REPL use strings of this form in their `file` field. + +If you happen to have a file where the name matches `REPL[\$i]`, first pass it through +`abspath`. +""" +function src_from_file_or_REPL(origin::AbstractString, args...) + # This Varargs design prevents an unnecessary error when Base.active_repl is undefined + # and `origin` does not match "REPL[$i]" + m = match(rREPL, origin) + if m !== nothing + return src_from_REPL(m.captures[1], args...) + end + isfile(origin) || return nothing + return read(origin, String) +end + +function src_from_REPL(origin::AbstractString, repl = Base.active_repl) + hist_idx = parse(Int, origin) + hp = repl.interface.modes[1].hist + return hp.history[hp.start_idx+hist_idx] +end + +function basepath(id::PkgId) + id.name ∈ ("Main", "Base", "Core") && return "" + loc = Base.locate_package(id) + loc === nothing && return "" + return dirname(dirname(loc)) +end + +""" + path = maybe_fix_path(path) + +Return a normalized, absolute path for a source file `path`. +""" +function maybe_fix_path(file) + if !isabspath(file) + # This may be a Base or Core method + newfile = Base.find_source_file(file) + if isa(newfile, AbstractString) + file = normpath(newfile) + end + end + return maybe_fixup_stdlib_path(file) +end + +safe_isfile(x) = try isfile(x); catch; false end +const BUILDBOT_STDLIB_PATH = dirname(abspath(String((@which uuid1()).file), "..", "..", "..")) +replace_buildbot_stdlibpath(str::String) = replace(str, BUILDBOT_STDLIB_PATH => Sys.STDLIB) +""" + path = maybe_fixup_stdlib_path(path::String) + +Return `path` corrected for julia issue [#26314](https://github.com/JuliaLang/julia/issues/26314) if applicable. +Otherwise, return the input `path` unchanged. + +Due to the issue mentioned above, location info for methods defined one of Julia's standard libraries +are, for non source Julia builds, given as absolute paths on the worker that built the `julia` executable. +This function corrects such a path to instead refer to the local path on the users drive. +""" +function maybe_fixup_stdlib_path(path) + if !safe_isfile(path) + maybe_stdlib_path = replace_buildbot_stdlibpath(path) + safe_isfile(maybe_stdlib_path) && return maybe_stdlib_path + end + return path +end + +function postpath(filename, pre) + idx = findfirst(pre, filename) + idx === nothing && error(pre, " not found in ", filename) + post = filename[first(idx) + length(pre) : end] + post[1:1] == Base.Filesystem.path_separator && return post[2:end] + return post +end + +# Robust across Julia versions +getpkgid(project::AbstractString, libname) = getpkgid(Base.project_deps_get(project, libname), libname) +getpkgid(id::PkgId, libname) = id +getpkgid(uuid::UUID, libname) = PkgId(uuid, libname) + +# Because IdDict's `setindex!` uses `@nospecialize` on both the key and value, it makes +# callers vulnerable to invalidation. These convenience utilities allow callers to insulate +# themselves from invalidation. These are used by Revise. +# example package triggering invalidation: StaticArrays (new `convert(Type{Array{T,N}}, ::AbstractArray)` methods) +invoked_setindex!(dct::IdDict{K,V}, @nospecialize(val), @nospecialize(key)) where {K,V} = Base.invokelatest(setindex!, dct, val, key)::typeof(dct) +invoked_get!(::Type{T}, dct::IdDict{K,V}, @nospecialize(key)) where {K,V,T<:V} = Base.invokelatest(get!, T, dct, key)::V diff --git a/packages/CodeTracking/test/runtests.jl b/packages/CodeTracking/test/runtests.jl new file mode 100644 index 0000000..cedb165 --- /dev/null +++ b/packages/CodeTracking/test/runtests.jl @@ -0,0 +1,297 @@ +# Note: some of CodeTracking's functionality can only be tested by Revise + +using CodeTracking +using Test, InteractiveUtils, REPL, LinearAlgebra, SparseArrays +# Note: ColorTypes needs to be installed, but note the intentional absence of `using ColorTypes` + +using CodeTracking: line_is_decl + +if !isempty(ARGS) && "revise" ∈ ARGS + # For running tests with and without Revise + using Revise +end + +isdefined(Main, :Revise) ? Main.Revise.includet("script.jl") : include("script.jl") + +@testset "CodeTracking.jl" begin + m = first(methods(f1)) + file, line = whereis(m) + scriptpath = normpath(joinpath(@__DIR__, "script.jl")) + @test file == scriptpath + @test line == (line_is_decl ? 2 : 4) + trace = try + call_throws() + catch + stacktrace(catch_backtrace()) + end + @test whereis(trace[2]) == (scriptpath, 9) + + src, line = definition(String, m) + @test src == chomp(""" + function f1(x, y) + # A comment + return x + y + end + """) + @test line == 2 + @test code_string(f1, Tuple{Any,Any}) == src + @test @code_string(f1(1, 2)) == src + + m = first(methods(f2)) + src, line = definition(String, m) + @test src == "f2(x, y) = x + y" + @test line == 14 + + m = first(methods(throws)) + src, line = definition(String, m) + @test startswith(src, "@noinline") + @test line == 7 + + m = first(methods(multilinesig)) + src, line = definition(String, m) + @test startswith(src, "@inline") + @test line == 16 + @test @code_string(multilinesig(1, "hi")) == src + @test_throws ErrorException("no unique matching method found for the specified argument types") @code_string(multilinesig(1, 2)) + + m = first(methods(f50)) + src, line = definition(String, m) + @test occursin("100x", src) + @test line == 22 + + # Issue #81 + m = which(hasrettype, (Int,)) + src, line = definition(String, m) + @test occursin("Float32", src) + @test line == 43 + + info = CodeTracking.PkgFiles(Base.PkgId(CodeTracking), nothing) + @test isempty(CodeTracking.basedir(info)) + + info = CodeTracking.PkgFiles(Base.PkgId(CodeTracking), []) + @test length(CodeTracking.srcfiles(info)) == 0 + + info = CodeTracking.PkgFiles(Base.PkgId(CodeTracking)) + @test Base.PkgId(info) === info.id + @test CodeTracking.basedir(info) == dirname(@__DIR__) + + io = PipeBuffer() + show(io, info) + str = read(io, String) + @test startswith(str, "PkgFiles(CodeTracking [da1fd8a2-8d9e-5ec2-8556-3022fb5608a2]):\n basedir:") + ioctx = IOContext(io, :compact=>true) + show(ioctx, info) + str = read(io, String) + @test match(r"PkgFiles\(CodeTracking, .*CodeTracking(\.jl)?, Any\[\]\)", str) !== nothing + + @test pkgfiles("ColorTypes") === nothing + @test_throws ErrorException pkgfiles("NotAPkg") + + # Test a method marked as missing + m = @which sum(1:5) + CodeTracking.method_info[m.sig] = missing + @test whereis(m) == (CodeTracking.maybe_fix_path(String(m.file)), m.line) + @test definition(m) === nothing + + # Test that definitions at the REPL work with `whereis` + ex = Base.parse_input_line("replfunc(x) = 1"; filename="REPL[1]") + eval(ex) + m = first(methods(replfunc)) + @test whereis(m) == ("REPL[1]", 1) + # Test with broken lookup + oldlookup = CodeTracking.method_lookup_callback[] + CodeTracking.method_lookup_callback[] = m -> error("oops") + @test whereis(m) == ("REPL[1]", 1) + CodeTracking.method_lookup_callback[] = oldlookup + + # Test implicit replacement of `BUILDBOT_STDLIB_PATH` + m = first(methods(Test.eval)) + @test isfile(whereis(m)[1]) + + # https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/150 + function f150() + x = 1 + 1 + @info "hello" + end + m = first(methods(f150)) + src = Base.uncompressed_ast(m) + idx = findfirst(lin -> String(lin.file) == @__FILE__, src.linetable) + lin = src.linetable[idx] + file, line = whereis(lin, m) + @test endswith(file, String(lin.file)) + idx = findfirst(lin -> String(lin.file) != @__FILE__, src.linetable) + lin = src.linetable[idx] + file, line = whereis(lin, m) + @test endswith(file, String(lin.file)) + + # Issues raised in #48 + m = @which(sum([1]; dims=1)) + if !isdefined(Main, :Revise) + def = definition(String, m) + @test def === nothing || isa(def[1], AbstractString) + def = definition(Expr, m) + @test def === nothing || isa(def, Expr) + else + def = definition(String, m) + @test isa(def[1], AbstractString) + def = definition(Expr, m) + @test isa(def, Expr) + end + + # Issue #64 + B = Hermitian(hcat([one(BigFloat) + im])) + m = @which cholesky(B) + @test startswith(definition(String, m)[1], "cholesky") + + # Ensure that we don't error on difficult cases + m = which(+, (AbstractSparseVector, AbstractSparseVector)) # defined inside an `@eval` + d = definition(String, m) + @test d === nothing || isa(d[1], AbstractString) + + # Check for existence of file + id = Base.PkgId("__PackagePrecompilationStatementModule") # not all Julia versions have this + mod = try Base.root_module(id) catch nothing end + if isa(mod, Module) + m = first(methods(getfield(mod, :eval))) + @test definition(String, m) === nothing + end + + # Related to issue [#51](https://github.com/timholy/CodeTracking.jl/issues/51) + # and https://github.com/JuliaDocs/Documenter.jl/issues/1779 + ex = :(f_no_linenum(::Int) = 1) + deleteat!(ex.args[2].args, 1) # delete the file & line number info + eval(ex) + @test code_string(f_no_linenum, (Int,)) === nothing + + # Invalidation-insulating methods used by Revise and perhaps others + d = IdDict{Union{String,Symbol},Union{Function,Vector{Function}}}() + CodeTracking.invoked_setindex!(d, sin, "sin") + @test CodeTracking.invoked_get!(Vector{Function}, d, :cos) isa Vector{Function} +end + +@testset "With Revise" begin + if isdefined(Main, :Revise) + m = @which gcd(10, 20) + sigs = signatures_at(Base.find_source_file(String(m.file)), m.line) + @test !isempty(sigs) + ex = @code_expr(gcd(10, 20)) + @test ex isa Expr + body = ex.args[2] + idx = findfirst(x -> isa(x, LineNumberNode), body.args) + @test occursin(String(m.file), String(body.args[idx].file)) + @test ex == code_expr(gcd, Tuple{Int,Int}) + + m = first(methods(edit)) + sigs = signatures_at(String(m.file), m.line) + @test !isempty(sigs) + sigs = signatures_at(Base.find_source_file(String(m.file)), m.line) + @test !isempty(sigs) + + # issue #23 + @test !isempty(signatures_at("script.jl", 9)) + + @test_throws ArgumentError signatures_at("nofile.jl", 1) + + if isdefined(Revise, :add_revise_deps) + Revise.add_revise_deps() + sigs = signatures_at(CodeTracking, "src/utils.jl", 5) + @test length(sigs) == 1 # only isn't available on julia 1.0 + @test first(sigs) == Tuple{typeof(CodeTracking.checkname), Expr, Any} + @test pkgfiles(CodeTracking).id == Base.PkgId(CodeTracking) + end + + # REPL (test copied from Revise) + if isdefined(Base, :active_repl) + hp = Base.active_repl.interface.modes[1].hist + fstr = "__fREPL__(x::Int16) = 0" + histidx = length(hp.history) + 1 - hp.start_idx + ex = Base.parse_input_line(fstr; filename="REPL[$histidx]") + f = Core.eval(Main, ex) + if ex.head === :toplevel + ex = ex.args[end] + end + push!(hp.history, fstr) + m = first(methods(f)) + @test definition(String, first(methods(f))) == (fstr, 1) + @test !isempty(signatures_at(String(m.file), m.line)) + pop!(hp.history) + elseif haskey(ENV, "CI") + error("CI Revise tests must be run with -i") + end + end +end + +(a_34)(x::T, y::T) where {T<:Integer} = no_op_err("&", T) +(b_34)(x::T, y::T) where {T<:Integer} = no_op_err("|", T) +c_34(x::T, y::T) where {T<:Integer} = no_op_err("xor", T) + +(d_34)(x::T, y::T) where {T<:Number} = x === y +(e_34)(x::T, y::T) where {T<:Real} = no_op_err("<" , T) +(f_34)(x::T, y::T) where {T<:Real} = no_op_err("<=", T) +l = @__LINE__ +@testset "#34 last character" begin + def, line = definition(String, @which d_34(1, 2)) + @test line == l - 3 + @test def == "(d_34)(x::T, y::T) where {T<:Number} = x === y" +end + +function g() + Base.@_inline_meta + print("hello") +end +@testset "inline macros" begin + def, line = CodeTracking.definition(String, @which g()) + @test def == """ + function g() + Base.@_inline_meta + print("hello") + end""" +end + +@testset "kwargs methods" begin + m = nothing + for i in 1:30 + s = Symbol("#func_2nd_kwarg#$i") + if isdefined(Main, s) + m = @eval $s + end + end + m === nothing && error("couldn't find keyword function") + body, loc = CodeTracking.definition(String, first(methods(m))) + @test loc == 28 + @test body == "func_2nd_kwarg(; kw=2) = true" +end + +@testset "method extensions" begin + body, _ = CodeTracking.definition(String, @which Foo.Bar.fit(1)) + @test body == """ + function Foo.Bar.fit(m) + return m + end""" + body, _ = CodeTracking.definition(String, @which Foo.Bar.fit(1, 2)) + @test body == "Foo.Bar.fit(a, b) = a + b" +end + +struct CallOverload + z +end +(f::CallOverload)(arg) = f.z + arg +struct Functor end +(::Functor)(x, y) = x+y +@testset "call syntax" begin + body, _ = CodeTracking.definition(String, @which CallOverload(1)(1)) + @test body == "(f::CallOverload)(arg) = f.z + arg" + + body, _ = CodeTracking.definition(String, @which Functor()(1,2)) + @test body == "(::Functor)(x, y) = x+y" +end + +if v"1.6" <= VERSION < v"1.9" +@testset "kwfuncs" begin + body, _ = CodeTracking.definition(String, @which fkw(; x=1)) + @test body == """ + function fkw(; x=1) + x + end""" +end +end diff --git a/packages/CodeTracking/test/script.jl b/packages/CodeTracking/test/script.jl new file mode 100644 index 0000000..4ed17ee --- /dev/null +++ b/packages/CodeTracking/test/script.jl @@ -0,0 +1,49 @@ +# NOTE: tests are sensitive to the line number at which statements appear +function f1(x, y) + # A comment + return x + y +end + +@noinline function throws() + x = nothing + error("oops") +end +@inline inlined() = throws() +call_throws() = inlined() + +f2(x, y) = x + y + +@inline function multilinesig(x::Int, + y::String) + z = x + 1 + return z +end + +function f50() # issue #50 + todB(x) = 10*log10(x) + println("100x is $(todB(100)) dB.") +end + +func_1st_nokwarg() = true +func_2nd_kwarg(; kw=2) = true + +module Foo +module Bar +function fit end +end +end + +function Foo.Bar.fit(m) + return m +end + +Foo.Bar.fit(a, b) = a + b + +# Issue #81 +function hasrettype(x::Real)::Float32 + return x*x + x +end + +function fkw(; x=1) + x +end diff --git a/packages/JSON/.gitignore b/packages/JSON/.gitignore new file mode 100644 index 0000000..edc6d3b --- /dev/null +++ b/packages/JSON/.gitignore @@ -0,0 +1,3 @@ +*.cov +*.mem +data/*.json diff --git a/packages/JSON/.travis.yml b/packages/JSON/.travis.yml new file mode 100644 index 0000000..1bc2f60 --- /dev/null +++ b/packages/JSON/.travis.yml @@ -0,0 +1,12 @@ +language: julia +os: + - osx + - linux +julia: + - 0.7 + - 1.0 + - nightly +notifications: + email: false +after_success: + - julia -e 'import Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())'; diff --git a/packages/JSON/LICENSE.md b/packages/JSON/LICENSE.md new file mode 100644 index 0000000..d916e61 --- /dev/null +++ b/packages/JSON/LICENSE.md @@ -0,0 +1,25 @@ +The Julia JSON package is licensed under the MIT Expat License: + +> Copyright (c) 2002: JSON.org, 2012–2016: Avik Sengupta, Stefan Karpinski, +> David de Laat, Dirk Gadsen, Milo Yip and other contributors +> – https://github.com/JuliaLang/JSON.jl/contributors +> and https://github.com/miloyip/nativejson-benchmark/contributors +> +> Permission is hereby granted, free of charge, to any person obtaining +> a copy of this software and associated documentation files (the +> "Software"), to deal in the Software without restriction, including +> without limitation the rights to use, copy, modify, merge, publish, +> distribute, sublicense, and/or sell copies of the Software, and to +> permit persons to whom the Software is furnished to do so, subject to +> the following conditions: +> +> The above copyright notice and this permission notice shall be +> included in all copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +> NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +> LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +> OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +> WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/packages/JSON/Project.toml b/packages/JSON/Project.toml new file mode 100644 index 0000000..2a0d1b3 --- /dev/null +++ b/packages/JSON/Project.toml @@ -0,0 +1,22 @@ +name = "JSON" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.20.1" + +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Mmap = "a63ad114-7e13-5084-954f-fe012c677804" +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[extras] +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +julia = "0.7, 1" + +[targets] +test = ["DataStructures", "Distributed", "FixedPointNumbers", "OffsetArrays", "Sockets", "Test"] diff --git a/packages/JSON/README.md b/packages/JSON/README.md new file mode 100644 index 0000000..9ccbd6d --- /dev/null +++ b/packages/JSON/README.md @@ -0,0 +1,108 @@ +# JSON.jl +### Parsing and printing JSON in pure Julia. + +[![Build Status](https://travis-ci.org/JuliaIO/JSON.jl.svg)](https://travis-ci.org/JuliaIO/JSON.jl) +[![Build status](https://ci.appveyor.com/api/projects/status/2sfomjwl29k6y6oy)](https://ci.appveyor.com/project/staticfloat/json-jl) +[![codecov.io](http://codecov.io/github/JuliaIO/JSON.jl/coverage.svg?branch=master)](http://codecov.io/github/JuliaIO/JSON.jl?branch=master) + +[![JSON](http://pkg.julialang.org/badges/JSON_0.3.svg)](http://pkg.julialang.org/?pkg=JSON&ver=0.3) +[![JSON](http://pkg.julialang.org/badges/JSON_0.4.svg)](http://pkg.julialang.org/?pkg=JSON&ver=0.4) +[![JSON](http://pkg.julialang.org/badges/JSON_0.5.svg)](http://pkg.julialang.org/?pkg=JSON&ver=0.5) +[![JSON](http://pkg.julialang.org/badges/JSON_0.6.svg)](http://pkg.julialang.org/?pkg=JSON&ver=0.6) + +**Installation**: `julia> Pkg.add("JSON")` + + +## Basic Usage + + +```julia +import JSON + +# JSON.parse - string or stream to Julia data structures +s = "{\"a_number\" : 5.0, \"an_array\" : [\"string\", 9]}" +j = JSON.parse(s) +# Dict{AbstractString,Any} with 2 entries: +# "an_array" => {"string",9} +# "a_number" => 5.0 + +# JSON.json - Julia data structures to a string +JSON.json([2,3]) +# "[2,3]" +JSON.json(j) +# "{\"an_array\":[\"string\",9],\"a_number\":5.0}" +``` + +## Documentation + + +```julia +JSON.print(io::IO, s::AbstractString) +JSON.print(io::IO, s::Union{Integer, AbstractFloat}) +JSON.print(io::IO, n::Nothing) +JSON.print(io::IO, b::Bool) +JSON.print(io::IO, a::AbstractDict) +JSON.print(io::IO, v::AbstractVector) +JSON.print{T, N}(io::IO, v::Array{T, N}) +``` + +Writes a compact (no extra whitespace or indentation) JSON representation +to the supplied IO. + +```julia +JSON.print(a::AbstractDict, indent) +JSON.print(io::IO, a::AbstractDict, indent) +``` + +Writes a JSON representation with newlines, and indentation if specified. Non-zero `indent` will be applied recursively to nested elements. + + +```julia +json(a::Any) +``` + +Returns a compact JSON representation as an `AbstractString`. + +```julia +JSON.parse(s::AbstractString; dicttype=Dict, inttype=Int64) +JSON.parse(io::IO; dicttype=Dict, inttype=Int64) +JSON.parsefile(filename::AbstractString; dicttype=Dict, inttype=Int64, use_mmap=true) +``` + +Parses a JSON `AbstractString` or IO stream into a nested `Array` or `Dict`. + +The `dicttype` indicates the dictionary type (`<: Associative`), or a function that +returns an instance of a dictionary type, +that JSON objects are parsed to. It defaults to `Dict` (the built-in Julia +dictionary), but a different type can be passed for additional functionality. +For example, if you `import DataStructures` +(assuming the [DataStructures +package](https://github.com/JuliaLang/DataStructures.jl) is +installed) + + - you can pass `dicttype=DataStructures.OrderedDict` to maintain the insertion order + of the items in the object; + - or you can pass `()->DefaultDict{String,Any}(Missing)` to having any non-found keys + return `missing` when you index the result. + + +The `inttype` argument controls how integers are parsed. If a number in a JSON +file is recognized to be an integer, it is parsed as one; otherwise it is parsed +as a `Float64`. The `inttype` defaults to `Int64`, but, for example, if you know +that your integer numbers are all small and want to save space, you can pass +`inttype=Int32`. Alternatively, if your JSON input has integers which are too large +for Int64, you can pass `inttype=Int128` or `inttype=BigInt`. `inttype` can be any +subtype of `Real`. + +```julia +JSONText(s::AbstractString) +``` +A wrapper around a Julia string representing JSON-formatted text, +which is inserted *as-is* in the JSON output of `JSON.print` and `JSON.json`. + +```julia +JSON.lower(p::Point2D) = [p.x, p.y] +``` + +Define a custom serialization rule for a particular data type. Must return a +value that can be directly serialized; see help for more details. diff --git a/packages/JSON/appveyor.yml b/packages/JSON/appveyor.yml new file mode 100644 index 0000000..912635f --- /dev/null +++ b/packages/JSON/appveyor.yml @@ -0,0 +1,43 @@ +environment: + matrix: + - julia_version: 0.7 + - julia_version: 1 + - julia_version: nightly + +platform: + - x86 # 32-bit + - x64 # 64-bit + +# # Uncomment the following lines to allow failures on nightly julia +# # (tests will run but not make your overall status red) +# matrix: +# allow_failures: +# - julia_version: nightly + +branches: + only: + - master + - /release-.*/ + +notifications: + - provider: Email + on_build_success: false + on_build_failure: false + on_build_status_changed: false + +install: + - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) + +build_script: + - echo "%JL_BUILD_SCRIPT%" + - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" + +test_script: + - echo "%JL_TEST_SCRIPT%" + - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" + +# # Uncomment to support code coverage upload. Should only be enabled for packages +# # which would have coverage gaps without running on Windows +# on_success: +# - echo "%JL_CODECOV_SCRIPT%" +# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" \ No newline at end of file diff --git a/packages/JSON/bench/bench.jl b/packages/JSON/bench/bench.jl new file mode 100644 index 0000000..a9b4be5 --- /dev/null +++ b/packages/JSON/bench/bench.jl @@ -0,0 +1,92 @@ +#!/usr/bin/julia --color=yes + +using ArgParse +using JSON + + +function bench(f, simulate=false) + fp = joinpath(JSON_DATA_DIR, string(f, ".json")) + if !isfile(fp) + println("Downloading benchmark file...") + download(DATA_SOURCES[f], fp) + end + GC.gc() # run gc so it doesn't affect benchmarks + t = if args["parse"]["parse-file"] + @elapsed JSON.parsefile(fp) + else + data = read(fp, String) + @elapsed JSON.Parser.parse(data) + end + + if !simulate + printstyled(" [Bench$FLAGS] "; color=:yellow) + println(f, " ", t, " seconds") + end + t +end + + +const JSON_DATA_DIR = joinpath(dirname(dirname(@__FILE__)), "data") +const s = ArgParseSettings(description="Benchmark JSON.jl") + +const DATA_SOURCES = Dict( + "canada" => "https://raw.githubusercontent.com/miloyip/nativejson-benchmark/v1.0.0/data/canada.json", + "citm_catalog" => "https://raw.githubusercontent.com/miloyip/nativejson-benchmark/v1.0.0/data/citm_catalog.json", + "citylots" => "https://raw.githubusercontent.com/zemirco/sf-city-lots-json/master/citylots.json", + "twitter" => "https://raw.githubusercontent.com/miloyip/nativejson-benchmark/v1.0.0/data/twitter.json") + +@add_arg_table s begin + "parse" + action = :command + help = "Run a JSON parser benchmark" + "list" + action = :command + help = "List available JSON files for use" +end + +@add_arg_table s["parse"] begin + "--include-compile", "-c" + help = "If set, include the compile time in measurements" + action = :store_true + "--parse-file", "-f" + help = "If set, measure JSON.parsefile, hence including IO time" + action = :store_true + "file" + help = "The JSON file to benchmark (leave out to benchmark all)" + required = false +end + +const args = parse_args(ARGS, s) + +if args["%COMMAND%"] == "parse" + const FLAGS = string( + args["parse"]["include-compile"] ? "C" : "", + args["parse"]["parse-file"] ? "F" : "") + + if args["parse"]["file"] ≠ nothing + const file = args["parse"]["file"] + + if !args["parse"]["include-compile"] + bench(file, true) + end + bench(file) + else + times = 1.0 + if args["parse"]["include-compile"] + error("Option --include-compile can only be used for single file.") + end + for k in sort(collect(keys(DATA_SOURCES))) + bench(k, true) # warm up compiler + end + for k in sort(collect(keys(DATA_SOURCES))) + times *= bench(k) # do benchmark + end + print_with_color(:yellow, " [Bench$FLAGS] ") + println("Total (G.M.) ", times^(1/length(DATA_SOURCES)), " seconds") + end +elseif args["%COMMAND%"] == "list" + println("Available benchmarks are:") + for k in sort(collect(keys(DATA_SOURCES))) + println(" • $k") + end +end diff --git a/packages/JSON/bench/micro.jl b/packages/JSON/bench/micro.jl new file mode 100644 index 0000000..9c3f653 --- /dev/null +++ b/packages/JSON/bench/micro.jl @@ -0,0 +1,56 @@ +# JSON Microbenchmarks +# 0.6 required for running benchmarks + +using JSON +using BenchmarkTools +using Dates + +const suite = BenchmarkGroup() + +suite["print"] = BenchmarkGroup(["serialize"]) +suite["pretty-print"] = BenchmarkGroup(["serialize"]) + +struct CustomListType + x::Int + y::Float64 + z::Union{CustomListType, Nothing} +end + +struct CustomTreeType + x::String + y::Union{CustomTreeType, Nothing} + z::Union{CustomTreeType, Nothing} +end + +list(x) = x == 0 ? nothing : CustomListType(1, 1.0, list(x - 1)) +tree(x) = x == 0 ? nothing : CustomTreeType("!!!", tree(x - 1), tree(x - 1)) + +const micros = Dict( + "integer" => 88, + "float" => -88.8, + "ascii" => "Hello World!", + "ascii-1024" => "x" ^ 1024, + "unicode" => "ສະ​ບາຍ​ດີ​ຊາວ​ໂລກ!", + "unicode-1024" => "ℜ" ^ 1024, + "bool" => true, + "null" => nothing, + "flat-homogenous-array-16" => collect(1:16), + "flat-homogenous-array-1024" => collect(1:1024), + "heterogenous-array" => [ + 1, 2, 3, 7, "A", "C", "E", "N", "Q", "R", "Shuttle to Grand Central"], + "nested-array-16^2" => [collect(1:16) for _ in 1:16], + "nested-array-16^3" => [[collect(1:16) for _ in 1:16] for _ in 1:16], + "small-dict" => Dict( + :a => :b, :c => "💙💙💙💙💙💙", :e => 10, :f => Dict(:a => :b)), + "flat-dict-128" => Dict(zip(collect(1:128), collect(1:128))), + "date" => Date(2016, 08, 09), + "matrix-16" => [i == j ? 1.0 : 0.0 for i in 1:16, j in 1:16], + "custom-list-128" => list(128), + "custom-tree-8" => tree(8)) + +for (k, v) in micros + io = IOBuffer() + suite["print"][k] = @benchmarkable JSON.print($(IOBuffer()), $v) + suite["pretty-print"][k] = @benchmarkable JSON.print( + $(IOBuffer()), $v, 4) +end diff --git a/packages/JSON/data/jsonchecker/fail01.json b/packages/JSON/data/jsonchecker/fail01.json new file mode 100644 index 0000000..92a451e --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail01.json @@ -0,0 +1 @@ +fable diff --git a/packages/JSON/data/jsonchecker/fail02.json b/packages/JSON/data/jsonchecker/fail02.json new file mode 100644 index 0000000..6b7c11e --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail02.json @@ -0,0 +1 @@ +["Unclosed array" \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail03.json b/packages/JSON/data/jsonchecker/fail03.json new file mode 100644 index 0000000..168c81e --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail03.json @@ -0,0 +1 @@ +{unquoted_key: "keys must be quoted"} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail04.json b/packages/JSON/data/jsonchecker/fail04.json new file mode 100644 index 0000000..9de168b --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail04.json @@ -0,0 +1 @@ +["extra comma",] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail05.json b/packages/JSON/data/jsonchecker/fail05.json new file mode 100644 index 0000000..ddf3ce3 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail05.json @@ -0,0 +1 @@ +["double extra comma",,] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail06.json b/packages/JSON/data/jsonchecker/fail06.json new file mode 100644 index 0000000..ed91580 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail06.json @@ -0,0 +1 @@ +[ , "<-- missing value"] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail07.json b/packages/JSON/data/jsonchecker/fail07.json new file mode 100644 index 0000000..8a96af3 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail07.json @@ -0,0 +1 @@ +["Comma after the close"], \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail08.json b/packages/JSON/data/jsonchecker/fail08.json new file mode 100644 index 0000000..b28479c --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail08.json @@ -0,0 +1 @@ +["Extra close"]] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail09.json b/packages/JSON/data/jsonchecker/fail09.json new file mode 100644 index 0000000..5815574 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail09.json @@ -0,0 +1 @@ +{"Extra comma": true,} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail10.json b/packages/JSON/data/jsonchecker/fail10.json new file mode 100644 index 0000000..5d8c004 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail10.json @@ -0,0 +1 @@ +{"Extra value after close": true} "misplaced quoted value" \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail11.json b/packages/JSON/data/jsonchecker/fail11.json new file mode 100644 index 0000000..76eb95b --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail11.json @@ -0,0 +1 @@ +{"Illegal expression": 1 + 2} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail12.json b/packages/JSON/data/jsonchecker/fail12.json new file mode 100644 index 0000000..77580a4 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail12.json @@ -0,0 +1 @@ +{"Illegal invocation": alert()} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail13.json b/packages/JSON/data/jsonchecker/fail13.json new file mode 100644 index 0000000..379406b --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail13.json @@ -0,0 +1 @@ +{"Numbers cannot have leading zeroes": 013} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail14.json b/packages/JSON/data/jsonchecker/fail14.json new file mode 100644 index 0000000..0ed366b --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail14.json @@ -0,0 +1 @@ +{"Numbers cannot be hex": 0x14} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail15.json b/packages/JSON/data/jsonchecker/fail15.json new file mode 100644 index 0000000..fc8376b --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail15.json @@ -0,0 +1 @@ +["Illegal backslash escape: \x15"] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail16.json b/packages/JSON/data/jsonchecker/fail16.json new file mode 100644 index 0000000..3fe21d4 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail16.json @@ -0,0 +1 @@ +[\naked] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail17.json b/packages/JSON/data/jsonchecker/fail17.json new file mode 100644 index 0000000..62b9214 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail17.json @@ -0,0 +1 @@ +["Illegal backslash escape: \017"] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail18.json b/packages/JSON/data/jsonchecker/fail18.json new file mode 100644 index 0000000..bd7f1d6 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail18.json @@ -0,0 +1,2 @@ +"mutliple" +"things" diff --git a/packages/JSON/data/jsonchecker/fail19.json b/packages/JSON/data/jsonchecker/fail19.json new file mode 100644 index 0000000..3b9c46f --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail19.json @@ -0,0 +1 @@ +{"Missing colon" null} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail20.json b/packages/JSON/data/jsonchecker/fail20.json new file mode 100644 index 0000000..27c1af3 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail20.json @@ -0,0 +1 @@ +{"Double colon":: null} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail21.json b/packages/JSON/data/jsonchecker/fail21.json new file mode 100644 index 0000000..6247457 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail21.json @@ -0,0 +1 @@ +{"Comma instead of colon", null} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail22.json b/packages/JSON/data/jsonchecker/fail22.json new file mode 100644 index 0000000..a775258 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail22.json @@ -0,0 +1 @@ +["Colon instead of comma": false] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail23.json b/packages/JSON/data/jsonchecker/fail23.json new file mode 100644 index 0000000..494add1 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail23.json @@ -0,0 +1 @@ +["Bad value", truth] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail24.json b/packages/JSON/data/jsonchecker/fail24.json new file mode 100644 index 0000000..caff239 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail24.json @@ -0,0 +1 @@ +['single quote'] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail25.json b/packages/JSON/data/jsonchecker/fail25.json new file mode 100644 index 0000000..8b7ad23 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail25.json @@ -0,0 +1 @@ +[" tab character in string "] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail26.json b/packages/JSON/data/jsonchecker/fail26.json new file mode 100644 index 0000000..845d26a --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail26.json @@ -0,0 +1 @@ +["tab\ character\ in\ string\ "] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail27.json b/packages/JSON/data/jsonchecker/fail27.json new file mode 100644 index 0000000..6b01a2c --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail27.json @@ -0,0 +1,2 @@ +["line +break"] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail28.json b/packages/JSON/data/jsonchecker/fail28.json new file mode 100644 index 0000000..621a010 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail28.json @@ -0,0 +1,2 @@ +["line\ +break"] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail29.json b/packages/JSON/data/jsonchecker/fail29.json new file mode 100644 index 0000000..47ec421 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail29.json @@ -0,0 +1 @@ +[0e] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail30.json b/packages/JSON/data/jsonchecker/fail30.json new file mode 100644 index 0000000..8ab0bc4 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail30.json @@ -0,0 +1 @@ +[0e+] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail31.json b/packages/JSON/data/jsonchecker/fail31.json new file mode 100644 index 0000000..1cce602 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail31.json @@ -0,0 +1 @@ +[0e+-1] \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail32.json b/packages/JSON/data/jsonchecker/fail32.json new file mode 100644 index 0000000..cb1f560 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail32.json @@ -0,0 +1 @@ +{"Comma instead of closing brace": true, diff --git a/packages/JSON/data/jsonchecker/fail33.json b/packages/JSON/data/jsonchecker/fail33.json new file mode 100644 index 0000000..ca5eb19 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail33.json @@ -0,0 +1 @@ +["mismatch"} \ No newline at end of file diff --git a/packages/JSON/data/jsonchecker/fail34.json b/packages/JSON/data/jsonchecker/fail34.json new file mode 100644 index 0000000..7ce16bd --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail34.json @@ -0,0 +1 @@ +{"garbage" before : "separator"} diff --git a/packages/JSON/data/jsonchecker/fail35.json b/packages/JSON/data/jsonchecker/fail35.json new file mode 100644 index 0000000..7a46973 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail35.json @@ -0,0 +1 @@ +{"no separator" diff --git a/packages/JSON/data/jsonchecker/fail36.json b/packages/JSON/data/jsonchecker/fail36.json new file mode 100644 index 0000000..bf08400 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail36.json @@ -0,0 +1 @@ +{"no closing brace": true diff --git a/packages/JSON/data/jsonchecker/fail37.json b/packages/JSON/data/jsonchecker/fail37.json new file mode 100644 index 0000000..558ed37 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail37.json @@ -0,0 +1 @@ +[ diff --git a/packages/JSON/data/jsonchecker/fail38.json b/packages/JSON/data/jsonchecker/fail38.json new file mode 100644 index 0000000..98232c6 --- /dev/null +++ b/packages/JSON/data/jsonchecker/fail38.json @@ -0,0 +1 @@ +{ diff --git a/packages/JSON/data/jsonchecker/pass01.json b/packages/JSON/data/jsonchecker/pass01.json new file mode 100644 index 0000000..2c10f22 --- /dev/null +++ b/packages/JSON/data/jsonchecker/pass01.json @@ -0,0 +1,58 @@ +[ + "JSON Test Pattern pass1", + {"object with 1 member":["array with 1 element"]}, + {}, + [], + -42, + true, + false, + null, + { + "integer": 1234567890, + "real": -9876.543210, + "e": 0.123456789e-12, + "E": 1.234567890E+34, + "": 23456789012E66, + "zero": 0, + "one": 1, + "space": " ", + "quote": "\"", + "backslash": "\\", + "controls": "\b\f\n\r\t", + "slash": "/ & \/", + "alpha": "abcdefghijklmnopqrstuvwyz", + "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", + "digit": "0123456789", + "0123456789": "digit", + "special": "`1~!@#$%^&*()_+-={':[,]}|;.?", + "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", + "true": true, + "false": false, + "null": null, + "array":[ ], + "object":{ }, + "address": "50 St. James Street", + "url": "http://www.JSON.org/", + "comment": "// /* */": " ", + " s p a c e d " :[1,2 , 3 + +, + +4 , 5 , 6 ,7 ],"compact":[1,2,3,4,5,6,7], + "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", + "quotes": "" \u0022 %22 0x22 034 "", + "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" +: "A key can be any string" + }, + 0.5 ,98.6 +, +99.44 +, + +1066, +1e1, +0.1e1, +1e-1, +1e00,2e+00,2e-00 +,"rosebud"] diff --git a/packages/JSON/data/jsonchecker/pass02.json b/packages/JSON/data/jsonchecker/pass02.json new file mode 100644 index 0000000..fea5710 --- /dev/null +++ b/packages/JSON/data/jsonchecker/pass02.json @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] diff --git a/packages/JSON/data/jsonchecker/pass03.json b/packages/JSON/data/jsonchecker/pass03.json new file mode 100644 index 0000000..4528d51 --- /dev/null +++ b/packages/JSON/data/jsonchecker/pass03.json @@ -0,0 +1,6 @@ +{ + "JSON Test Pattern pass3": { + "The outermost value": "must be an object or array.", + "In this test": "It is an object." + } +} diff --git a/packages/JSON/data/jsonchecker/readme.txt b/packages/JSON/data/jsonchecker/readme.txt new file mode 100644 index 0000000..321d89d --- /dev/null +++ b/packages/JSON/data/jsonchecker/readme.txt @@ -0,0 +1,3 @@ +Test suite from http://json.org/JSON_checker/. + +If the JSON_checker is working correctly, it must accept all of the pass*.json files and reject all of the fail*.json files. diff --git a/packages/JSON/data/roundtrip/roundtrip01.json b/packages/JSON/data/roundtrip/roundtrip01.json new file mode 100644 index 0000000..500db4a --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip01.json @@ -0,0 +1 @@ +[null] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip02.json b/packages/JSON/data/roundtrip/roundtrip02.json new file mode 100644 index 0000000..de601e3 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip02.json @@ -0,0 +1 @@ +[true] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip03.json b/packages/JSON/data/roundtrip/roundtrip03.json new file mode 100644 index 0000000..67b2f07 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip03.json @@ -0,0 +1 @@ +[false] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip04.json b/packages/JSON/data/roundtrip/roundtrip04.json new file mode 100644 index 0000000..6e7ea63 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip04.json @@ -0,0 +1 @@ +[0] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip05.json b/packages/JSON/data/roundtrip/roundtrip05.json new file mode 100644 index 0000000..6dfd298 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip05.json @@ -0,0 +1 @@ +["foo"] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip06.json b/packages/JSON/data/roundtrip/roundtrip06.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip06.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip07.json b/packages/JSON/data/roundtrip/roundtrip07.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip07.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip08.json b/packages/JSON/data/roundtrip/roundtrip08.json new file mode 100644 index 0000000..bfa3412 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip08.json @@ -0,0 +1 @@ +[0,1] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip09.json b/packages/JSON/data/roundtrip/roundtrip09.json new file mode 100644 index 0000000..9f5dd4e --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip09.json @@ -0,0 +1 @@ +{"foo":"bar"} \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip10.json b/packages/JSON/data/roundtrip/roundtrip10.json new file mode 100644 index 0000000..2355b4d --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip10.json @@ -0,0 +1 @@ +{"a":null,"foo":"bar"} \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip11.json b/packages/JSON/data/roundtrip/roundtrip11.json new file mode 100644 index 0000000..99d21a2 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip11.json @@ -0,0 +1 @@ +[-1] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip12.json b/packages/JSON/data/roundtrip/roundtrip12.json new file mode 100644 index 0000000..56c78be --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip12.json @@ -0,0 +1 @@ +[-2147483648] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip13.json b/packages/JSON/data/roundtrip/roundtrip13.json new file mode 100644 index 0000000..029580f --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip13.json @@ -0,0 +1 @@ +[-1234567890123456789] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip14.json b/packages/JSON/data/roundtrip/roundtrip14.json new file mode 100644 index 0000000..d865800 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip14.json @@ -0,0 +1 @@ +[-9223372036854775808] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip15.json b/packages/JSON/data/roundtrip/roundtrip15.json new file mode 100644 index 0000000..bace2a0 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip15.json @@ -0,0 +1 @@ +[1] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip16.json b/packages/JSON/data/roundtrip/roundtrip16.json new file mode 100644 index 0000000..dfe696d --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip16.json @@ -0,0 +1 @@ +[2147483647] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip17.json b/packages/JSON/data/roundtrip/roundtrip17.json new file mode 100644 index 0000000..6640b07 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip17.json @@ -0,0 +1 @@ +[4294967295] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip18.json b/packages/JSON/data/roundtrip/roundtrip18.json new file mode 100644 index 0000000..a3ab143 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip18.json @@ -0,0 +1 @@ +[1234567890123456789] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip19.json b/packages/JSON/data/roundtrip/roundtrip19.json new file mode 100644 index 0000000..8ab4a50 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip19.json @@ -0,0 +1 @@ +[9223372036854775807] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip20.json b/packages/JSON/data/roundtrip/roundtrip20.json new file mode 100644 index 0000000..92df1df --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip20.json @@ -0,0 +1 @@ +[0.0] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip21.json b/packages/JSON/data/roundtrip/roundtrip21.json new file mode 100644 index 0000000..cfef815 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip21.json @@ -0,0 +1 @@ +[-0.0] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip22.json b/packages/JSON/data/roundtrip/roundtrip22.json new file mode 100644 index 0000000..a7b7eef --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip22.json @@ -0,0 +1 @@ +[1.2345] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip23.json b/packages/JSON/data/roundtrip/roundtrip23.json new file mode 100644 index 0000000..b553e84 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip23.json @@ -0,0 +1 @@ +[-1.2345] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip24.json b/packages/JSON/data/roundtrip/roundtrip24.json new file mode 100644 index 0000000..f01efb6 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip24.json @@ -0,0 +1 @@ +[5e-324] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip25.json b/packages/JSON/data/roundtrip/roundtrip25.json new file mode 100644 index 0000000..cdef14d --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip25.json @@ -0,0 +1 @@ +[2.225073858507201e-308] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip26.json b/packages/JSON/data/roundtrip/roundtrip26.json new file mode 100644 index 0000000..f4121b7 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip26.json @@ -0,0 +1 @@ +[2.2250738585072014e-308] \ No newline at end of file diff --git a/packages/JSON/data/roundtrip/roundtrip27.json b/packages/JSON/data/roundtrip/roundtrip27.json new file mode 100644 index 0000000..17ce521 --- /dev/null +++ b/packages/JSON/data/roundtrip/roundtrip27.json @@ -0,0 +1 @@ +[1.7976931348623157e308] \ No newline at end of file diff --git a/packages/JSON/src/Common.jl b/packages/JSON/src/Common.jl new file mode 100644 index 0000000..55b1fe5 --- /dev/null +++ b/packages/JSON/src/Common.jl @@ -0,0 +1,11 @@ +""" +Internal implementation detail. +""" +module Common + +using Unicode + +include("bytes.jl") +include("errors.jl") + +end diff --git a/packages/JSON/src/JSON.jl b/packages/JSON/src/JSON.jl new file mode 100644 index 0000000..66fb855 --- /dev/null +++ b/packages/JSON/src/JSON.jl @@ -0,0 +1,31 @@ +VERSION < v"0.7.0-beta2.199" && __precompile__() + +module JSON + +export json # returns a compact (or indented) JSON representation as a string +export JSONText # string wrapper to insert raw JSON into JSON output + +include("Common.jl") + +# Parser modules +include("Parser.jl") + +# Writer modules +include("Serializations.jl") +include("Writer.jl") + +# stuff to re-"export" +# note that this package does not actually export anything except `json` but +# all of the following are part of the public interface in one way or another +using .Parser: parse, parsefile +using .Writer: show_json, json, lower, print, StructuralContext, show_element, + show_string, show_key, show_pair, show_null, begin_array, + end_array, begin_object, end_object, indent, delimit, separate, + JSONText +using .Serializations: Serialization, CommonSerialization, + StandardSerialization + +# for pretty-printed (non-compact) output, JSONText must be re-parsed: +Writer.lower(json::JSONText) = parse(json.s) + +end # module diff --git a/packages/JSON/src/Parser.jl b/packages/JSON/src/Parser.jl new file mode 100644 index 0000000..b7556bb --- /dev/null +++ b/packages/JSON/src/Parser.jl @@ -0,0 +1,444 @@ +module Parser # JSON + +using Mmap +using ..Common + +include("pushvector.jl") + +""" +Like `isspace`, but work on bytes and includes only the four whitespace +characters defined by the JSON standard: space, tab, line feed, and carriage +return. +""" +isjsonspace(b::UInt8) = b == SPACE || b == TAB || b == NEWLINE || b == RETURN + +""" +Like `isdigit`, but for bytes. +""" +isjsondigit(b::UInt8) = DIGIT_ZERO ≤ b ≤ DIGIT_NINE + +abstract type ParserState end + +mutable struct MemoryParserState <: ParserState + utf8::String + s::Int +end + +# it is convenient to access MemoryParserState like a Vector{UInt8} to avoid copies +Base.@propagate_inbounds Base.getindex(state::MemoryParserState, i::Int) = codeunit(state.utf8, i) +Base.length(state::MemoryParserState) = sizeof(state.utf8) +Base.unsafe_convert(::Type{Ptr{UInt8}}, state::MemoryParserState) = Base.unsafe_convert(Ptr{UInt8}, state.utf8) + +mutable struct StreamingParserState{T <: IO} <: ParserState + io::T + cur::UInt8 + used::Bool + utf8array::PushVector{UInt8, Vector{UInt8}} +end +StreamingParserState(io::IO) = StreamingParserState(io, 0x00, true, PushVector{UInt8}()) + +struct ParserContext{DictType, IntType} end + +""" +Return the byte at the current position of the `ParserState`. If there is no +byte (that is, the `ParserState` is done), then an error is thrown that the +input ended unexpectedly. +""" +@inline function byteat(ps::MemoryParserState) + @inbounds if hasmore(ps) + return ps[ps.s] + else + _error(E_UNEXPECTED_EOF, ps) + end +end + +@inline function byteat(ps::StreamingParserState) + if ps.used + ps.used = false + if eof(ps.io) + _error(E_UNEXPECTED_EOF, ps) + else + ps.cur = read(ps.io, UInt8) + end + end + ps.cur +end + +""" +Like `byteat`, but with no special bounds check and error message. Useful when +a current byte is known to exist. +""" +@inline current(ps::MemoryParserState) = ps[ps.s] +@inline current(ps::StreamingParserState) = byteat(ps) + +""" +Require the current byte of the `ParserState` to be the given byte, and then +skip past that byte. Otherwise, an error is thrown. +""" +@inline function skip!(ps::ParserState, c::UInt8) + if byteat(ps) == c + incr!(ps) + else + _error_expected_char(c, ps) + end +end +@noinline _error_expected_char(c, ps) = _error("Expected '$(Char(c))' here", ps) + +function skip!(ps::ParserState, cs::UInt8...) + for c in cs + skip!(ps, c) + end +end + +""" +Move the `ParserState` to the next byte. +""" +@inline incr!(ps::MemoryParserState) = (ps.s += 1) +@inline incr!(ps::StreamingParserState) = (ps.used = true) + +""" +Move the `ParserState` to the next byte, and return the value at the byte before +the advancement. If the `ParserState` is already done, then throw an error. +""" +@inline advance!(ps::ParserState) = (b = byteat(ps); incr!(ps); b) + +""" +Return `true` if there is a current byte, and `false` if all bytes have been +exausted. +""" +@inline hasmore(ps::MemoryParserState) = ps.s ≤ length(ps) +@inline hasmore(ps::StreamingParserState) = true # no more now ≠ no more ever + +""" +Remove as many whitespace bytes as possible from the `ParserState` starting from +the current byte. +""" +@inline function chomp_space!(ps::ParserState) + @inbounds while hasmore(ps) && isjsonspace(current(ps)) + incr!(ps) + end +end + + +# Used for line counts +function _count_before(haystack::AbstractString, needle::Char, _end::Int) + count = 0 + for (i,c) in enumerate(haystack) + i >= _end && return count + count += c == needle + end + return count +end + + +# Throws an error message with an indicator to the source +@noinline function _error(message::AbstractString, ps::MemoryParserState) + orig = ps.utf8 + lines = _count_before(orig, '\n', ps.s) + # Replace all special multi-line/multi-space characters with a space. + strnl = replace(orig, r"[\b\f\n\r\t\s]" => " ") + li = (ps.s > 20) ? ps.s - 9 : 1 # Left index + ri = min(lastindex(orig), ps.s + 20) # Right index + error(message * + "\nLine: " * string(lines) * + "\nAround: ..." * strnl[li:ri] * "..." * + "\n " * (" " ^ (ps.s - li)) * "^\n" + ) +end + +@noinline function _error(message::AbstractString, ps::StreamingParserState) + error("$message\n ...when parsing byte with value '$(current(ps))'") +end + +# PARSING + +""" +Given a `ParserState`, after possibly any amount of whitespace, return the next +parseable value. +""" +function parse_value(pc::ParserContext, ps::ParserState) + chomp_space!(ps) + + @inbounds byte = byteat(ps) + if byte == STRING_DELIM + parse_string(ps) + elseif isjsondigit(byte) || byte == MINUS_SIGN + parse_number(pc, ps) + elseif byte == OBJECT_BEGIN + parse_object(pc, ps) + elseif byte == ARRAY_BEGIN + parse_array(pc, ps) + else + parse_jsconstant(ps::ParserState) + end +end + +function parse_jsconstant(ps::ParserState) + c = advance!(ps) + if c == LATIN_T # true + skip!(ps, LATIN_R, LATIN_U, LATIN_E) + true + elseif c == LATIN_F # false + skip!(ps, LATIN_A, LATIN_L, LATIN_S, LATIN_E) + false + elseif c == LATIN_N # null + skip!(ps, LATIN_U, LATIN_L, LATIN_L) + nothing + else + _error(E_UNEXPECTED_CHAR, ps) + end +end + +function parse_array(pc::ParserContext, ps::ParserState) + result = Any[] + @inbounds incr!(ps) # Skip over opening '[' + chomp_space!(ps) + if byteat(ps) ≠ ARRAY_END # special case for empty array + @inbounds while true + push!(result, parse_value(pc, ps)) + chomp_space!(ps) + byteat(ps) == ARRAY_END && break + skip!(ps, DELIMITER) + end + end + + @inbounds incr!(ps) + result +end + + +function parse_object(pc::ParserContext{DictType, <:Real}, ps::ParserState) where DictType + obj = DictType() + keyT = keytype(typeof(obj)) + + incr!(ps) # Skip over opening '{' + chomp_space!(ps) + if byteat(ps) ≠ OBJECT_END # special case for empty object + @inbounds while true + # Read key + chomp_space!(ps) + byteat(ps) == STRING_DELIM || _error(E_BAD_KEY, ps) + key = parse_string(ps) + chomp_space!(ps) + skip!(ps, SEPARATOR) + # Read value + value = parse_value(pc, ps) + chomp_space!(ps) + obj[keyT === Symbol ? Symbol(key) : convert(keyT, key)] = value + byteat(ps) == OBJECT_END && break + skip!(ps, DELIMITER) + end + end + + incr!(ps) + obj +end + + +utf16_is_surrogate(c::UInt16) = (c & 0xf800) == 0xd800 +utf16_get_supplementary(lead::UInt16, trail::UInt16) = Char(UInt32(lead-0xd7f7)<<10 + trail) + +function read_four_hex_digits!(ps::ParserState) + local n::UInt16 = 0 + + for _ in 1:4 + b = advance!(ps) + n = n << 4 + if isjsondigit(b) + b - DIGIT_ZERO + elseif LATIN_A ≤ b ≤ LATIN_F + b - (LATIN_A - UInt8(10)) + elseif LATIN_UPPER_A ≤ b ≤ LATIN_UPPER_F + b - (LATIN_UPPER_A - UInt8(10)) + else + _error(E_BAD_ESCAPE, ps) + end + end + + n +end + +function read_unicode_escape!(ps) + u1 = read_four_hex_digits!(ps) + if utf16_is_surrogate(u1) + skip!(ps, BACKSLASH) + skip!(ps, LATIN_U) + u2 = read_four_hex_digits!(ps) + utf16_get_supplementary(u1, u2) + else + Char(u1) + end +end + +function parse_string(ps::ParserState) + b = IOBuffer() + incr!(ps) # skip opening quote + while true + c = advance!(ps) + + if c == BACKSLASH + c = advance!(ps) + if c == LATIN_U # Unicode escape + write(b, read_unicode_escape!(ps)) + else + c = get(ESCAPES, c, 0x00) + c == 0x00 && _error(E_BAD_ESCAPE, ps) + write(b, c) + end + continue + elseif c < SPACE + _error(E_BAD_CONTROL, ps) + elseif c == STRING_DELIM + return String(take!(b)) + end + + write(b, c) + end +end + +""" +Return `true` if the given bytes vector, starting at `from` and ending at `to`, +has a leading zero. +""" +function hasleadingzero(bytes, from::Int, to::Int) + c = bytes[from] + from + 1 < to && c == UInt8('-') && + bytes[from + 1] == DIGIT_ZERO && isjsondigit(bytes[from + 2]) || + from < to && to > from + 1 && c == DIGIT_ZERO && + isjsondigit(bytes[from + 1]) +end + +""" +Parse a float from the given bytes vector, starting at `from` and ending at the +byte before `to`. Bytes enclosed should all be ASCII characters. +""" +function float_from_bytes(bytes, from::Int, to::Int) + # The ccall is not ideal (Base.tryparse would be better), but it actually + # makes an 2× difference to performance + hasvalue, val = ccall(:jl_try_substrtod, Tuple{Bool, Float64}, + (Ptr{UInt8}, Csize_t, Csize_t), bytes, from - 1, to - from + 1) + hasvalue ? val : nothing +end + +""" +Parse an integer from the given bytes vector, starting at `from` and ending at +the byte before `to`. Bytes enclosed should all be ASCII characters. +""" +function int_from_bytes(pc::ParserContext{<:Any,IntType}, + ps::ParserState, + bytes, + from::Int, + to::Int) where IntType <: Real + @inbounds isnegative = bytes[from] == MINUS_SIGN ? (from += 1; true) : false + num = IntType(0) + @inbounds for i in from:to + c = bytes[i] + dig = c - DIGIT_ZERO + if dig < 0x10 + num = IntType(10) * num + IntType(dig) + else + _error(E_BAD_NUMBER, ps) + end + end + ifelse(isnegative, -num, num) +end + +function number_from_bytes(pc::ParserContext, + ps::ParserState, + isint::Bool, + bytes, + from::Int, + to::Int) + @inbounds if hasleadingzero(bytes, from, to) + _error(E_LEADING_ZERO, ps) + end + + if isint + @inbounds if to == from && bytes[from] == MINUS_SIGN + _error(E_BAD_NUMBER, ps) + end + int_from_bytes(pc, ps, bytes, from, to) + else + res = float_from_bytes(bytes, from, to) + res === nothing ? _error(E_BAD_NUMBER, ps) : res + end +end + + +function parse_number(pc::ParserContext, ps::ParserState) + # Determine the end of the floating point by skipping past ASCII values + # 0-9, +, -, e, E, and . + number = ps.utf8array + isint = true + + @inbounds while hasmore(ps) + c = current(ps) + + if isjsondigit(c) || c == MINUS_SIGN + push!(number, UInt8(c)) + elseif c in (PLUS_SIGN, LATIN_E, LATIN_UPPER_E, DECIMAL_POINT) + push!(number, UInt8(c)) + isint = false + else + break + end + + incr!(ps) + end + + v = number_from_bytes(pc, ps, isint, number, 1, length(number)) + resize!(number, 0) + return v +end + +unparameterize_type(x) = x # Fallback for nontypes -- functions etc +function unparameterize_type(T::Type) + candidate = typeintersect(T, AbstractDict{String, Any}) + candidate <: Union{} ? T : candidate +end + +# Workaround for slow dynamic dispatch for creating objects +const DEFAULT_PARSERCONTEXT = ParserContext{Dict{String, Any}, Int64}() +function _get_parsercontext(dicttype, inttype) + if dicttype == Dict{String, Any} && inttype == Int64 + DEFAULT_PARSERCONTEXT + else + ParserContext{unparameterize_type(dicttype), inttype}.instance + end +end + +function parse(str::AbstractString; + dicttype=Dict{String,Any}, + inttype::Type{<:Real}=Int64) + pc = _get_parsercontext(dicttype, inttype) + ps = MemoryParserState(str, 1) + v = parse_value(pc, ps) + chomp_space!(ps) + if hasmore(ps) + _error(E_EXPECTED_EOF, ps) + end + v +end + +function parse(io::IO; + dicttype=Dict{String,Any}, + inttype::Type{<:Real}=Int64) + pc = _get_parsercontext(dicttype, inttype) + ps = StreamingParserState(io) + parse_value(pc, ps) +end + +function parsefile(filename::AbstractString; + dicttype=Dict{String, Any}, + inttype::Type{<:Real}=Int64, + use_mmap=true) + sz = filesize(filename) + open(filename) do io + s = use_mmap ? String(Mmap.mmap(io, Vector{UInt8}, sz)) : read(io, String) + parse(s; dicttype=dicttype, inttype=inttype) + end +end + +# Efficient implementations of some of the above for in-memory parsing +include("specialized.jl") + +end # module Parser diff --git a/packages/JSON/src/Serializations.jl b/packages/JSON/src/Serializations.jl new file mode 100644 index 0000000..e4398ce --- /dev/null +++ b/packages/JSON/src/Serializations.jl @@ -0,0 +1,39 @@ +""" +JSON writer serialization contexts. + +This module defines the `Serialization` abstract type and several concrete +implementations, as they relate to JSON. +""" +module Serializations + +using ..Common + +""" +A `Serialization` defines how objects are lowered to JSON format. +""" +abstract type Serialization end + +""" +The `CommonSerialization` comes with a default set of rules for serializing +Julia types to their JSON equivalents. Additional rules are provided either by +packages explicitly defining `JSON.show_json` for this serialization, or by the +`JSON.lower` method. Most concrete implementations of serializers should subtype +`CommonSerialization`, unless it is desirable to bypass the `lower` system, in +which case `Serialization` should be subtyped. +""" +abstract type CommonSerialization <: Serialization end + +""" +The `StandardSerialization` defines a common, standard JSON serialization format +that is optimized to: + +- strictly follow the JSON standard +- be useful in the greatest number of situations + +All serializations defined for `CommonSerialization` are inherited by +`StandardSerialization`. It is therefore generally advised to add new +serialization behaviour to `CommonSerialization`. +""" +struct StandardSerialization <: CommonSerialization end + +end diff --git a/packages/JSON/src/Writer.jl b/packages/JSON/src/Writer.jl new file mode 100644 index 0000000..5c4cc63 --- /dev/null +++ b/packages/JSON/src/Writer.jl @@ -0,0 +1,357 @@ +module Writer + +using Dates +using ..Common +using ..Serializations: Serialization, StandardSerialization, + CommonSerialization + +using Unicode + + +""" +Internal JSON.jl implementation detail; do not depend on this type. + +A JSON primitive that wraps around any composite type to enable `Dict`-like +serialization. +""" +struct CompositeTypeWrapper{T} + wrapped::T + fns::Vector{Symbol} +end + +CompositeTypeWrapper(x, syms) = CompositeTypeWrapper(x, collect(syms)) +CompositeTypeWrapper(x) = CompositeTypeWrapper(x, fieldnames(typeof(x))) + +""" + lower(x) + +Return a value of a JSON-encodable primitive type that `x` should be lowered +into before encoding as JSON. Supported types are: `AbstractDict` to JSON +objects, `Tuple` and `AbstractVector` to JSON arrays, `AbstractArray` to nested +JSON arrays, `AbstractString`, `Symbol`, `Enum`, or `Char` to JSON string, +`Integer` and `AbstractFloat` to JSON number, `Bool` to JSON boolean, and +`Nothing` to JSON null, or any other types with a `show_json` method defined. + +Extensions of this method should preserve the property that the return value is +one of the aforementioned types. If first lowering to some intermediate type is +required, then extensions should call `lower` before returning a value. + +Note that the return value need not be *recursively* lowered—this function may +for instance return an `AbstractArray{Any, 1}` whose elements are not JSON +primitives. +""" +function lower(a) + if nfields(a) > 0 + CompositeTypeWrapper(a) + else + error("Cannot serialize type $(typeof(a))") + end +end + +# To avoid allocating an intermediate string, we directly define `show_json` +# for this type instead of lowering it to a string first (which would +# allocate). However, the `show_json` method does call `lower` so as to allow +# users to change the lowering of their `Enum` or even `AbstractString` +# subtypes if necessary. +const IsPrintedAsString = Union{ + Dates.TimeType, Char, Type, AbstractString, Enum, Symbol} +lower(x::IsPrintedAsString) = x + +lower(m::Module) = throw(ArgumentError("cannot serialize Module $m as JSON")) +lower(x::Real) = convert(Float64, x) +lower(x::Base.AbstractSet) = collect(x) + +""" +Abstract supertype of all JSON and JSON-like structural writer contexts. +""" +abstract type StructuralContext <: IO end + +""" +Internal implementation detail. + +A JSON structural context around an `IO` object. Structural writer contexts +define the behaviour of serializing JSON structural objects, such as objects, +arrays, and strings to JSON. The translation of Julia types to JSON structural +objects is not handled by a `JSONContext`, but by a `Serialization` wrapper +around it. Abstract supertype of `PrettyContext` and `CompactContext`. Data can +be written to a JSON context in the usual way, but often higher-level operations +such as `begin_array` or `begin_object` are preferred to directly writing bytes +to the stream. +""" +abstract type JSONContext <: StructuralContext end + +""" +Internal implementation detail. + +Keeps track of the current location in the array or object, which winds and +unwinds during serialization. +""" +mutable struct PrettyContext{T<:IO} <: JSONContext + io::T + step::Int # number of spaces to step + state::Int # number of steps at present + first::Bool # whether an object/array was just started +end +PrettyContext(io::IO, step) = PrettyContext(io, step, 0, false) + +""" +Internal implementation detail. + +For compact printing, which in JSON is fully recursive. +""" +mutable struct CompactContext{T<:IO} <: JSONContext + io::T + first::Bool +end +CompactContext(io::IO) = CompactContext(io, false) + +""" +Internal implementation detail. + +Implements an IO context safe for printing into JSON strings. +""" +struct StringContext{T<:IO} <: IO + io::T +end + +# These aliases make defining additional methods on `show_json` easier. +const CS = CommonSerialization +const SC = StructuralContext + +# Low-level direct access +Base.write(io::JSONContext, byte::UInt8) = write(io.io, byte) +Base.write(io::StringContext, byte::UInt8) = + write(io.io, ESCAPED_ARRAY[byte + 0x01]) +#= turn on if there's a performance benefit +write(io::StringContext, char::Char) = + char <= '\x7f' ? write(io, ESCAPED_ARRAY[UInt8(c) + 0x01]) : + Base.print(io, c) +=# + +""" + indent(io::StructuralContext) + +If appropriate, write a newline to the given context, then indent it by the +appropriate number of spaces. Otherwise, do nothing. +""" +@inline function indent(io::PrettyContext) + write(io, NEWLINE) + for _ in 1:io.state + write(io, SPACE) + end +end +@inline indent(io::CompactContext) = nothing + +""" + separate(io::StructuralContext) + +Write a colon, followed by a space if appropriate, to the given context. +""" +@inline separate(io::PrettyContext) = write(io, SEPARATOR, SPACE) +@inline separate(io::CompactContext) = write(io, SEPARATOR) + +""" + delimit(io::StructuralContext) + +If this is not the first item written in a collection, write a comma in the +structural context. Otherwise, do not write a comma, but set a flag that the +first element has been written already. +""" +@inline function delimit(io::JSONContext) + if !io.first + write(io, DELIMITER) + end + io.first = false +end + +for kind in ("object", "array") + beginfn = Symbol("begin_", kind) + beginsym = Symbol(uppercase(kind), "_BEGIN") + endfn = Symbol("end_", kind) + endsym = Symbol(uppercase(kind), "_END") + # Begin and end objects + @eval function $beginfn(io::PrettyContext) + write(io, $beginsym) + io.state += io.step + io.first = true + end + @eval $beginfn(io::CompactContext) = (write(io, $beginsym); io.first = true) + @eval function $endfn(io::PrettyContext) + io.state -= io.step + if !io.first + indent(io) + end + write(io, $endsym) + io.first = false + end + @eval $endfn(io::CompactContext) = (write(io, $endsym); io.first = false) +end + +""" + show_string(io::IO, str) + +Print `str` as a JSON string (that is, properly escaped and wrapped by double +quotes) to the given IO object `io`. +""" +function show_string(io::IO, x) + write(io, STRING_DELIM) + Base.print(StringContext(io), x) + write(io, STRING_DELIM) +end + +""" + show_null(io::IO) + +Print the string `null` to the given IO object `io`. +""" +show_null(io::IO) = Base.print(io, "null") + +""" + show_element(io::StructuralContext, s, x) + +Print object `x` as an element of a JSON array to context `io` using rules +defined by serialization `s`. +""" +function show_element(io::JSONContext, s, x) + delimit(io) + indent(io) + show_json(io, s, x) +end + +""" + show_key(io::StructuralContext, k) + +Print string `k` as the key of a JSON key-value pair to context `io`. +""" +function show_key(io::JSONContext, k) + delimit(io) + indent(io) + show_string(io, k) + separate(io) +end + +""" + show_pair(io::StructuralContext, s, k, v) + +Print the key-value pair defined by `k => v` as JSON to context `io`, using +rules defined by serialization `s`. +""" +function show_pair(io::JSONContext, s, k, v) + show_key(io, k) + show_json(io, s, v) +end +show_pair(io::JSONContext, s, kv) = show_pair(io, s, first(kv), last(kv)) + +# Default serialization rules for CommonSerialization (CS) +function show_json(io::SC, s::CS, x::IsPrintedAsString) + # We need this check to allow `lower(x::Enum)` overrides to work if needed; + # it should be optimized out if `lower` is a no-op + lx = lower(x) + if x === lx + show_string(io, x) + else + show_json(io, s, lx) + end +end + +function show_json(io::SC, s::CS, x::Union{Integer, AbstractFloat}) + if isfinite(x) + Base.print(io, x) + else + show_null(io) + end +end + +show_json(io::SC, ::CS, ::Nothing) = show_null(io) +show_json(io::SC, ::CS, ::Missing) = show_null(io) + +function show_json(io::SC, s::CS, a::AbstractDict) + begin_object(io) + for kv in a + show_pair(io, s, kv) + end + end_object(io) +end + +function show_json(io::SC, s::CS, kv::Pair) + begin_object(io) + show_pair(io, s, kv) + end_object(io) +end + +function show_json(io::SC, s::CS, x::CompositeTypeWrapper) + begin_object(io) + for fn in x.fns + show_pair(io, s, fn, getfield(x.wrapped, fn)) + end + end_object(io) +end + +function show_json(io::SC, s::CS, x::Union{AbstractVector, Tuple}) + begin_array(io) + for elt in x + show_element(io, s, elt) + end + end_array(io) +end + +""" +Serialize a multidimensional array to JSON in column-major format. That is, +`json([1 2 3; 4 5 6]) == "[[1,4],[2,5],[3,6]]"`. +""" +function show_json(io::SC, s::CS, A::AbstractArray{<:Any,n}) where n + begin_array(io) + newdims = ntuple(_ -> :, n - 1) + for j in axes(A, n) + show_element(io, s, view(A, newdims..., j)) + end + end_array(io) +end + +# special case for 0-dimensional arrays +show_json(io::SC, s::CS, A::AbstractArray{<:Any,0}) = show_json(io, s, A[]) + +show_json(io::SC, s::CS, a) = show_json(io, s, lower(a)) + +# Fallback show_json for non-SC types +""" +Serialize Julia object `obj` to IO `io` using the behaviour described by `s`. If +`indent` is provided, then the JSON will be pretty-printed; otherwise it will be +printed on one line. If pretty-printing is enabled, then a trailing newline will +be printed; otherwise there will be no trailing newline. +""" +function show_json(io::IO, s::Serialization, obj; indent=nothing) + ctx = indent === nothing ? CompactContext(io) : PrettyContext(io, indent) + show_json(ctx, s, obj) + if indent !== nothing + println(io) + end +end + +""" + JSONText(s::AbstractString) + +`JSONText` is a wrapper around a Julia string representing JSON-formatted +text, which is inserted *as-is* in the JSON output of `JSON.print` and `JSON.json` +for compact output, and is otherwise re-parsed for pretty-printed output. + +`s` *must* contain valid JSON text. Otherwise compact output will contain +the malformed `s` and other serialization output will throw a parsing exception. +""" +struct JSONText + s::String +end +show_json(io::CompactContext, s::CS, json::JSONText) = write(io, json.s) +# other contexts for JSONText are handled by lower(json) = parse(json.s) + +print(io::IO, obj, indent) = + show_json(io, StandardSerialization(), obj; indent=indent) +print(io::IO, obj) = show_json(io, StandardSerialization(), obj) + +print(a, indent) = print(stdout, a, indent) +print(a) = print(stdout, a) + +json(a) = sprint(print, a) +json(a, indent) = sprint(print, a, indent) + +end diff --git a/packages/JSON/src/bytes.jl b/packages/JSON/src/bytes.jl new file mode 100644 index 0000000..57b92a8 --- /dev/null +++ b/packages/JSON/src/bytes.jl @@ -0,0 +1,67 @@ +# The following bytes have significant meaning in JSON +const BACKSPACE = UInt8('\b') +const TAB = UInt8('\t') +const NEWLINE = UInt8('\n') +const FORM_FEED = UInt8('\f') +const RETURN = UInt8('\r') +const SPACE = UInt8(' ') +const STRING_DELIM = UInt8('"') +const PLUS_SIGN = UInt8('+') +const DELIMITER = UInt8(',') +const MINUS_SIGN = UInt8('-') +const DECIMAL_POINT = UInt8('.') +const SOLIDUS = UInt8('/') +const DIGIT_ZERO = UInt8('0') +const DIGIT_NINE = UInt8('9') +const SEPARATOR = UInt8(':') +const LATIN_UPPER_A = UInt8('A') +const LATIN_UPPER_E = UInt8('E') +const LATIN_UPPER_F = UInt8('F') +const ARRAY_BEGIN = UInt8('[') +const BACKSLASH = UInt8('\\') +const ARRAY_END = UInt8(']') +const LATIN_A = UInt8('a') +const LATIN_B = UInt8('b') +const LATIN_E = UInt8('e') +const LATIN_F = UInt8('f') +const LATIN_L = UInt8('l') +const LATIN_N = UInt8('n') +const LATIN_R = UInt8('r') +const LATIN_S = UInt8('s') +const LATIN_T = UInt8('t') +const LATIN_U = UInt8('u') +const OBJECT_BEGIN = UInt8('{') +const OBJECT_END = UInt8('}') + +const ESCAPES = Dict( + STRING_DELIM => STRING_DELIM, + BACKSLASH => BACKSLASH, + SOLIDUS => SOLIDUS, + LATIN_B => BACKSPACE, + LATIN_F => FORM_FEED, + LATIN_N => NEWLINE, + LATIN_R => RETURN, + LATIN_T => TAB) + +const REVERSE_ESCAPES = Dict(reverse(p) for p in ESCAPES) +const ESCAPED_ARRAY = Vector{Vector{UInt8}}(undef, 256) +for c in 0x00:0xFF + ESCAPED_ARRAY[c + 1] = if c == SOLIDUS + [SOLIDUS] # don't escape this one + elseif c ≥ 0x80 + [c] # UTF-8 character copied verbatim + elseif haskey(REVERSE_ESCAPES, c) + [BACKSLASH, REVERSE_ESCAPES[c]] + elseif iscntrl(Char(c)) || !isprint(Char(c)) + UInt8[BACKSLASH, LATIN_U, string(c, base=16, pad=4)...] + else + [c] + end +end + +export BACKSPACE, TAB, NEWLINE, FORM_FEED, RETURN, SPACE, STRING_DELIM, + PLUS_SIGN, DELIMITER, MINUS_SIGN, DECIMAL_POINT, SOLIDUS, DIGIT_ZERO, + DIGIT_NINE, SEPARATOR, LATIN_UPPER_A, LATIN_UPPER_E, LATIN_UPPER_F, + ARRAY_BEGIN, BACKSLASH, ARRAY_END, LATIN_A, LATIN_B, LATIN_E, LATIN_F, + LATIN_L, LATIN_N, LATIN_R, LATIN_S, LATIN_T, LATIN_U, OBJECT_BEGIN, + OBJECT_END, ESCAPES, REVERSE_ESCAPES, ESCAPED_ARRAY diff --git a/packages/JSON/src/errors.jl b/packages/JSON/src/errors.jl new file mode 100644 index 0000000..c9c1c87 --- /dev/null +++ b/packages/JSON/src/errors.jl @@ -0,0 +1,12 @@ +# The following errors may be thrown by the parser +const E_EXPECTED_EOF = "Expected end of input" +const E_UNEXPECTED_EOF = "Unexpected end of input" +const E_UNEXPECTED_CHAR = "Unexpected character" +const E_BAD_KEY = "Invalid object key" +const E_BAD_ESCAPE = "Invalid escape sequence" +const E_BAD_CONTROL = "ASCII control character in string" +const E_LEADING_ZERO = "Invalid leading zero in number" +const E_BAD_NUMBER = "Invalid number" + +export E_EXPECTED_EOF, E_UNEXPECTED_EOF, E_UNEXPECTED_CHAR, E_BAD_KEY, + E_BAD_ESCAPE, E_BAD_CONTROL, E_LEADING_ZERO, E_BAD_NUMBER diff --git a/packages/JSON/src/pushvector.jl b/packages/JSON/src/pushvector.jl new file mode 100644 index 0000000..01399f1 --- /dev/null +++ b/packages/JSON/src/pushvector.jl @@ -0,0 +1,33 @@ +# This is a vector wrapper that we use as a workaround for `push!` +# being slow (it always calls into the runtime even if the underlying buffer, +# has enough space). Here we keep track of the length using an extra field +mutable struct PushVector{T, A<:AbstractVector{T}} <: AbstractVector{T} + v::A + l::Int +end + +# Default length of 20 should be enough to never need to grow in most cases +PushVector{T}() where {T} = PushVector(Vector{T}(undef, 20), 0) + +Base.unsafe_convert(::Type{Ptr{UInt8}}, v::PushVector) = pointer(v.v) +Base.length(v::PushVector) = v.l +Base.size(v::PushVector) = (v.l,) +@inline function Base.getindex(v::PushVector, i) + @boundscheck checkbounds(v, i) + @inbounds v.v[i] +end + +function Base.push!(v::PushVector, i) + v.l += 1 + if v.l > length(v.v) + resize!(v.v, v.l * 2) + end + v.v[v.l] = i + return v +end + +function Base.resize!(v::PushVector, l::Integer) + # Only support shrinking for now, since that is all we need + @assert l <= v.l + v.l = l +end diff --git a/packages/JSON/src/specialized.jl b/packages/JSON/src/specialized.jl new file mode 100644 index 0000000..e204299 --- /dev/null +++ b/packages/JSON/src/specialized.jl @@ -0,0 +1,144 @@ +function maxsize_buffer(maxsize::Int) + IOBuffer(maxsize=maxsize) +end + +# Specialized functions for increased performance when JSON is in-memory +function parse_string(ps::MemoryParserState) + # "Dry Run": find length of string so we can allocate the right amount of + # memory from the start. Does not do full error checking. + fastpath, len = predict_string(ps) + + # Now read the string itself: + + # Fast path occurs when the string has no escaped characters. This is quite + # often the case in real-world data, especially when keys are short strings. + # We can just copy the data from the buffer in this case. + if fastpath + s = ps.s + ps.s = s + len + 2 # byte after closing quote + return unsafe_string(pointer(ps.utf8)+s, len) + else + String(take!(parse_string(ps, maxsize_buffer(len)))) + end +end + +""" +Scan through a string at the current parser state and return a tuple containing +information about the string. This function avoids memory allocation where +possible. + +The first element of the returned tuple is a boolean indicating whether the +string may be copied directly from the parser state. Special casing string +parsing when there are no escaped characters leads to substantially increased +performance in common situations. + +The second element of the returned tuple is an integer representing the exact +length of the string, in bytes when encoded as UTF-8. This information is useful +for pre-sizing a buffer to contain the parsed string. + +This function will throw an error if: + + - invalid control characters are found + - an invalid unicode escape is read + - the string is not terminated + +No error is thrown when other invalid backslash escapes are encountered. +""" +function predict_string(ps::MemoryParserState) + e = length(ps) + fastpath = true # true if no escapes in this string, so it can be copied + len = 0 # the number of UTF8 bytes the string contains + + s = ps.s + 1 # skip past opening string character " + @inbounds while s <= e + c = ps[s] + if c == BACKSLASH + fastpath = false + (s += 1) > e && break + if ps[s] == LATIN_U # Unicode escape + t = ps.s + ps.s = s + 1 + len += write(devnull, read_unicode_escape!(ps)) + s = ps.s + ps.s = t + continue + end + elseif c == STRING_DELIM + return fastpath, len + elseif c < SPACE + ps.s = s + _error(E_BAD_CONTROL, ps) + end + len += 1 + s += 1 + end + + ps.s = s + _error(E_UNEXPECTED_EOF, ps) +end + +""" +Parse the string starting at the parser state’s current location into the given +pre-sized IOBuffer. The only correctness checking is for escape sequences, so the +passed-in buffer must exactly represent the amount of space needed for parsing. +""" +function parse_string(ps::MemoryParserState, b::IOBuffer) + s = ps.s + e = length(ps) + + s += 1 # skip past opening string character " + len = b.maxsize + @inbounds while b.size < len + c = ps[s] + if c == BACKSLASH + s += 1 + s > e && break + c = ps[s] + if c == LATIN_U # Unicode escape + ps.s = s + 1 + write(b, read_unicode_escape!(ps)) + s = ps.s + continue + else + c = get(ESCAPES, c, 0x00) + if c == 0x00 + ps.s = s + _error(E_BAD_ESCAPE, ps) + end + end + end + + # UTF8-encoded non-ascii characters will be copied verbatim, which is + # the desired behaviour + write(b, c) + s += 1 + end + + # don't worry about non-termination or other edge cases; those should have + # been caught in the dry run. + ps.s = s + 1 + b +end + +function parse_number(pc::ParserContext, ps::MemoryParserState) + s = p = ps.s + e = length(ps) + isint = true + + # Determine the end of the floating point by skipping past ASCII values + # 0-9, +, -, e, E, and . + while p ≤ e + @inbounds c = ps[p] + if isjsondigit(c) || MINUS_SIGN == c # no-op + elseif PLUS_SIGN == c || LATIN_E == c || LATIN_UPPER_E == c || + DECIMAL_POINT == c + isint = false + else + break + end + p += 1 + end + ps.s = p + + number_from_bytes(pc, ps, isint, ps, s, p - 1) +end diff --git a/packages/JSON/test/async.jl b/packages/JSON/test/async.jl new file mode 100644 index 0000000..1612a6e --- /dev/null +++ b/packages/JSON/test/async.jl @@ -0,0 +1,109 @@ +finished_async_tests = RemoteChannel() + +using Sockets + +@async begin + s = listen(7777) + s = accept(s) + + Base.start_reading(s) + + @test JSON.parse(s) != nothing # a + @test JSON.parse(s) != nothing # b + validate_c(s) # c + @test JSON.parse(s) != nothing # d + validate_svg_tviewer_menu(s) # svg_tviewer_menu + @test JSON.parse(s) != nothing # gmaps + @test JSON.parse(s) != nothing # colors1 + @test JSON.parse(s) != nothing # colors2 + @test JSON.parse(s) != nothing # colors3 + @test JSON.parse(s) != nothing # twitter + @test JSON.parse(s) != nothing # facebook + validate_flickr(s) # flickr + @test JSON.parse(s) != nothing # youtube + @test JSON.parse(s) != nothing # iphone + @test JSON.parse(s) != nothing # customer + @test JSON.parse(s) != nothing # product + @test JSON.parse(s) != nothing # interop + validate_unicode(s) # unicode + @test JSON.parse(s) != nothing # issue5 + @test JSON.parse(s) != nothing # dollars + @test JSON.parse(s) != nothing # brackets + + put!(finished_async_tests, nothing) +end + +w = connect("localhost", 7777) + +@test JSON.parse(a) != nothing +write(w, a) + +@test JSON.parse(b) != nothing +write(w, b) + +validate_c(c) +write(w, c) + +@test JSON.parse(d) != nothing +write(w, d) + +validate_svg_tviewer_menu(svg_tviewer_menu) +write(w, svg_tviewer_menu) + +@test JSON.parse(gmaps) != nothing +write(w, gmaps) + +@test JSON.parse(colors1) != nothing +write(w, colors1) + +@test JSON.parse(colors2) != nothing +write(w, colors2) + +@test JSON.parse(colors3) != nothing +write(w, colors3) + +@test JSON.parse(twitter) != nothing +write(w, twitter) + +@test JSON.parse(facebook) != nothing +write(w, facebook) + +validate_flickr(flickr) +write(w, flickr) + +@test JSON.parse(youtube) != nothing +write(w, youtube) + +@test JSON.parse(iphone) != nothing +write(w, iphone) + +@test JSON.parse(customer) != nothing +write(w, customer) + +@test JSON.parse(product) != nothing +write(w, product) + +@test JSON.parse(interop) != nothing +write(w, interop) + +validate_unicode(unicode) +write(w, unicode) + +# issue #5 +issue5 = "[\"A\",\"B\",\"C\\n\"]" +JSON.parse(issue5) +write(w, issue5) + +# $ escaping issue +dollars = ["all of the \$s", "µniçø∂\$"] +json_dollars = json(dollars) +@test JSON.parse(json_dollars) != nothing +write(w, json_dollars) + +# unmatched brackets +brackets = Dict("foo"=>"ba}r", "be}e]p"=>"boo{p") +json_brackets = json(brackets) +@test JSON.parse(json_brackets) != nothing +write(w, json_dollars) + +fetch(finished_async_tests) diff --git a/packages/JSON/test/enum.jl b/packages/JSON/test/enum.jl new file mode 100644 index 0000000..ead3d99 --- /dev/null +++ b/packages/JSON/test/enum.jl @@ -0,0 +1,4 @@ +@enum Animal zebra aardvark horse +@test json(zebra) == "\"zebra\"" +@test json([aardvark, horse, Dict("z" => zebra)]) == + "[\"aardvark\",\"horse\",{\"z\":\"zebra\"}]" diff --git a/packages/JSON/test/indentation.jl b/packages/JSON/test/indentation.jl new file mode 100644 index 0000000..98fa5f0 --- /dev/null +++ b/packages/JSON/test/indentation.jl @@ -0,0 +1,10 @@ +# check indented json has same final value as non indented +fb = JSON.parse(facebook) +fbjson1 = json(fb, 2) +fbjson2 = json(fb) +@test JSON.parse(fbjson1) == JSON.parse(fbjson2) + +ev = JSON.parse(svg_tviewer_menu) +ejson1 = json(ev, 2) +ejson2 = json(ev) +@test JSON.parse(ejson1) == JSON.parse(ejson2) diff --git a/packages/JSON/test/json-checker.jl b/packages/JSON/test/json-checker.jl new file mode 100644 index 0000000..7d0594b --- /dev/null +++ b/packages/JSON/test/json-checker.jl @@ -0,0 +1,28 @@ +# Run modified JSON checker tests + +const JSON_DATA_DIR = joinpath(dirname(@__DIR__), "data") + +for i in 1:38 + file = "fail$(lpad(string(i), 2, "0")).json" + filepath = joinpath(JSON_DATA_DIR, "jsonchecker", file) + + @test_throws ErrorException JSON.parsefile(filepath) +end + +for i in 1:3 + # Test that the files parse successfully and match streaming parser + tf = joinpath(JSON_DATA_DIR, "jsonchecker", "pass$(lpad(string(i), 2, "0")).json") + @test JSON.parsefile(tf) == open(JSON.parse, tf) +end + +# Run JSON roundtrip tests (check consistency of .json) + +roundtrip(data) = JSON.json(JSON.Parser.parse(data)) + +for i in 1:27 + file = "roundtrip$(lpad(string(i), 2, "0")).json" + filepath = joinpath(JSON_DATA_DIR, "roundtrip", file) + + rt = roundtrip(read(filepath, String)) + @test rt == roundtrip(rt) +end diff --git a/packages/JSON/test/json-samples.jl b/packages/JSON/test/json-samples.jl new file mode 100644 index 0000000..2df326f --- /dev/null +++ b/packages/JSON/test/json-samples.jl @@ -0,0 +1,644 @@ +#Examples from http://json.org/example.html +a="{\"menu\": { + \"id\": \"file\", + \"value\": \"File\", + \"popup\": { + \"menuitem\": [ + {\"value\": \"New\", \"onclick\": \"CreateNewDoc()\"}, + {\"value\": \"Open\", \"onclick\": \"OpenDoc()\"}, + {\"value\": \"Close\", \"onclick\": \"CloseDoc()\"} + ] + } + }} + " + + +b="{ + \"glossary\": { + \"title\": \"example glossary\", + \"GlossDiv\": { + \"title\": \"S\", + \"GlossList\": { + \"GlossEntry\": { + \"ID\": \"SGML\", + \"SortAs\": \"SGML\", + \"GlossTerm\": \"Standard Generalized Markup Language\", + \"Acronym\": \"SGML\", + \"Abbrev\": \"ISO 8879:1986\", + \"GlossDef\": { + \"para\": \"A meta-markup language, used to create markup languages such as DocBook.\", + \"GlossSeeAlso\": [\"GML\", \"XML\"] + }, + \"GlossSee\": \"markup\" + } + } + } + } +} +" + +const c = """ +{"widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "name": "sun1", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36.5, + "style": "bold", + "name": "text1", + "hOffset": 250, + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } +}}""" +function validate_c(c) + j = JSON.parse(c) + @test j != nothing + @test typeof(j["widget"]["image"]["hOffset"]) == Int64 + @test j["widget"]["image"]["hOffset"] == 250 + @test typeof(j["widget"]["text"]["size"]) == Float64 + @test j["widget"]["text"]["size"] == 36.5 +end + +d = "{\"web-app\": { + \"servlet\": [ + { + \"servlet-name\": \"cofaxCDS\", + \"servlet-class\": \"org.cofax.cds.CDSServlet\", + \"init-param\": { + \"configGlossary:installationAt\": \"Philadelphia, PA\", + \"configGlossary:adminEmail\": \"ksm@pobox.com\", + \"configGlossary:poweredBy\": \"Cofax\", + \"configGlossary:poweredByIcon\": \"/images/cofax.gif\", + \"configGlossary:staticPath\": \"/content/static\", + \"templateProcessorClass\": \"org.cofax.WysiwygTemplate\", + \"templateLoaderClass\": \"org.cofax.FilesTemplateLoader\", + \"templatePath\": \"templates\", + \"templateOverridePath\": \"\", + \"defaultListTemplate\": \"listTemplate.htm\", + \"defaultFileTemplate\": \"articleTemplate.htm\", + \"useJSP\": false, + \"jspListTemplate\": \"listTemplate.jsp\", + \"jspFileTemplate\": \"articleTemplate.jsp\", + \"cachePackageTagsTrack\": 200, + \"cachePackageTagsStore\": 200, + \"cachePackageTagsRefresh\": 60, + \"cacheTemplatesTrack\": 100, + \"cacheTemplatesStore\": 50, + \"cacheTemplatesRefresh\": 15, + \"cachePagesTrack\": 200, + \"cachePagesStore\": 100, + \"cachePagesRefresh\": 10, + \"cachePagesDirtyRead\": 10, + \"searchEngineListTemplate\": \"forSearchEnginesList.htm\", + \"searchEngineFileTemplate\": \"forSearchEngines.htm\", + \"searchEngineRobotsDb\": \"WEB-INF/robots.db\", + \"useDataStore\": true, + \"dataStoreClass\": \"org.cofax.SqlDataStore\", + \"redirectionClass\": \"org.cofax.SqlRedirection\", + \"dataStoreName\": \"cofax\", + \"dataStoreDriver\": \"com.microsoft.jdbc.sqlserver.SQLServerDriver\", + \"dataStoreUrl\": \"jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon\", + \"dataStoreUser\": \"sa\", + \"dataStorePassword\": \"dataStoreTestQuery\", + \"dataStoreTestQuery\": \"SET NOCOUNT ON;select test='test';\", + \"dataStoreLogFile\": \"/usr/local/tomcat/logs/datastore.log\", + \"dataStoreInitConns\": 10, + \"dataStoreMaxConns\": 100, + \"dataStoreConnUsageLimit\": 100, + \"dataStoreLogLevel\": \"debug\", + \"maxUrlLength\": 500}}, + { + \"servlet-name\": \"cofaxEmail\", + \"servlet-class\": \"org.cofax.cds.EmailServlet\", + \"init-param\": { + \"mailHost\": \"mail1\", + \"mailHostOverride\": \"mail2\"}}, + { + \"servlet-name\": \"cofaxAdmin\", + \"servlet-class\": \"org.cofax.cds.AdminServlet\"}, + + { + \"servlet-name\": \"fileServlet\", + \"servlet-class\": \"org.cofax.cds.FileServlet\"}, + { + \"servlet-name\": \"cofaxTools\", + \"servlet-class\": \"org.cofax.cms.CofaxToolsServlet\", + \"init-param\": { + \"templatePath\": \"toolstemplates/\", + \"log\": 1, + \"logLocation\": \"/usr/local/tomcat/logs/CofaxTools.log\", + \"logMaxSize\": \"\", + \"dataLog\": 1, + \"dataLogLocation\": \"/usr/local/tomcat/logs/dataLog.log\", + \"dataLogMaxSize\": \"\", + \"removePageCache\": \"/content/admin/remove?cache=pages&id=\", + \"removeTemplateCache\": \"/content/admin/remove?cache=templates&id=\", + \"fileTransferFolder\": \"/usr/local/tomcat/webapps/content/fileTransferFolder\", + \"lookInContext\": 1, + \"adminGroupID\": 4, + \"betaServer\": true}}], + \"servlet-mapping\": { + \"cofaxCDS\": \"/\", + \"cofaxEmail\": \"/cofaxutil/aemail/*\", + \"cofaxAdmin\": \"/admin/*\", + \"fileServlet\": \"/static/*\", + \"cofaxTools\": \"/tools/*\"}, + + \"taglib\": { + \"taglib-uri\": \"cofax.tld\", + \"taglib-location\": \"/WEB-INF/tlds/cofax.tld\"}}}" + +const svg_tviewer_menu = """ +{"menu": { + "header": "SVG\\tViewer\\u03b1", + "items": [ + {"id": "Open"}, + {"id": "OpenNew", "label": "Open New"}, + null, + {"id": "ZoomIn", "label": "Zoom In"}, + {"id": "ZoomOut", "label": "Zoom Out"}, + {"id": "OriginalView", "label": "Original View"}, + null, + {"id": "Quality"}, + {"id": "Pause"}, + {"id": "Mute"}, + null, + {"id": "Find", "label": "Find..."}, + {"id": "FindAgain", "label": "Find Again"}, + {"id": "Copy"}, + {"id": "CopyAgain", "label": "Copy Again"}, + {"id": "CopySVG", "label": "Copy SVG"}, + {"id": "ViewSVG", "label": "View SVG"}, + {"id": "ViewSource", "label": "View Source"}, + {"id": "SaveAs", "label": "Save As"}, + null, + {"id": "Help"}, + {"id": "About", "label": "About Adobe SVG Viewer..."} + ] +}}""" +function validate_svg_tviewer_menu(str) + j = JSON.parse(str) + @test j != nothing + @test typeof(j) == Dict{String, Any} + @test length(j) == 1 + @test typeof(j["menu"]) == Dict{String, Any} + @test length(j["menu"]) == 2 + @test j["menu"]["header"] == "SVG\tViewerα" + @test isa(j["menu"]["items"], Vector{Any}) + @test length(j["menu"]["items"]) == 22 + @test j["menu"]["items"][3] == nothing + @test j["menu"]["items"][2]["id"] == "OpenNew" + @test j["menu"]["items"][2]["label"] == "Open New" +end + + +#Example JSON strings from http://www.jquery4u.com/json/10-example-json-files/ + +gmaps= "{\"markers\": [ + { + \"point\":\"new GLatLng(40.266044,-74.718479)\", + \"homeTeam\":\"Lawrence Library\", + \"awayTeam\":\"LUGip\", + \"markerImage\":\"images/red.png\", + \"information\": \"Linux users group meets second Wednesday of each month.\", + \"fixture\":\"Wednesday 7pm\", + \"capacity\":\"\", + \"previousScore\":\"\" + }, + { + \"point\":\"new GLatLng(40.211600,-74.695702)\", + \"homeTeam\":\"Hamilton Library\", + \"awayTeam\":\"LUGip HW SIG\", + \"markerImage\":\"images/white.png\", + \"information\": \"Linux users can meet the first Tuesday of the month to work out harward and configuration issues.\", + \"fixture\":\"Tuesday 7pm\", + \"capacity\":\"\", + \"tv\":\"\" + }, + { + \"point\":\"new GLatLng(40.294535,-74.682012)\", + \"homeTeam\":\"Applebees\", + \"awayTeam\":\"After LUPip Mtg Spot\", + \"markerImage\":\"images/newcastle.png\", + \"information\": \"Some of us go there after the main LUGip meeting, drink brews, and talk.\", + \"fixture\":\"Wednesday whenever\", + \"capacity\":\"2 to 4 pints\", + \"tv\":\"\" + } +] }" + +colors1 = "{ + \"colorsArray\":[{ + \"colorName\":\"red\", + \"hexValue\":\"#f00\" + }, + { + \"colorName\":\"green\", + \"hexValue\":\"#0f0\" + }, + { + \"colorName\":\"blue\", + \"hexValue\":\"#00f\" + }, + { + \"colorName\":\"cyan\", + \"hexValue\":\"#0ff\" + }, + { + \"colorName\":\"magenta\", + \"hexValue\":\"#f0f\" + }, + { + \"colorName\":\"yellow\", + \"hexValue\":\"#ff0\" + }, + { + \"colorName\":\"black\", + \"hexValue\":\"#000\" + } + ] +}" + +colors2 = "{ + \"colorsArray\":[{ + \"red\":\"#f00\", + \"green\":\"#0f0\", + \"blue\":\"#00f\", + \"cyan\":\"#0ff\", + \"magenta\":\"#f0f\", + \"yellow\":\"#ff0\", + \"black\":\"#000\" + } + ] +}" + +colors3 = "{ + \"red\":\"#f00\", + \"green\":\"#0f0\", + \"blue\":\"#00f\", + \"cyan\":\"#0ff\", + \"magenta\":\"#f0f\", + \"yellow\":\"#ff0\", + \"black\":\"#000\" +}" + +twitter = "{\"results\":[ + + {\"text\":\"@twitterapi http://tinyurl.com/ctrefg\", + \"to_user_id\":396524, + \"to_user\":\"TwitterAPI\", + \"from_user\":\"jkoum\", + \"metadata\": + { + \"result_type\":\"popular\", + \"recent_retweets\": 109 + }, + \"id\":1478555574, + \"from_user_id\":1833773, + \"iso_language_code\":\"nl\", + \"source\":\"twitter\", + \"profile_image_url\":\"http://s3.amazonaws.com/twitter_production/profile_images/118412707/2522215727_a5f07da155_b_normal.jpg\", + \"created_at\":\"Wed, 08 Apr 2009 19:22:10 +0000\"}], + \"since_id\":0, + \"max_id\":1480307926, + \"refresh_url\":\"?since_id=1480307926&q=%40twitterapi\", + \"results_per_page\":15, + \"next_page\":\"?page=2&max_id=1480307926&q=%40twitterapi\", + \"completed_in\":0.031704, + \"page\":1, + \"query\":\"%40twitterapi\"}" + +facebook= "{ + \"data\": [ + { + \"id\": \"X999_Y999\", + \"from\": { + \"name\": \"Tom Brady\", \"id\": \"X12\" + }, + \"message\": \"Looking forward to 2010!\", + \"actions\": [ + { + \"name\": \"Comment\", + \"link\": \"http://www.facebook.com/X999/posts/Y999\" + }, + { + \"name\": \"Like\", + \"link\": \"http://www.facebook.com/X999/posts/Y999\" + } + ], + \"type\": \"status\", + \"created_time\": \"2010-08-02T21:27:44+0000\", + \"updated_time\": \"2010-08-02T21:27:44+0000\" + }, + { + \"id\": \"X998_Y998\", + \"from\": { + \"name\": \"Peyton Manning\", \"id\": \"X18\" + }, + \"message\": \"Where's my contract?\", + \"actions\": [ + { + \"name\": \"Comment\", + \"link\": \"http://www.facebook.com/X998/posts/Y998\" + }, + { + \"name\": \"Like\", + \"link\": \"http://www.facebook.com/X998/posts/Y998\" + } + ], + \"type\": \"status\", + \"created_time\": \"2010-08-02T21:27:44+0000\", + \"updated_time\": \"2010-08-02T21:27:44+0000\" + } + ] +}" + +const flickr = """{ + "title": "Talk On Travel Pool", + "link": "http://www.flickr.com/groups/talkontravel/pool/", + "description": "Travel and vacation photos from around the world.", + "modified": "2009-02-02T11:10:27Z", + "generator": "http://www.flickr.com/", + "totalItems":222, + "items": [ + { + "title": "View from the hotel", + "link": "http://www.flickr.com/photos/33112458@N08/3081564649/in/pool-998875@N22", + "media": {"m":"http://farm4.static.flickr.com/3037/3081564649_4a6569750c_m.jpg"}, + "date_taken": "2008-12-04T04:43:03-08:00", + "description": "

Talk On Travel has added a photo to the pool:

\\"View

", + "published": "2008-12-04T12:43:03Z", + "author": "nobody@flickr.com (Talk On Travel)", + "author_id": "33112458@N08", + "tags": "spain dolphins tenerife canaries lagomera aqualand playadelasamericas junglepark losgigantos loscristines talkontravel" + } + ] +}""" +function validate_flickr(str) + k = JSON.parse(str) + @test k != nothing + @test k["totalItems"] == 222 + @test k["items"][1]["description"][12] == '\"' +end + +youtube = "{\"apiVersion\":\"2.0\", + \"data\":{ + \"updated\":\"2010-01-07T19:58:42.949Z\", + \"totalItems\":800, + \"startIndex\":1, + \"itemsPerPage\":1, + \"items\":[ + {\"id\":\"hYB0mn5zh2c\", + \"uploaded\":\"2007-06-05T22:07:03.000Z\", + \"updated\":\"2010-01-07T13:26:50.000Z\", + \"uploader\":\"GoogleDeveloperDay\", + \"category\":\"News\", + \"title\":\"Google Developers Day US - Maps API Introduction\", + \"description\":\"Google Maps API Introduction ...\", + \"tags\":[ + \"GDD07\",\"GDD07US\",\"Maps\" + ], + \"thumbnail\":{ + \"default\":\"http://i.ytimg.com/vi/hYB0mn5zh2c/default.jpg\", + \"hqDefault\":\"http://i.ytimg.com/vi/hYB0mn5zh2c/hqdefault.jpg\" + }, + \"player\":{ + \"default\":\"http://www.youtube.com/watch?v\u003dhYB0mn5zh2c\" + }, + \"content\":{ + \"1\":\"rtsp://v5.cache3.c.youtube.com/CiILENy.../0/0/0/video.3gp\", + \"5\":\"http://www.youtube.com/v/hYB0mn5zh2c?f...\", + \"6\":\"rtsp://v1.cache1.c.youtube.com/CiILENy.../0/0/0/video.3gp\" + }, + \"duration\":2840, + \"aspectRatio\":\"widescreen\", + \"rating\":4.63, + \"ratingCount\":68, + \"viewCount\":220101, + \"favoriteCount\":201, + \"commentCount\":22, + \"status\":{ + \"value\":\"restricted\", + \"reason\":\"limitedSyndication\" + }, + \"accessControl\":{ + \"syndicate\":\"allowed\", + \"commentVote\":\"allowed\", + \"rate\":\"allowed\", + \"list\":\"allowed\", + \"comment\":\"allowed\", + \"embed\":\"allowed\", + \"videoRespond\":\"moderated\" + } + } + ] + } +}" + +iphone = "{ + \"menu\": { + \"header\": \"xProgress SVG Viewer\", + \"items\": [ + { + \"id\": \"Open\" + }, + { + \"id\": \"OpenNew\", + \"label\": \"Open New\" + }, + null, + { + \"id\": \"ZoomIn\", + \"label\": \"Zoom In\" + }, + { + \"id\": \"ZoomOut\", + \"label\": \"Zoom Out\" + }, + { + \"id\": \"OriginalView\", + \"label\": \"Original View\" + }, + null, + { + \"id\": \"Quality\" + }, + { + \"id\": \"Pause\" + }, + { + \"id\": \"Mute\" + }, + null, + { + \"id\": \"Find\", + \"label\": \"Find...\" + }, + { + \"id\": \"FindAgain\", + \"label\": \"Find Again\" + }, + { + \"id\": \"Copy\" + }, + { + \"id\": \"CopyAgain\", + \"label\": \"Copy Again\" + }, + { + \"id\": \"CopySVG\", + \"label\": \"Copy SVG\" + }, + { + \"id\": \"ViewSVG\", + \"label\": \"View SVG\" + }, + { + \"id\": \"ViewSource\", + \"label\": \"View Source\" + }, + { + \"id\": \"SaveAs\", + \"label\": \"Save As\" + }, + null, + { + \"id\": \"Help\" + }, + { + \"id\": \"About\", + \"label\": \"About xProgress CVG Viewer...\" + } + ] + } +}" + +customer = "{ + \"firstName\": \"John\", + \"lastName\": \"Smith\", + \"age\": 25, + \"address\": + { + \"streetAddress\": \"21 2nd Street\", + \"city\": \"New York\", + \"state\": \"NY\", + \"postalCode\": \"10021\" + }, + \"phoneNumber\": + [ + { + \"type\": \"home\", + \"number\": \"212 555-1234\" + }, + { + \"type\": \"fax\", + \"number\": \"646 555-4567\" + } + ] + }" + + product = "{ + \"name\":\"Product\", + \"properties\": + { + \"id\": + { + \"type\":\"number\", + \"description\":\"Product identifier\", + \"required\":true + }, + \"name\": + { + \"description\":\"Name of the product\", + \"type\":\"string\", + \"required\":true + }, + \"price\": + { + \"type\":\"number\", + \"minimum\":0, + \"required\":true + }, + \"tags\": + { + \"type\":\"array\", + \"items\": + { + \"type\":\"string\" + } + } + } +}" + +interop = "{ + \"ResultSet\": { + \"totalResultsAvailable\": \"1827221\", + \"totalResultsReturned\": 2, + \"firstResultPosition\": 1, + \"Result\": [ + { + \"Title\": \"potato jpg\", + \"Summary\": \"Kentang Si bungsu dari keluarga Solanum tuberosum L ini ternyata memiliki khasiat untuk mengurangi kerutan jerawat bintik hitam dan kemerahan pada kulit Gunakan seminggu sekali sebagai\", + \"Url\": \"http://www.mediaindonesia.com/spaw/uploads/images/potato.jpg\", + \"ClickUrl\": \"http://www.mediaindonesia.com/spaw/uploads/images/potato.jpg\", + \"RefererUrl\": \"http://www.mediaindonesia.com/mediaperempuan/index.php?ar_id=Nzkw\", + \"FileSize\": 22630, + \"FileFormat\": \"jpeg\", + \"Height\": \"362\", + \"Width\": \"532\", + \"Thumbnail\": { + \"Url\": \"http://thm-a01.yimg.com/nimage/557094559c18f16a\", + \"Height\": \"98\", + \"Width\": \"145\" + } + }, + { + \"Title\": \"potato jpg\", + \"Summary\": \"Introduction of puneri aloo This is a traditional potato preparation flavoured with curry leaves and peanuts and can be eaten on fasting day Preparation time 10 min\", + \"Url\": \"http://www.infovisual.info/01/photo/potato.jpg\", + \"ClickUrl\": \"http://www.infovisual.info/01/photo/potato.jpg\", + \"RefererUrl\": \"http://sundayfood.com/puneri-aloo-indian-%20recipe\", + \"FileSize\": 119398, + \"FileFormat\": \"jpeg\", + \"Height\": \"685\", + \"Width\": \"1024\", + \"Thumbnail\": { + \"Url\": \"http://thm-a01.yimg.com/nimage/7fa23212efe84b64\", + \"Height\": \"107\", + \"Width\": \"160\" + } + } + ] + } +}" + +const unicode = """ +{"অলিম্পিকস": { + "অ্যাথলেট": "২২টি দেশ থেকে ২,০৩৫ জন প্রতিযোগী", + "ইভেন্ট": "২২টি ইভেন্টের মধ্যে ছিল দড়ি টানাটানি", + "রেকর্ড": [ + {"১০০মি. স্প্রিন্ট": "রেজি ওয়াকার, দক্ষিণ আফ্রিকা"}, + {"Marathon": "জনি হেইস"}, + {" ফ্রি-স্টাইল সাঁতার": "Henry Taylor, Britain"} + ] +}} +""" +function validate_unicode(str) + u = JSON.parse(str) + @test u != nothing + @test u["অলিম্পিকস"]["রেকর্ড"][2]["Marathon"] == "জনি হেইস" +end diff --git a/packages/JSON/test/lowering.jl b/packages/JSON/test/lowering.jl new file mode 100644 index 0000000..388cff1 --- /dev/null +++ b/packages/JSON/test/lowering.jl @@ -0,0 +1,37 @@ +module TestLowering + +using JSON +using Test +using Dates +using FixedPointNumbers: Fixed + +@test JSON.json(Date(2016, 8, 3)) == "\"2016-08-03\"" + +@test JSON.json(:x) == "\"x\"" +@test_throws ArgumentError JSON.json(Base) + +struct Type151{T} + x::T +end + +@test JSON.parse(JSON.json(Type151)) == string(Type151) + +JSON.lower(v::Type151{T}) where {T} = Dict(:type => T, :value => v.x) +@test JSON.parse(JSON.json(Type151(1.0))) == Dict( + "type" => "Float64", + "value" => 1.0) + +fixednum = Fixed{Int16, 15}(0.1234) +@test JSON.parse(JSON.json(fixednum)) == convert(Float64, fixednum) + +# test that the default string-serialization of enums can be overriden by +# `lower` if needed +@enum Fruit apple orange banana +JSON.lower(x::Fruit) = string("Fruit: ", x) +@test JSON.json(apple) == "\"Fruit: apple\"" + +@enum Vegetable carrot tomato potato +JSON.lower(x::Vegetable) = Dict(string(x) => Int(x)) +@test JSON.json(potato) == "{\"potato\":2}" + +end diff --git a/packages/JSON/test/parser/dicttype.jl b/packages/JSON/test/parser/dicttype.jl new file mode 100644 index 0000000..6e4d328 --- /dev/null +++ b/packages/JSON/test/parser/dicttype.jl @@ -0,0 +1,22 @@ +MissingDict() = DataStructures.DefaultDict{String,Any}(Missing) + +@testset for T in [ + DataStructures.OrderedDict, + Dict{Symbol, Int32}, + MissingDict +] + val = JSON.parse("{\"x\": 3}", dicttype=T) + @test length(val) == 1 + key = collect(keys(val))[1] + @test string(key) == "x" + @test val[key] == 3 + + if T == MissingDict + @test val isa DataStructures.DefaultDict{String} + @test val["y"] === missing + else + @test val isa T + @test_throws KeyError val["y"] + end +end + diff --git a/packages/JSON/test/parser/inttype.jl b/packages/JSON/test/parser/inttype.jl new file mode 100644 index 0000000..30e9ca1 --- /dev/null +++ b/packages/JSON/test/parser/inttype.jl @@ -0,0 +1,16 @@ +@testset for T in [Int32, Int64, Int128, BigInt] + val = JSON.parse("{\"x\": 3}", inttype=T) + @test isa(val, Dict{String, Any}) + @test length(val) == 1 + key = collect(keys(val))[1] + @test string(key) == "x" + value = val[key] + @test value == 3 + @test typeof(value) == T +end + +@testset begin + teststr = """{"201736327611975630": 18005722827070440994}""" + val = JSON.parse(teststr, inttype=Int128) + @test val == Dict{String,Any}("201736327611975630"=> 18005722827070440994) +end diff --git a/packages/JSON/test/parser/invalid-input.jl b/packages/JSON/test/parser/invalid-input.jl new file mode 100644 index 0000000..924f225 --- /dev/null +++ b/packages/JSON/test/parser/invalid-input.jl @@ -0,0 +1,33 @@ +const FAILURES = [ + # Unexpected character in array + "[1,2,3/4,5,6,7]", + # Unexpected character in object + "{\"1\":2, \"2\":3 _ \"4\":5}", + # Invalid escaped character + "[\"alpha\\α\"]", + "[\"\\u05AG\"]", + # Invalid 'simple' and 'unknown value' + "[tXXe]", + "[fail]", + "∞", + # Invalid number + "[5,2,-]", + "[5,2,+β]", + # Incomplete escape + "\"\\", + # Control character + "\"\0\"", + # Issue #99 + "[\"🍕\"_\"🍕\"", + # Issue #260 + "1997-03-03", + "1997.1-", +] + +@testset for fail in FAILURES + # Test memory parser + @test_throws ErrorException JSON.parse(fail) + + # Test streaming parser + @test_throws ErrorException JSON.parse(IOBuffer(fail)) +end diff --git a/packages/JSON/test/parser/parsefile.jl b/packages/JSON/test/parser/parsefile.jl new file mode 100644 index 0000000..f5b9f6c --- /dev/null +++ b/packages/JSON/test/parser/parsefile.jl @@ -0,0 +1,10 @@ +tmppath, io = mktemp() +write(io, facebook) +close(io) +if Sys.iswindows() + # don't use mmap on Windows, to avoid ERROR: unlink: operation not permitted (EPERM) + @test haskey(JSON.parsefile(tmppath; use_mmap=false), "data") +else + @test haskey(JSON.parsefile(tmppath), "data") +end +rm(tmppath) diff --git a/packages/JSON/test/regression/issue021.jl b/packages/JSON/test/regression/issue021.jl new file mode 100644 index 0000000..856f820 --- /dev/null +++ b/packages/JSON/test/regression/issue021.jl @@ -0,0 +1,4 @@ +test21 = "[\r\n{\r\n\"a\": 1,\r\n\"b\": 2\r\n},\r\n{\r\n\"a\": 3,\r\n\"b\": 4\r\n}\r\n]" +a = JSON.parse(test21) +@test isa(a, Vector{Any}) +@test length(a) == 2 diff --git a/packages/JSON/test/regression/issue026.jl b/packages/JSON/test/regression/issue026.jl new file mode 100644 index 0000000..ff9ea6d --- /dev/null +++ b/packages/JSON/test/regression/issue026.jl @@ -0,0 +1,2 @@ +obj = JSON.parse("{\"a\":2e10}") +@test obj["a"] == 2e10 diff --git a/packages/JSON/test/regression/issue057.jl b/packages/JSON/test/regression/issue057.jl new file mode 100644 index 0000000..1797a8a --- /dev/null +++ b/packages/JSON/test/regression/issue057.jl @@ -0,0 +1,2 @@ +obj = JSON.parse("{\"\U0001d712\":\"\\ud835\\udf12\"}") +@test(obj["𝜒"] == "𝜒") diff --git a/packages/JSON/test/regression/issue109.jl b/packages/JSON/test/regression/issue109.jl new file mode 100644 index 0000000..6dc2d9d --- /dev/null +++ b/packages/JSON/test/regression/issue109.jl @@ -0,0 +1,8 @@ +mutable struct t109 + i::Int +end + +let iob = IOBuffer() + JSON.print(iob, t109(1)) + @test get(JSON.parse(String(take!(iob))), "i", 0) == 1 +end diff --git a/packages/JSON/test/regression/issue152.jl b/packages/JSON/test/regression/issue152.jl new file mode 100644 index 0000000..5b4a01b --- /dev/null +++ b/packages/JSON/test/regression/issue152.jl @@ -0,0 +1,2 @@ +@test json([Int64[] Int64[]]) == "[[],[]]" +@test json([Int64[] Int64[]]') == "[]" diff --git a/packages/JSON/test/regression/issue163.jl b/packages/JSON/test/regression/issue163.jl new file mode 100644 index 0000000..5ace4fa --- /dev/null +++ b/packages/JSON/test/regression/issue163.jl @@ -0,0 +1 @@ +@test Float32(JSON.parse(json(2.1f-8))) == 2.1f-8 diff --git a/packages/JSON/test/runtests.jl b/packages/JSON/test/runtests.jl new file mode 100644 index 0000000..e732e5d --- /dev/null +++ b/packages/JSON/test/runtests.jl @@ -0,0 +1,80 @@ +using JSON +using Test +using Dates +using Distributed: RemoteChannel +using OffsetArrays + +import DataStructures + +include("json-samples.jl") + +@testset "Parser" begin + @testset "Parser Failures" begin + include("parser/invalid-input.jl") + end + + @testset "parsefile" begin + include("parser/parsefile.jl") + end + + @testset "dicttype" begin + include("parser/dicttype.jl") + end + + @testset "inttype" begin + include("parser/inttype.jl") + end + + @testset "Miscellaneous" begin + # test for single values + @test JSON.parse("true") == true + @test JSON.parse("null") == nothing + @test JSON.parse("\"hello\"") == "hello" + @test JSON.parse("\"a\"") == "a" + @test JSON.parse("1") == 1 + @test JSON.parse("1.5") == 1.5 + @test JSON.parse("[true]") == [true] + end +end + +@testset "Serializer" begin + @testset "Standard Serializer" begin + include("standard-serializer.jl") + end + + @testset "Lowering" begin + include("lowering.jl") + end + + @testset "Custom Serializer" begin + include("serializer.jl") + end +end + +@testset "Integration" begin + # ::Nothing values should be encoded as null + testDict = Dict("a" => nothing) + nothingJson = JSON.json(testDict) + nothingDict = JSON.parse(nothingJson) + @test testDict == nothingDict + + @testset "async" begin + include("async.jl") + end + + @testset "indentation" begin + include("indentation.jl") + end + + @testset "JSON Checker" begin + include("json-checker.jl") + end +end + +@testset "Regression" begin + @testset "for issue #$i" for i in [21, 26, 57, 109, 152, 163] + include("regression/issue$(lpad(string(i), 3, "0")).jl") + end +end + +# Check that printing to the default stdout doesn't fail diff --git a/packages/JSON/test/serializer.jl b/packages/JSON/test/serializer.jl new file mode 100644 index 0000000..87927fe --- /dev/null +++ b/packages/JSON/test/serializer.jl @@ -0,0 +1,95 @@ +module TestSerializer + +using JSON +using Test + +# to define a new serialization behaviour, import these first +import JSON.Serializations: CommonSerialization, StandardSerialization +import JSON: StructuralContext + +# those names are long so we can define some type aliases +const CS = CommonSerialization +const SC = StructuralContext + +# for test harness purposes +function sprint_kwarg(f, args...; kwargs...) + b = IOBuffer() + f(b, args...; kwargs...) + String(take!(b)) +end + +# issue #168: Print NaN and Inf as Julia would +struct NaNSerialization <: CS end +JSON.show_json(io::SC, ::NaNSerialization, f::AbstractFloat) = Base.print(io, f) + +@test sprint(JSON.show_json, NaNSerialization(), [NaN, Inf, -Inf, 0.0]) == + "[NaN,Inf,-Inf,0.0]" + +@test sprint_kwarg( + JSON.show_json, + NaNSerialization(), + [NaN, Inf, -Inf, 0.0]; + indent=4 +) == """ +[ + NaN, + Inf, + -Inf, + 0.0 +] +""" + +# issue #170: Print JavaScript functions directly +struct JSSerialization <: CS end +struct JSFunction + data::String +end + +function JSON.show_json(io::SC, ::JSSerialization, f::JSFunction) + first = true + for line in split(f.data, '\n') + if !first + JSON.indent(io) + end + first = false + Base.print(io, line) + end +end + +@test sprint_kwarg(JSON.show_json, JSSerialization(), Any[ + 1, + 2, + JSFunction("function test() {\n return 1;\n}") +]; indent=2) == """ +[ + 1, + 2, + function test() { + return 1; + } +] +""" + +# test serializing a type without any fields +struct SingletonType end +@test_throws ErrorException json(SingletonType()) + +# test printing to stdout +let filename = tempname() + open(filename, "w") do f + redirect_stdout(f) do + JSON.print(Any[1, 2, 3.0]) + end + end + @test read(filename, String) == "[1,2,3.0]" + rm(filename) +end + +# issue #184: serializing a 0-dimensional array +@test sprint(JSON.show_json, JSON.StandardSerialization(), view([184], 1)) == "184" + +# test serializing with a JSONText object +@test json([JSONText("{\"bar\": [3,4,5]}"),314159]) == "[{\"bar\": [3,4,5]},314159]" +@test json([JSONText("{\"bar\": [3,4,5]}"),314159], 1) == "[\n {\n \"bar\": [\n 3,\n 4,\n 5\n ]\n },\n 314159\n]\n" + +end diff --git a/packages/JSON/test/standard-serializer.jl b/packages/JSON/test/standard-serializer.jl new file mode 100644 index 0000000..034bfc4 --- /dev/null +++ b/packages/JSON/test/standard-serializer.jl @@ -0,0 +1,72 @@ +@testset "Symbol" begin + symtest = Dict(:symbolarray => [:apple, :pear], :symbolsingleton => :hello) + @test (JSON.json(symtest) == "{\"symbolarray\":[\"apple\",\"pear\"],\"symbolsingleton\":\"hello\"}" + || JSON.json(symtest) == "{\"symbolsingleton\":\"hello\",\"symbolarray\":[\"apple\",\"pear\"]}") +end + +@testset "Floats" begin + @test sprint(JSON.print, [NaN]) == "[null]" + @test sprint(JSON.print, [Inf]) == "[null]" +end + +@testset "Union{Nothing,T} (old Nullable)" begin + @test sprint(JSON.print, Union{Any,Nothing}[nothing]) == "[null]" + @test sprint(JSON.print, Union{Int64,Nothing}[nothing]) == "[null]" + @test sprint(JSON.print, Union{Int64,Nothing}[1]) == "[1]" +end + +@testset "Char" begin + @test json('a') == "\"a\"" + @test json('\\') == "\"\\\\\"" + @test json('\n') == "\"\\n\"" + @test json('🍩') =="\"🍩\"" +end + +@testset "Enum" begin + include("enum.jl") +end + +@testset "Type" begin + @test sprint(JSON.print, Float64) == string("\"Float64\"") +end + +@testset "Module" begin + @test_throws ArgumentError sprint(JSON.print, JSON) +end + +@testset "Dates" begin + @test json(Date("2016-04-13")) == "\"2016-04-13\"" + @test json([Date("2016-04-13"), Date("2016-04-12")]) == "[\"2016-04-13\",\"2016-04-12\"]" + @test json(DateTime("2016-04-13T00:00:00")) == "\"2016-04-13T00:00:00\"" + @test json([DateTime("2016-04-13T00:00:00"), DateTime("2016-04-12T00:00:00")]) == "[\"2016-04-13T00:00:00\",\"2016-04-12T00:00:00\"]" +end + +@testset "Null bytes" begin + zeros = Dict("\0" => "\0") + json_zeros = json(zeros) + @test occursin("\\u0000", json_zeros) + @test !occursin("\\0", json_zeros) + @test JSON.parse(json_zeros) == zeros +end + +@testset "Arrays" begin + # Printing an empty array or Dict shouldn't cause a BoundsError + @test json(String[]) == "[]" + @test json(Dict()) == "{}" + + #Multidimensional arrays + @test json([0 1; 2 0]) == "[[0,2],[1,0]]" + @test json(OffsetArray([0 1; 2 0], 0:1, 10:11)) == "[[0,2],[1,0]]" +end + +@testset "Pairs" begin + @test json(1 => 2) == "{\"1\":2}" + @test json(:foo => 2) == "{\"foo\":2}" + @test json([1, 2] => [3, 4]) == "{\"$([1, 2])\":[3,4]}" + @test json([1 => 2]) == "[{\"1\":2}]" +end + +@testset "Sets" begin + @test json(Set()) == "[]" + @test json(Set([1, 2])) in ["[1,2]", "[2,1]"] +end diff --git a/packages/JSONRPC/.github/pull_request_template.md b/packages/JSONRPC/.github/pull_request_template.md new file mode 100644 index 0000000..685d099 --- /dev/null +++ b/packages/JSONRPC/.github/pull_request_template.md @@ -0,0 +1,5 @@ +Fixes #. + +For every PR, please check the following: +- [ ] End-user documentation check. If this PR requires end-user documentation in the Julia VS Code extension docs, please add that at https://github.com/julia-vscode/docs. +- [ ] Changelog mention. If this PR should be mentioned in the CHANGELOG for the Julia VS Code extension, please open a PR against https://github.com/julia-vscode/julia-vscode/blob/master/CHANGELOG.md with those changes. diff --git a/packages/JSONRPC/.github/workflows/jlpkgbutler-butler-workflow.yml b/packages/JSONRPC/.github/workflows/jlpkgbutler-butler-workflow.yml new file mode 100644 index 0000000..70544cc --- /dev/null +++ b/packages/JSONRPC/.github/workflows/jlpkgbutler-butler-workflow.yml @@ -0,0 +1,22 @@ +name: Run the Julia Package Butler + +on: + push: + branches: + - main + - master + schedule: + - cron: '0 */1 * * *' + workflow_dispatch: + +jobs: + butler: + name: "Run Package Butler" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: davidanthoff/julia-pkgbutler@releases/v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + ssh-private-key: ${{ secrets.JLPKGBUTLER_TOKEN }} + channel: stable diff --git a/packages/JSONRPC/.github/workflows/jlpkgbutler-ci-master-workflow.yml b/packages/JSONRPC/.github/workflows/jlpkgbutler-ci-master-workflow.yml new file mode 100644 index 0000000..09cff08 --- /dev/null +++ b/packages/JSONRPC/.github/workflows/jlpkgbutler-ci-master-workflow.yml @@ -0,0 +1,40 @@ +name: Run CI on main + +on: + push: + branches: + - main + - master + workflow_dispatch: + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + julia-version: ['1.0', '1.1', '1.2', '1.3', '1.4', '1.5', '1.6', '1.7', '1.8'] + julia-arch: [x64, x86] + os: [ubuntu-latest, windows-latest, macOS-latest] + exclude: + - os: macOS-latest + julia-arch: x86 + + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.julia-version }} + arch: ${{ matrix.julia-arch }} + - uses: julia-actions/julia-buildpkg@v1 + env: + PYTHON: "" + - uses: julia-actions/julia-runtest@v1 + env: + PYTHON: "" + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v3 + with: + files: ./lcov.info + flags: unittests + token: ${{ secrets.CODECOV_TOKEN }} + \ No newline at end of file diff --git a/packages/JSONRPC/.github/workflows/jlpkgbutler-ci-pr-workflow.yml b/packages/JSONRPC/.github/workflows/jlpkgbutler-ci-pr-workflow.yml new file mode 100644 index 0000000..9114217 --- /dev/null +++ b/packages/JSONRPC/.github/workflows/jlpkgbutler-ci-pr-workflow.yml @@ -0,0 +1,36 @@ +name: Run CI on PR + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + julia-version: ['1.0', '1.1', '1.2', '1.3', '1.4', '1.5', '1.6', '1.7', '1.8'] + julia-arch: [x64, x86] + os: [ubuntu-latest, windows-latest, macOS-latest] + exclude: + - os: macOS-latest + julia-arch: x86 + + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.julia-version }} + arch: ${{ matrix.julia-arch }} + - uses: julia-actions/julia-buildpkg@v1 + env: + PYTHON: "" + - uses: julia-actions/julia-runtest@v1 + env: + PYTHON: "" + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v3 + with: + files: ./lcov.info + flags: unittests + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/packages/JSONRPC/.github/workflows/jlpkgbutler-codeformat-pr-workflow.yml b/packages/JSONRPC/.github/workflows/jlpkgbutler-codeformat-pr-workflow.yml new file mode 100644 index 0000000..411bed4 --- /dev/null +++ b/packages/JSONRPC/.github/workflows/jlpkgbutler-codeformat-pr-workflow.yml @@ -0,0 +1,23 @@ +name: Code Formatting + +on: + push: + branches: + - main + - master + workflow_dispatch: + +jobs: + format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/julia-codeformat@releases/v1 + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: Format files using DocumentFormat + title: '[AUTO] Format files using DocumentFormat' + body: '[DocumentFormat.jl](https://github.com/julia-vscode/DocumentFormat.jl) would suggest these formatting changes' + labels: no changelog diff --git a/packages/JSONRPC/.github/workflows/jlpkgbutler-compathelper-workflow.yml b/packages/JSONRPC/.github/workflows/jlpkgbutler-compathelper-workflow.yml new file mode 100644 index 0000000..b315831 --- /dev/null +++ b/packages/JSONRPC/.github/workflows/jlpkgbutler-compathelper-workflow.yml @@ -0,0 +1,20 @@ +name: Run CompatHelper + +on: + schedule: + - cron: '00 * * * *' + issues: + types: [opened, reopened] + workflow_dispatch: + +jobs: + CompatHelper: + name: "Run CompatHelper.jl" + runs-on: ubuntu-latest + steps: + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/packages/JSONRPC/.github/workflows/jlpkgbutler-tagbot-workflow.yml b/packages/JSONRPC/.github/workflows/jlpkgbutler-tagbot-workflow.yml new file mode 100644 index 0000000..d3ca956 --- /dev/null +++ b/packages/JSONRPC/.github/workflows/jlpkgbutler-tagbot-workflow.yml @@ -0,0 +1,17 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: + +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.JLPKGBUTLER_TOKEN }} + branches: true diff --git a/packages/JSONRPC/.gitignore b/packages/JSONRPC/.gitignore new file mode 100644 index 0000000..722d5e7 --- /dev/null +++ b/packages/JSONRPC/.gitignore @@ -0,0 +1 @@ +.vscode diff --git a/packages/JSONRPC/.jlpkgbutler.toml b/packages/JSONRPC/.jlpkgbutler.toml new file mode 100644 index 0000000..b72304f --- /dev/null +++ b/packages/JSONRPC/.jlpkgbutler.toml @@ -0,0 +1 @@ +template = "bach" diff --git a/packages/JSONRPC/LICENSE.md b/packages/JSONRPC/LICENSE.md new file mode 100644 index 0000000..9872e80 --- /dev/null +++ b/packages/JSONRPC/LICENSE.md @@ -0,0 +1,7 @@ +Copyright 2020-2022 David Anthoff + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/packages/JSONRPC/Project.toml b/packages/JSONRPC/Project.toml new file mode 100644 index 0000000..e031720 --- /dev/null +++ b/packages/JSONRPC/Project.toml @@ -0,0 +1,19 @@ +name = "JSONRPC" +uuid = "b9b8584e-8fd3-41f9-ad0c-7255d428e418" +authors = ["David Anthoff "] +version = "1.3.5-DEV" + +[deps] +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" + +[compat] +JSON = "0.20, 0.21" +julia = "1" + +[targets] +test = ["Test", "Sockets"] diff --git a/packages/JSONRPC/README.md b/packages/JSONRPC/README.md new file mode 100644 index 0000000..b09c320 --- /dev/null +++ b/packages/JSONRPC/README.md @@ -0,0 +1,5 @@ +# JSONRPC + +An implementation for JSON RPC 2.0. See [the specification](https://www.jsonrpc.org/specification) for details. + +Currently, only JSON RPC 2.0 is supported. This package can act as both a client & a server. \ No newline at end of file diff --git a/packages/JSONRPC/src/JSONRPC.jl b/packages/JSONRPC/src/JSONRPC.jl new file mode 100644 index 0000000..cd7204a --- /dev/null +++ b/packages/JSONRPC/src/JSONRPC.jl @@ -0,0 +1,7 @@ +module JSONRPC + +import JSON, UUIDs + +include("packagedef.jl") + +end diff --git a/packages/JSONRPC/src/core.jl b/packages/JSONRPC/src/core.jl new file mode 100644 index 0000000..6883200 --- /dev/null +++ b/packages/JSONRPC/src/core.jl @@ -0,0 +1,267 @@ +struct JSONRPCError <: Exception + code::Int + msg::AbstractString + data::Any +end + +function Base.showerror(io::IO, ex::JSONRPCError) + error_code_as_string = if ex.code == -32700 + "ParseError" + elseif ex.code == -32600 + "InvalidRequest" + elseif ex.code == -32601 + "MethodNotFound" + elseif ex.code == -32602 + "InvalidParams" + elseif ex.code == -32603 + "InternalError" + elseif ex.code == -32099 + "serverErrorStart" + elseif ex.code == -32000 + "serverErrorEnd" + elseif ex.code == -32002 + "ServerNotInitialized" + elseif ex.code == -32001 + "UnknownErrorCode" + elseif ex.code == -32800 + "RequestCancelled" + elseif ex.code == -32801 + "ContentModified" + else + "Unkonwn" + end + + print(io, error_code_as_string) + print(io, ": ") + print(io, ex.msg) + if ex.data !== nothing + print(io, " (") + print(io, ex.data) + print(io, ")") + end +end + +mutable struct JSONRPCEndpoint{IOIn <: IO,IOOut <: IO} + pipe_in::IOIn + pipe_out::IOOut + + out_msg_queue::Channel{Any} + in_msg_queue::Channel{Any} + + outstanding_requests::Dict{String,Channel{Any}} + + err_handler::Union{Nothing,Function} + + status::Symbol + + read_task::Union{Nothing,Task} + write_task::Union{Nothing,Task} +end + +JSONRPCEndpoint(pipe_in, pipe_out, err_handler = nothing) = + JSONRPCEndpoint(pipe_in, pipe_out, Channel{Any}(Inf), Channel{Any}(Inf), Dict{String,Channel{Any}}(), err_handler, :idle, nothing, nothing) + +function write_transport_layer(stream, response) + response_utf8 = transcode(UInt8, response) + n = length(response_utf8) + write(stream, "Content-Length: $n\r\n\r\n") + write(stream, response_utf8) + flush(stream) +end + +function read_transport_layer(stream) + header_dict = Dict{String,String}() + line = chomp(readline(stream)) + # Check whether the socket was closed + if line == "" + return nothing + end + while length(line) > 0 + h_parts = split(line, ":") + header_dict[chomp(h_parts[1])] = chomp(h_parts[2]) + line = chomp(readline(stream)) + end + message_length = parse(Int, header_dict["Content-Length"]) + message_str = String(read(stream, message_length)) + return message_str +end + +Base.isopen(x::JSONRPCEndpoint) = x.status != :closed && isopen(x.pipe_in) && isopen(x.pipe_out) + +function Base.run(x::JSONRPCEndpoint) + x.status == :idle || error("Endpoint is not idle.") + + x.write_task = @async try + try + for msg in x.out_msg_queue + if isopen(x.pipe_out) + write_transport_layer(x.pipe_out, msg) + else + # TODO Reconsider at some point whether this should be treated as an error. + break + end + end + finally + close(x.out_msg_queue) + end + catch err + bt = catch_backtrace() + if x.err_handler !== nothing + x.err_handler(err, bt) + else + Base.display_error(stderr, err, bt) + end + end + + x.read_task = @async try + try + while true + message = read_transport_layer(x.pipe_in) + + if message === nothing || x.status == :closed + break + end + + message_dict = JSON.parse(message) + + if haskey(message_dict, "method") + try + put!(x.in_msg_queue, message_dict) + catch err + if err isa InvalidStateException + break + else + rethrow(err) + end + end + else + # This must be a response + id_of_request = message_dict["id"] + + channel_for_response = x.outstanding_requests[id_of_request] + put!(channel_for_response, message_dict) + end + end + finally + close(x.in_msg_queue) + end + catch err + bt = catch_backtrace() + if x.err_handler !== nothing + x.err_handler(err, bt) + else + Base.display_error(stderr, err, bt) + end + end + + x.status = :running +end + +function send_notification(x::JSONRPCEndpoint, method::AbstractString, params) + check_dead_endpoint!(x) + + message = Dict("jsonrpc" => "2.0", "method" => method, "params" => params) + + message_json = JSON.json(message) + + put!(x.out_msg_queue, message_json) + + return nothing +end + +function send_request(x::JSONRPCEndpoint, method::AbstractString, params) + check_dead_endpoint!(x) + + id = string(UUIDs.uuid4()) + message = Dict("jsonrpc" => "2.0", "method" => method, "params" => params, "id" => id) + + response_channel = Channel{Any}(1) + x.outstanding_requests[id] = response_channel + + message_json = JSON.json(message) + + put!(x.out_msg_queue, message_json) + + response = take!(response_channel) + + if haskey(response, "result") + return response["result"] + elseif haskey(response, "error") + error_code = response["error"]["code"] + error_msg = response["error"]["message"] + error_data = get(response["error"], "data", nothing) + throw(JSONRPCError(error_code, error_msg, error_data)) + else + throw(JSONRPCError(0, "ERROR AT THE TRANSPORT LEVEL", nothing)) + end +end + +function get_next_message(endpoint::JSONRPCEndpoint) + check_dead_endpoint!(endpoint) + + msg = take!(endpoint.in_msg_queue) + + return msg +end + +function Base.iterate(endpoint::JSONRPCEndpoint, state = nothing) + check_dead_endpoint!(endpoint) + + try + return take!(endpoint.in_msg_queue), nothing + catch err + if err isa InvalidStateException + return nothing + else + rethrow(err) + end + end +end + +function send_success_response(endpoint, original_request, result) + check_dead_endpoint!(endpoint) + + response = Dict("jsonrpc" => "2.0", "id" => original_request["id"], "result" => result) + + response_json = JSON.json(response) + + put!(endpoint.out_msg_queue, response_json) +end + +function send_error_response(endpoint, original_request, code, message, data) + check_dead_endpoint!(endpoint) + + response = Dict("jsonrpc" => "2.0", "id" => original_request["id"], "error" => Dict("code" => code, "message" => message, "data" => data)) + + response_json = JSON.json(response) + + put!(endpoint.out_msg_queue, response_json) +end + +function Base.close(endpoint::JSONRPCEndpoint) + flush(endpoint) + + endpoint.status = :closed + isopen(endpoint.in_msg_queue) && close(endpoint.in_msg_queue) + isopen(endpoint.out_msg_queue) && close(endpoint.out_msg_queue) + + fetch(endpoint.write_task) + # TODO we would also like to close the read Task + # But unclear how to do that without also closing + # the socket, which we don't want to do + # fetch(endpoint.read_task) +end + +function Base.flush(endpoint::JSONRPCEndpoint) + check_dead_endpoint!(endpoint) + + while isready(endpoint.out_msg_queue) + yield() + end +end + +function check_dead_endpoint!(endpoint) + status = endpoint.status + status === :running && return + error("Endpoint is not running, the current state is $(status).") +end diff --git a/packages/JSONRPC/src/interface_def.jl b/packages/JSONRPC/src/interface_def.jl new file mode 100644 index 0000000..4cd303f --- /dev/null +++ b/packages/JSONRPC/src/interface_def.jl @@ -0,0 +1,87 @@ +abstract type Outbound end + +function JSON.Writer.CompositeTypeWrapper(t::Outbound) + fns = collect(fieldnames(typeof(t))) + dels = Int[] + for i = 1:length(fns) + f = fns[i] + if getfield(t, f) isa Missing + push!(dels, i) + end + end + deleteat!(fns, dels) + JSON.Writer.CompositeTypeWrapper(t, Tuple(fns)) +end + +function JSON.lower(a::Outbound) + if nfields(a) > 0 + JSON.Writer.CompositeTypeWrapper(a) + else + nothing + end +end + +function field_allows_missing(field::Expr) + field.head == :(::) && field.args[2] isa Expr && + field.args[2].head == :curly && field.args[2].args[1] == :Union && + any(i -> i == :Missing, field.args[2].args) +end + +function field_type(field::Expr, typename::String) + if field.args[2] isa Expr && field.args[2].head == :curly && field.args[2].args[1] == :Union + if length(field.args[2].args) == 3 && (field.args[2].args[2] == :Missing || field.args[2].args[3] == :Missing) + return field.args[2].args[2] == :Missing ? field.args[2].args[3] : field.args[2].args[2] + else + # We return Any for now, which will lead to no type conversion + return :Any + end + else + return field.args[2] + end +end + +function get_kwsignature_for_field(field::Expr) + fieldname = field.args[1] + fieldtype = field.args[2] + default_value = field_allows_missing(field) ? missing : :(error("You must provide a value for the $fieldname field.")) + + return Expr(:kw, Expr(Symbol("::"), fieldname, fieldtype), default_value) +end + +macro dict_readable(arg) + tname = arg.args[2] isa Expr ? arg.args[2].args[1] : arg.args[2] + count_real_fields = count(field -> !(field isa LineNumberNode), arg.args[3].args) + ex = quote + $((arg)) + + $(count_real_fields > 0 ? :( + function $tname(; $((get_kwsignature_for_field(field) for field in arg.args[3].args if !(field isa LineNumberNode))...)) + $tname($((field.args[1] for field in arg.args[3].args if !(field isa LineNumberNode))...)) + end + ) : nothing) + + function $tname(dict::Dict) + end + end + + fex = :($((tname))()) + for field in arg.args[3].args + if !(field isa LineNumberNode) + fieldname = string(field.args[1]) + fieldtype = field_type(field, string(tname)) + if fieldtype isa Expr && fieldtype.head == :curly && fieldtype.args[2] != :Any + f = :($(fieldtype.args[2]).(dict[$fieldname])) + elseif fieldtype != :Any + f = :($(fieldtype)(dict[$fieldname])) + else + f = :(dict[$fieldname]) + end + if field_allows_missing(field) + f = :(haskey(dict, $fieldname) ? $f : missing) + end + push!(fex.args, f) + end + end + push!(ex.args[end].args[2].args, fex) + return esc(ex) +end diff --git a/packages/JSONRPC/src/packagedef.jl b/packages/JSONRPC/src/packagedef.jl new file mode 100644 index 0000000..e57c713 --- /dev/null +++ b/packages/JSONRPC/src/packagedef.jl @@ -0,0 +1,5 @@ +export JSONRPCEndpoint, send_notification, send_request, send_success_response, send_error_response + +include("core.jl") +include("typed.jl") +include("interface_def.jl") diff --git a/packages/JSONRPC/src/typed.jl b/packages/JSONRPC/src/typed.jl new file mode 100644 index 0000000..34cfe3b --- /dev/null +++ b/packages/JSONRPC/src/typed.jl @@ -0,0 +1,88 @@ +abstract type AbstractMessageType end + +struct NotificationType{TPARAM} <: AbstractMessageType + method::String +end + +struct RequestType{TPARAM,TR} <: AbstractMessageType + method::String +end + +function NotificationType(method::AbstractString, ::Type{TPARAM}) where TPARAM + return NotificationType{TPARAM}(method) +end + +function RequestType(method::AbstractString, ::Type{TPARAM}, ::Type{TR}) where {TPARAM,TR} + return RequestType{TPARAM,TR}(method) +end + +get_param_type(::NotificationType{TPARAM}) where {TPARAM} = TPARAM +get_param_type(::RequestType{TPARAM,TR}) where {TPARAM,TR} = TPARAM +get_return_type(::RequestType{TPARAM,TR}) where {TPARAM,TR} = TR + +function send(x::JSONRPCEndpoint, request::RequestType{TPARAM,TR}, params::TPARAM) where {TPARAM,TR} + res = send_request(x, request.method, params) + return typed_res(res, TR)::TR +end + +# `send_request` must have returned nothing in this case, we pass this on +# so that we get an error in the typecast at the end of `send` +# if that is not the case. +typed_res(res, TR::Type{Nothing}) = res +typed_res(res, TR::Type{<:T}) where {T <: AbstractArray{Any}} = T(res) +typed_res(res, TR::Type{<:AbstractArray{T}}) where T = T.(res) +typed_res(res, TR::Type) = TR(res) + +function send(x::JSONRPCEndpoint, notification::NotificationType{TPARAM}, params::TPARAM) where TPARAM + send_notification(x, notification.method, params) +end + +struct Handler + message_type::AbstractMessageType + func::Function +end + +mutable struct MsgDispatcher + _handlers::Dict{String,Handler} + _currentlyHandlingMsg::Bool + + function MsgDispatcher() + new(Dict{String,Handler}(), false) + end +end + +function Base.setindex!(dispatcher::MsgDispatcher, func::Function, message_type::AbstractMessageType) + dispatcher._handlers[message_type.method] = Handler(message_type, func) +end + +function dispatch_msg(x::JSONRPCEndpoint, dispatcher::MsgDispatcher, msg) + dispatcher._currentlyHandlingMsg = true + try + method_name = msg["method"] + handler = get(dispatcher._handlers, method_name, nothing) + if handler !== nothing + param_type = get_param_type(handler.message_type) + params = param_type === Nothing ? nothing : param_type <: NamedTuple ? convert(param_type,(;(Symbol(i[1])=>i[2] for i in msg["params"])...)) : param_type(msg["params"]) + + res = handler.func(x, params) + + if handler.message_type isa RequestType + if res isa JSONRPCError + send_error_response(x, msg, res.code, res.msg, res.data) + elseif res isa get_return_type(handler.message_type) + send_success_response(x, msg, res) + else + error_msg = "The handler for the '$method_name' request returned a value of type $(typeof(res)), which is not a valid return type according to the request definition." + send_error_response(x, msg, -32603, error_msg, nothing) + error(error_msg) + end + end + else + error("Unknown method $method_name.") + end + finally + dispatcher._currentlyHandlingMsg = false + end +end + +is_currently_handling_msg(d::MsgDispatcher) = d._currentlyHandlingMsg diff --git a/packages/JSONRPC/test/runtests.jl b/packages/JSONRPC/test/runtests.jl new file mode 100644 index 0000000..9c3ce4f --- /dev/null +++ b/packages/JSONRPC/test/runtests.jl @@ -0,0 +1,34 @@ +using Test +using JSON +using JSONRPC +using JSONRPC: typed_res, @dict_readable, Outbound +using Sockets + +@dict_readable struct Foo <: Outbound + fieldA::Int + fieldB::String + fieldC::Union{Missing,String} + fieldD::Union{String,Missing} +end + +@dict_readable struct Foo2 <: Outbound + fieldA::Union{Nothing,Int} + fieldB::Vector{Int} +end + +Base.:(==)(a::Foo2,b::Foo2) = a.fieldA == b.fieldA && a.fieldB == b.fieldB + +@testset "JSONRPC" begin + include("test_core.jl") + include("test_interface_def.jl") + include("test_typed.jl") + + @testset "check response type" begin + @test typed_res(nothing, Nothing) isa Nothing + @test typed_res([1,"2",3], Vector{Any}) isa Vector{Any} + @test typed_res([1,2,3], Vector{Int}) isa Vector{Int} + @test typed_res([1,2,3], Vector{Float64}) isa Vector{Float64} + @test typed_res(['f','o','o'], String) isa String + @test typed_res("foo", String) isa String + end +end diff --git a/packages/JSONRPC/test/test_core.jl b/packages/JSONRPC/test/test_core.jl new file mode 100644 index 0000000..8e23f8c --- /dev/null +++ b/packages/JSONRPC/test/test_core.jl @@ -0,0 +1,14 @@ +@testset "Core" begin + @test sprint(showerror, JSONRPC.JSONRPCError(-32700, "FOO", "BAR")) == "ParseError: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32600, "FOO", "BAR")) == "InvalidRequest: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32601, "FOO", "BAR")) == "MethodNotFound: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32602, "FOO", "BAR")) == "InvalidParams: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32603, "FOO", "BAR")) == "InternalError: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32099, "FOO", "BAR")) == "serverErrorStart: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32000, "FOO", "BAR")) == "serverErrorEnd: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32002, "FOO", "BAR")) == "ServerNotInitialized: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32001, "FOO", "BAR")) == "UnknownErrorCode: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32800, "FOO", "BAR")) == "RequestCancelled: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(-32801, "FOO", "BAR")) == "ContentModified: FOO (BAR)" + @test sprint(showerror, JSONRPC.JSONRPCError(1, "FOO", "BAR")) == "Unkonwn: FOO (BAR)" +end diff --git a/packages/JSONRPC/test/test_interface_def.jl b/packages/JSONRPC/test/test_interface_def.jl new file mode 100644 index 0000000..f500856 --- /dev/null +++ b/packages/JSONRPC/test/test_interface_def.jl @@ -0,0 +1,32 @@ +@testset "Interface Definition" begin + + @test_throws ErrorException Foo() + + a = Foo(fieldA=1, fieldB="A") + + @test a.fieldA == 1 + @test a.fieldB == "A" + @test a.fieldC === missing + @test a.fieldD === missing + + b = Foo(fieldA=1, fieldB="A", fieldC="B", fieldD="C") + + @test b.fieldA == 1 + @test b.fieldB == "A" + @test b.fieldC == "B" + @test b.fieldD == "C" + + @test Foo(JSON.parse(JSON.json(a))) == a + @test Foo(JSON.parse(JSON.json(b))) == b + + c = Foo2(fieldA=nothing, fieldB=[1,2]) + + @test c.fieldA === nothing + @test c.fieldB == [1,2] + @test Foo2(JSON.parse(JSON.json(c))) == c + + d = Foo2(fieldA=3, fieldB=[1,2]) + @test d.fieldA === 3 + @test d.fieldB == [1,2] + @test Foo2(JSON.parse(JSON.json(d))) == d +end diff --git a/packages/JSONRPC/test/test_typed.jl b/packages/JSONRPC/test/test_typed.jl new file mode 100644 index 0000000..c299422 --- /dev/null +++ b/packages/JSONRPC/test/test_typed.jl @@ -0,0 +1,109 @@ +@testset "Message dispatcher" begin + + if Sys.iswindows() + global_socket_name1 = "\\\\.\\pipe\\jsonrpc-testrun1" + elseif Sys.isunix() + global_socket_name1 = joinpath(tempdir(), "jsonrpc-testrun1") + else + error("Unknown operating system.") + end + + request1_type = JSONRPC.RequestType("request1", Foo, String) + request2_type = JSONRPC.RequestType("request2", Nothing, String) + notify1_type = JSONRPC.NotificationType("notify1", String) + + global g_var = "" + + server_is_up = Base.Condition() + + server_task = @async try + server = listen(global_socket_name1) + notify(server_is_up) + sock = accept(server) + global conn = JSONRPC.JSONRPCEndpoint(sock, sock) + global msg_dispatcher = JSONRPC.MsgDispatcher() + + msg_dispatcher[request1_type] = (conn, params) -> begin + @test JSONRPC.is_currently_handling_msg(msg_dispatcher) + params.fieldA == 1 ? "YES" : "NO" + end + msg_dispatcher[request2_type] = (conn, params) -> JSONRPC.JSONRPCError(-32600, "Our message", nothing) + msg_dispatcher[notify1_type] = (conn, params) -> g_var = params + + run(conn) + + for msg in conn + JSONRPC.dispatch_msg(conn, msg_dispatcher, msg) + end + catch err + Base.display_error(stderr, err, catch_backtrace()) + end + + wait(server_is_up) + + sock2 = connect(global_socket_name1) + conn2 = JSONRPCEndpoint(sock2, sock2) + + run(conn2) + + JSONRPC.send(conn2, notify1_type, "TEST") + + res = JSONRPC.send(conn2, request1_type, Foo(fieldA=1, fieldB="FOO")) + + @test res == "YES" + @test g_var == "TEST" + + @test_throws JSONRPC.JSONRPCError(-32600, "Our message", nothing) JSONRPC.send(conn2, request2_type, nothing) + + close(conn2) + close(sock2) + close(conn) + + fetch(server_task) + + # Now we test a faulty server + + if Sys.iswindows() + global_socket_name2 = "\\\\.\\pipe\\jsonrpc-testrun2" + elseif Sys.isunix() + global_socket_name2 = joinpath(tempdir(), "jsonrpc-testrun2") + else + error("Unknown operating system.") + end + + server_is_up = Base.Condition() + + server_task2 = @async try + server = listen(global_socket_name2) + notify(server_is_up) + sock = accept(server) + global conn = JSONRPC.JSONRPCEndpoint(sock, sock) + global msg_dispatcher = JSONRPC.MsgDispatcher() + + msg_dispatcher[request2_type] = (conn, params)->34 # The request type requires a `String` return, so this tests whether we get an error. + + run(conn) + + for msg in conn + @test_throws ErrorException("The handler for the 'request2' request returned a value of type $Int, which is not a valid return type according to the request definition.") JSONRPC.dispatch_msg(conn, msg_dispatcher, msg) + end + catch err + Base.display_error(stderr, err, catch_backtrace()) + end + + wait(server_is_up) + + sock2 = connect(global_socket_name2) + conn2 = JSONRPCEndpoint(sock2, sock2) + + run(conn2) + + @test_throws JSONRPC.JSONRPCError(-32603, "The handler for the 'request2' request returned a value of type $Int, which is not a valid return type according to the request definition.", nothing) JSONRPC.send(conn2, request2_type, nothing) + + close(conn2) + close(sock2) + close(conn) + + fetch(server_task) + +end diff --git a/packages/JuliaInterpreter/.github/workflows/CI.yml b/packages/JuliaInterpreter/.github/workflows/CI.yml new file mode 100644 index 0000000..7d9fdd6 --- /dev/null +++ b/packages/JuliaInterpreter/.github/workflows/CI.yml @@ -0,0 +1,48 @@ +name: CI +on: + pull_request: + push: + branches: + - master + tags: '*' +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.6' # LTS + - '1' # current stable + - 'nightly' + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x64 + env: + PYTHON: "" + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v1 + with: + file: lcov.info diff --git a/packages/JuliaInterpreter/.github/workflows/Documenter.yml b/packages/JuliaInterpreter/.github/workflows/Documenter.yml new file mode 100644 index 0000000..8faa2e2 --- /dev/null +++ b/packages/JuliaInterpreter/.github/workflows/Documenter.yml @@ -0,0 +1,18 @@ +name: Documenter +on: + push: + branches: [master] + tags: [v*] + pull_request: + +jobs: + Documenter: + name: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/julia-buildpkg@latest + - uses: julia-actions/julia-docdeploy@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/packages/JuliaInterpreter/.github/workflows/TagBot.yml b/packages/JuliaInterpreter/.github/workflows/TagBot.yml new file mode 100644 index 0000000..f49313b --- /dev/null +++ b/packages/JuliaInterpreter/.github/workflows/TagBot.yml @@ -0,0 +1,15 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/packages/JuliaInterpreter/.github/workflows/check_builtins.yml b/packages/JuliaInterpreter/.github/workflows/check_builtins.yml new file mode 100644 index 0000000..fd3b6e7 --- /dev/null +++ b/packages/JuliaInterpreter/.github/workflows/check_builtins.yml @@ -0,0 +1,30 @@ +name: CI +on: + pull_request: + push: + branches: + - master + tags: '*' +jobs: + test: + name: 'Check builtins.jl consistency' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: nightly + arch: x64 + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - run: julia -e 'import Pkg; Pkg.add(["DeepDiffs", "Test"])' + - uses: julia-actions/julia-buildpkg@v1 + - run: julia test/check_builtins.jl diff --git a/packages/JuliaInterpreter/.gitignore b/packages/JuliaInterpreter/.gitignore new file mode 100644 index 0000000..c037984 --- /dev/null +++ b/packages/JuliaInterpreter/.gitignore @@ -0,0 +1,8 @@ +*.jl.cov +*.jl.*.cov +*.jl.mem +deps/build.log +docs/build/ +test/results.md +Manifest.toml +!/test/code_coverage/coverage_example.jl.cov diff --git a/packages/JuliaInterpreter/LICENSE.md b/packages/JuliaInterpreter/LICENSE.md new file mode 100644 index 0000000..fc64b81 --- /dev/null +++ b/packages/JuliaInterpreter/LICENSE.md @@ -0,0 +1,22 @@ +The JuliaInterpreter.jl package is licensed under the MIT "Expat" License: + +> Copyright (c) 2017: Keno Fischer. +> +> Permission is hereby granted, free of charge, to any person obtaining a copy +> of this software and associated documentation files (the "Software"), to deal +> in the Software without restriction, including without limitation the rights +> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is +> furnished to do so, subject to the following conditions: +> +> The above copyright notice and this permission notice shall be included in all +> copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. +> diff --git a/packages/JuliaInterpreter/Project.toml b/packages/JuliaInterpreter/Project.toml new file mode 100644 index 0000000..fce96d8 --- /dev/null +++ b/packages/JuliaInterpreter/Project.toml @@ -0,0 +1,34 @@ +name = "JuliaInterpreter" +uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +version = "0.9.20" + +[deps] +CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[compat] +CodeTracking = "0.5.9, 1" +julia = "1.6" + +[extras] +CassetteOverlay = "d78b62d4-37fa-4a6f-acd8-2f19986eb9ee" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DeepDiffs = "ab62b9b5-e342-54a8-a765-a90f495de1a6" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +FunctionWrappers = "069b7b12-0de2-55c6-9aab-29f3d0a68a2e" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +LoopVectorization = "bdcacae8-1622-11e9-2a5c-532679323890" +Mmap = "a63ad114-7e13-5084-954f-fe012c677804" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Tensors = "48a634ad-e948-5137-8d70-aa71f2a747f4" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["CassetteOverlay", "DataFrames", "Dates", "DeepDiffs", "Distributed", "FunctionWrappers", "HTTP", "LinearAlgebra", "Logging", "LoopVectorization", "Mmap", "PyCall", "SHA", "SparseArrays", "Tensors", "Test"] diff --git a/packages/JuliaInterpreter/README.md b/packages/JuliaInterpreter/README.md new file mode 100644 index 0000000..6170c40 --- /dev/null +++ b/packages/JuliaInterpreter/README.md @@ -0,0 +1,22 @@ +# JuliaInterpreter + +*An interpreter for Julia code.* + +| **Documentation** | **Build Status** | +|:-------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------:| +| [![][docs-stable-img]][docs-stable-url] | [![][gh-actions-img]][gh-actions-url] [![][codecov-img]][codecov-url] | + +## Installation + +```jl +]add JuliaInterpreter +``` + +[docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg +[docs-stable-url]: https://JuliaDebug.github.io/JuliaInterpreter.jl/stable + +[gh-actions-img]: https://github.com/JuliaDebug/JuliaInterpreter.jl/actions/workflows/CI.yml/badge.svg +[gh-actions-url]: https://github.com/JuliaDebug/JuliaInterpreter.jl/actions/workflows/CI.yml + +[codecov-img]: https://codecov.io/gh/JuliaDebug/JuliaInterpreter.jl/branch/master/graph/badge.svg +[codecov-url]: https://codecov.io/gh/JuliaDebug/JuliaInterpreter.jl diff --git a/packages/JuliaInterpreter/benchmark/.gitignore b/packages/JuliaInterpreter/benchmark/.gitignore new file mode 100644 index 0000000..604134c --- /dev/null +++ b/packages/JuliaInterpreter/benchmark/.gitignore @@ -0,0 +1 @@ +tune.json \ No newline at end of file diff --git a/packages/JuliaInterpreter/benchmark/README.md b/packages/JuliaInterpreter/benchmark/README.md new file mode 100644 index 0000000..b9edf77 --- /dev/null +++ b/packages/JuliaInterpreter/benchmark/README.md @@ -0,0 +1,9 @@ +The benchmarks are recommended to be run using PkgBenchmark.jl as: + +``` +using PkgBenchmark +results = benchmarkpkg("JuliaInterpreter") +``` + +See the [PkgBenchmark](https://juliaci.github.io/PkgBenchmark.jl/stable/index.html) documentation for what +analysis is possible on `result`. \ No newline at end of file diff --git a/packages/JuliaInterpreter/benchmark/benchmarks.jl b/packages/JuliaInterpreter/benchmark/benchmarks.jl new file mode 100644 index 0000000..80da56a --- /dev/null +++ b/packages/JuliaInterpreter/benchmark/benchmarks.jl @@ -0,0 +1,68 @@ +using JuliaInterpreter +using BenchmarkTools + +const SUITE = BenchmarkGroup() + +# Recursively call itself +f(i, j) = i == 0 ? j : f(i - 1, j + 1) +SUITE["recursive self 1_000"] = @benchmarkable @interpret f(1_000, 0) + +# Long stack trace calling other functions +f0(i) = i +for i in 1:1_000 + @eval $(Symbol("f", i))(i) = $(Symbol("f", i-1))(i) +end +SUITE["recursive other 1_000"] = @benchmarkable @interpret f1000(1) + +# Tight loop +function f(X) + s = 0 + for x in X + s += x + end + return s +end +const X = rand(1:10, 10_000) +SUITE["tight loop 10_000"] = @benchmarkable @interpret f(X) + +# Throwing and catching an error over a large stacktrace +function g0(i) + try + g1(i) + catch e + e + end +end +for i in 1:1_000 + @eval $(Symbol("g", i))(i) = $(Symbol("g", i+1))(i) +end +g1001(i) = error() +SUITE["throw long 1_000"] = @benchmarkable @interpret g0(1) + +# Function with many statements +macro do_thing(expr, N) + e = Expr(:block) + for i in 1:N + push!(e.args, esc(expr)) + end + return e +end + +function counter() + a = 0 + @do_thing(a = a + 1, 5_000) + return a +end +SUITE["long function 5_000"] = @benchmarkable @interpret counter() + +# Ccall +function ccall_ptr(ptr, x, y) + ccall(ptr, Int, (Int, Int), x, y) +end +const ptr = @cfunction(+, Int, (Int, Int)) +SUITE["ccall ptr"] = @benchmarkable @interpret ccall_ptr(ptr, 1, 5) + +function powf(a, b) + ccall(("powf", Base.Math.libm), Float32, (Float32,Float32), a, b) +end +SUITE["ccall library"] = @benchmarkable @interpret powf(2, 3) \ No newline at end of file diff --git a/packages/JuliaInterpreter/bin/generate_builtins.jl b/packages/JuliaInterpreter/bin/generate_builtins.jl new file mode 100644 index 0000000..bc6da0d --- /dev/null +++ b/packages/JuliaInterpreter/bin/generate_builtins.jl @@ -0,0 +1,321 @@ +# This file generates builtins.jl. +# Should be run on the latest Julia nightly +using InteractiveUtils + +# All bultins added since 1.6. Needs to be updated whenever a new builtin is added +const RECENTLY_ADDED = Core.Builtin[ + Core._call_in_world_total, Core.donotdelete, + Core.get_binding_type, Core.set_binding_type!, + Core.getglobal, Core.setglobal!, + Core.modifyfield!, Core.replacefield!, Core.swapfield!, + Core.finalizer, Core._compute_sparams, Core._svec_ref, + Core.compilerbarrier +] +const kwinvoke = Core.kwfunc(Core.invoke) + +function scopedname(f) + io = IOBuffer() + show(io, f) + fstr = String(take!(io)) + occursin('.', fstr) && return fstr + tn = typeof(f).name + Base.isexported(tn.module, Symbol(fstr)) && return fstr + fsym = Symbol(fstr) + isdefined(tn.module, fsym) && return string(tn.module) * '.' * fstr + return "Base." * fstr +end + +function nargs(f, table, id) + # Look up the expected number of arguments in Core.Compiler.tfunc data + if id !== nothing + minarg, maxarg, tfunc = table[id] + else + minarg = 0 + maxarg = typemax(Int) + end + # Specialize arrayref and arrayset for small numbers of arguments + if f == Core.arrayref + maxarg = 5 + elseif f == Core.arrayset + maxarg = 6 + end + return minarg, maxarg +end + +function generate_fcall_nargs(fname, minarg, maxarg) + # Generate a separate call for each number of arguments + maxarg < typemax(Int) || error("call this only for constrained number of arguments") + annotation = fname == "fieldtype" ? "::Type" : "" + wrapper = "if nargs == " + for nargs = minarg:maxarg + wrapper *= "$nargs\n " + argcall = "" + for i = 1:nargs + argcall *= "@lookup(frame, args[$(i+1)])" + if i < nargs + argcall *= ", " + end + end + wrapper *= "return Some{Any}($fname($argcall)$annotation)" + if nargs < maxarg + wrapper *= "\n elseif nargs == " + end + end + wrapper *= "\n else" + wrapper *= "\n return Some{Any}($fname(getargs(args, frame)...)$annotation)" # to throw the correct error + wrapper *= "\n end" + return wrapper +end + +function generate_fcall(f, table, id) + minarg, maxarg = nargs(f, table, id) + fname = scopedname(f) + if maxarg < typemax(Int) + return generate_fcall_nargs(fname, minarg, maxarg) + end + # A built-in with arbitrary or unknown number of arguments. + # This will (unfortunately) use dynamic dispatch. + return "return Some{Any}($fname(getargs(args, frame)...))" +end + +# `io` is for the generated source file +# `intrinsicsfile` is the path to Julia's `src/intrinsics.h` file +function generate_builtins(file::String) + open(file, "w") do io + generate_builtins(io::IO) + end +end +function generate_builtins(io::IO) + pat = r"(ADD_I|ALIAS)\((\w*)," + print(io, +""" +# This file is generated by `generate_builtins.jl`. Do not edit by hand. + +function getargs(args, frame) + nargs = length(args)-1 # skip f + callargs = resize!(frame.framedata.callargs, nargs) + for i = 1:nargs + callargs[i] = @lookup(frame, args[i+1]) + end + return callargs +end + +const kwinvoke = Core.kwfunc(Core.invoke) + +function maybe_recurse_expanded_builtin(frame, new_expr) + f = new_expr.args[1] + if isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction) + return maybe_evaluate_builtin(frame, new_expr, true) + else + return new_expr + end +end + +\"\"\" + ret = maybe_evaluate_builtin(frame, call_expr, expand::Bool) + +If `call_expr` is to a builtin function, evaluate it, returning the result inside a `Some` wrapper. +Otherwise, return `call_expr`. + +If `expand` is true, `Core._apply_iterate` calls will be resolved as a call to the applied function. +\"\"\" +function maybe_evaluate_builtin(frame, call_expr, expand::Bool) + args = call_expr.args + nargs = length(args) - 1 + fex = args[1] + if isa(fex, QuoteNode) + f = fex.value + else + f = @lookup(frame, fex) + end + + @static if isdefined(Core, :OpaqueClosure) + if f isa Core.OpaqueClosure + return Some{Any}(f(args...)) + end + end + if !(isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction)) + return call_expr + end + # By having each call appearing statically in the "switch" block below, + # each gets call-site optimized. +""") + firstcall = true + for ft in subtypes(Core.Builtin) + ft === Core.IntrinsicFunction && continue + ft === typeof(kwinvoke) && continue # handle this one later + head = firstcall ? "if" : "elseif" + firstcall = false + f = ft.instance + fname = scopedname(f) + # Tuple is common, especially for returned values from calls. It's worth avoiding + # dynamic dispatch through a call to `ntuple`. + if f === tuple + print(io, +""" + $head f === tuple + return Some{Any}(ntupleany(i->@lookup(frame, args[i+1]), length(args)-1)) +""") + continue + elseif f === Core._apply_iterate + # Resolve varargs calls + print(io, +""" + $head f === Core._apply_iterate + argswrapped = getargs(args, frame) + if !expand + return Some{Any}(Core._apply_iterate(argswrapped...)) + end + aw1 = argswrapped[1]::Function + @assert aw1 === Core.iterate || aw1 === Core.Compiler.iterate || aw1 === Base.iterate "cannot handle `_apply_iterate` with non iterate as first argument, got \$(aw1), \$(typeof(aw1))" + new_expr = Expr(:call, argswrapped[2]) + popfirst!(argswrapped) # pop the iterate + popfirst!(argswrapped) # pop the function + argsflat = append_any(argswrapped...) + for x in argsflat + push!(new_expr.args, QuoteNode(x)) + end + return maybe_recurse_expanded_builtin(frame, new_expr) +""") + continue + elseif f === Core.invoke + fstr = scopedname(f) + print(io, +""" + $head f === $fstr + if !expand + argswrapped = getargs(args, frame) + return Some{Any}($fstr(argswrapped...)) + end + # This uses the original arguments to avoid looking them up twice + # See #442 + return Expr(:call, invoke, args[2:end]...) +""") + continue + elseif f === Core._call_latest + print(io, +""" + elseif f === Core._call_latest + args = getargs(args, frame) + if !expand + return Some{Any}(Core._call_latest(args...)) + end + new_expr = Expr(:call, args[1]) + popfirst!(args) + for x in args + push!(new_expr.args, QuoteNode(x)) + end + return maybe_recurse_expanded_builtin(frame, new_expr) +""") + end + + id = findfirst(isequal(f), Core.Compiler.T_FFUNC_KEY) + fcall = generate_fcall(f, Core.Compiler.T_FFUNC_VAL, id) + if f in RECENTLY_ADDED + print(io, +""" + $head @static isdefined($(ft.name.module), $(repr(nameof(f)))) && f === $fname + $fcall +""") + else + print(io, +""" + $head f === $fname + $fcall +""") + end + firstcall = false + end + print(io, +""" + # Intrinsics +""") + print(io, +""" + elseif f === Base.cglobal + if nargs == 1 + call_expr = copy(call_expr) + args2 = args[2] + call_expr.args[2] = isa(args2, QuoteNode) ? args2 : @lookup(frame, args2) + return Some{Any}(Core.eval(moduleof(frame), call_expr)) + elseif nargs == 2 + call_expr = copy(call_expr) + args2 = args[2] + call_expr.args[2] = isa(args2, QuoteNode) ? args2 : @lookup(frame, args2) + call_expr.args[3] = @lookup(frame, args[3]) + return Some{Any}(Core.eval(moduleof(frame), call_expr)) + end +""") + # Extract any intrinsics that support varargs + fva = [] + minmin, maxmax = typemax(Int), 0 + for fsym in names(Core.Intrinsics) + fsym === :Intrinsics && continue + isdefined(Base, fsym) || continue + f = getfield(Base, fsym) + id = reinterpret(Int32, f) + 1 + minarg, maxarg = nargs(f, Core.Compiler.T_IFUNC, id) + if maxarg == typemax(Int) + push!(fva, f) + else + minmin = min(minmin, minarg) + maxmax = max(maxmax, maxarg) + end + end + for f in fva + id = reinterpret(Int32, f) + 1 + fname = scopedname(f) + fcall = generate_fcall(f, Core.Compiler.T_IFUNC, id) + print(io, +""" + elseif f === $fname + $fcall + end +""") + end + # Now handle calls with bounded numbers of args + print(io, +""" + if isa(f, Core.IntrinsicFunction) + cargs = getargs(args, frame) + @static if isdefined(Core.Intrinsics, :have_fma) + if f === Core.Intrinsics.have_fma && length(cargs) == 1 + cargs1 = cargs[1] + if cargs1 == Float64 + return Some{Any}(FMA_FLOAT64[]) + elseif cargs1 == Float32 + return Some{Any}(FMA_FLOAT32[]) + elseif cargs1 == Float16 + return Some{Any}(FMA_FLOAT16[]) + end + end + end + if f === Core.Intrinsics.muladd_float && length(cargs) == 3 + a, b, c = cargs + Ta, Tb, Tc = typeof(a), typeof(b), typeof(c) + if !(Ta == Tb == Tc) + error("muladd_float: types of a, b, and c must match") + end + if Ta == Float64 && FMA_FLOAT64[] + f = Core.Intrinsics.fma_float + elseif Ta == Float32 && FMA_FLOAT32[] + f = Core.Intrinsics.fma_float + elseif Ta == Float16 && FMA_FLOAT16[] + f = Core.Intrinsics.fma_float + end + end + return Some{Any}(ccall(:jl_f_intrinsic_call, Any, (Any, Ptr{Any}, UInt32), f, cargs, length(cargs))) +""") + print(io, +""" + end + if isa(f, typeof(kwinvoke)) + return Some{Any}(kwinvoke(getargs(args, frame)...)) + end + return call_expr +end +""") +end + +builtins_dir = get(ENV, "JULIAINTERPRETER_BUILTINS_DIR", joinpath(@__DIR__, "..", "src")) +generate_builtins(joinpath(builtins_dir, "builtins.jl")) diff --git a/packages/JuliaInterpreter/docs/Manifest.toml b/packages/JuliaInterpreter/docs/Manifest.toml new file mode 100644 index 0000000..fad80f8 --- /dev/null +++ b/packages/JuliaInterpreter/docs/Manifest.toml @@ -0,0 +1,119 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.8.2" +manifest_format = "2.0" +project_hash = "f8f17ea8030e6083899e9cc89b9b04cd28b94813" + +[[deps.ANSIColoredPrinters]] +git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" +uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9" +version = "0.0.1" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.CodeTracking]] +deps = ["InteractiveUtils", "UUIDs"] +git-tree-sha1 = "cc4bd91eba9cdbbb4df4746124c22c0832a460d6" +uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +version = "1.1.1" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "c36550cb29cbe373e95b3f40486b9a4148f89ffd" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.2" + +[[deps.Documenter]] +deps = ["ANSIColoredPrinters", "Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] +git-tree-sha1 = "6030186b00a38e9d0434518627426570aac2ef95" +uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +version = "0.27.23" + +[[deps.IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "0.2.2" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "3c837543ddb02250ef42f4738347454f95079d4e" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.3" + +[[deps.JuliaInterpreter]] +deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] +path = ".." +uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +version = "0.9.15" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.Parsers]] +deps = ["Dates", "SnoopPrecompile"] +git-tree-sha1 = "cceb0257b662528ecdf0b4b4302eb00e767b38e7" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.5.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.SnoopPrecompile]] +git-tree-sha1 = "f604441450a3c0569830946e5b33b78c928e1a85" +uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" +version = "1.0.1" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/packages/JuliaInterpreter/docs/Project.toml b/packages/JuliaInterpreter/docs/Project.toml new file mode 100644 index 0000000..7ee2981 --- /dev/null +++ b/packages/JuliaInterpreter/docs/Project.toml @@ -0,0 +1,7 @@ +[deps] +CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +JuliaInterpreter = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" diff --git a/packages/JuliaInterpreter/docs/make.jl b/packages/JuliaInterpreter/docs/make.jl new file mode 100644 index 0000000..ded4791 --- /dev/null +++ b/packages/JuliaInterpreter/docs/make.jl @@ -0,0 +1,29 @@ +using Documenter, JuliaInterpreter, Test, CodeTracking + +DocMeta.setdocmeta!(JuliaInterpreter, :DocTestSetup, :( + begin + using JuliaInterpreter + JuliaInterpreter.clear_caches() + JuliaInterpreter.remove() + end); recursive=true) + +makedocs( + modules = [JuliaInterpreter], + clean = false, + strict = true, + format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"), + sitename = "JuliaInterpreter.jl", + authors = "Keno Fischer, Tim Holy, Kristoffer Carlsson, and others", + linkcheck = !("skiplinks" in ARGS), + pages = [ + "Home" => "index.md", + "ast.md", + "internals.md", + "dev_reference.md", + ], +) + +deploydocs( + repo = "github.com/JuliaDebug/JuliaInterpreter.jl.git", + push_preview = true +) diff --git a/packages/JuliaInterpreter/docs/src/ast.md b/packages/JuliaInterpreter/docs/src/ast.md new file mode 100644 index 0000000..b6dd656 --- /dev/null +++ b/packages/JuliaInterpreter/docs/src/ast.md @@ -0,0 +1,176 @@ +!!! note + This page and the next are designed to teach a little more about the internals. + Depending on your interest, you may be able to skip them. + +# Lowered representation + +JuliaInterpreter uses the lowered representation of code. +The key advantage of lowered representation is that it is fairly well circumscribed: + +- There are only a limited number of legal statements that can appear in lowered code +- Each statement is "unpacked" to essentially do one thing +- Scoping of variables is simplified via the slot mechanism, described below +- Names are fully resolved by module +- Macros are expanded + +[Julia AST](https://docs.julialang.org/en/v1/devdocs/ast/) describes the kinds of +objects that can appear in lowered code. + +Let's start with a demonstration on a simple function: + +```julia +function summer(A::AbstractArray{T}) where T + s = zero(T) + for a in A + s += a + end + return s +end + +A = [1, 2, 5] +``` + +To interpret lowered representation, it maybe be useful to rewrite the body of `summer` in the following ways. +First let's use an intermediate representation that expands the `for a in A ... end` loop: + +```julia + s = zero(T) + temp = iterate(A) # `for` loops get lowered to `iterate/while` loops + while temp !== nothing + a, state = temp + s += a + temp = iterate(A, state) + end + return s +``` + +The lowered code takes the additional step of resolving the names by module and turning all the +branching into `@goto/@label` equivalents: + +```julia + # Code starting at line 2 (the first line of the body) + s = Main.zero(T) # T corresponds to the first parameter, i.e., $(Expr(:static_parameter, 1)) + + # Code starting at line 3 + temp = Base.iterate(A) # here temp = @_4 + if temp === nothing # this comparison gets stored as %4, and %5 stores !(temp===nothing) + @goto block4 + end + + @label block2 + ## BEGIN block2 + a, state = temp[1], temp[2] # these correspond to the `getfield` calls, state is %9 + + # Code starting at line 4 + s = s + a + + # Code starting at line 5 + temp = iterate(A, state) # A is also %2 + if temp === nothing + @goto block4 # the `while` condition was false + end + ## END block2 + + @goto block2 # here the `while` condition is still true + + # Code starting at line 6 + @label block4 + ## BEGIN block4 + return s + ## END block4 +``` + +This has very close correspondence to the lowered representation: + +```julia +julia> code = @code_lowered debuginfo=:source summer(A) +CodeInfo( + @ REPL[1]:2 within `summer' +1 ─ s = Main.zero($(Expr(:static_parameter, 1))) +│ @ REPL[1]:3 within `summer' +│ %2 = A +│ @_4 = Base.iterate(%2) +│ %4 = @_4 === nothing +│ %5 = Base.not_int(%4) +└── goto #4 if not %5 +2 ┄ %7 = @_4 +│ a = Core.getfield(%7, 1) +│ %9 = Core.getfield(%7, 2) +│ @ REPL[1]:4 within `summer' +│ s = s + a +│ @_4 = Base.iterate(%2, %9) +│ %12 = @_4 === nothing +│ %13 = Base.not_int(%12) +└── goto #4 if not %13 +3 ─ goto #2 + @ REPL[1]:6 within `summer' +4 ┄ return s +) +``` +!!! note + Not all Julia versions support `debuginfo`. If the command above fails for you, + just omit the `debuginfo=:source` portion. + +To understand this package's internals, you need to familiarize yourself with these +`CodeInfo` objects. +The lines that start with `@ REPL[1]:n` indicate the source line of the succeeding +block of statements; here we defined this method in the REPL, so the source file is `REPL[1]`; +the number after the colon is the line number. + +The numbers on the left correspond to [basic blocks](https://en.wikipedia.org/wiki/Basic_block), +as we annotated with `@label block2` above. +When used in statements these are printed with a hash, e.g., in `goto #4 if not %5`, the +`#4` refers to basic block 4. +The numbers in the next column--e.g., `%2`, refer to +[single static assignment (SSA) values](https://en.wikipedia.org/wiki/Static_single_assignment_form). +Each statement (each line of this printout) corresponds to a single SSA value, +but only those used later in the code are printed using assignment syntax. +Wherever a previous SSA value is used, it's referenced by an `SSAValue` and printed as `%5`; +for example, in `goto #4 if not %5`, the `%5` is the result of evaluating the 5th statement, +which is `(Base.not_int)(%4)`, which in turn refers to the result of statement 4. +Finally, temporary variables here are shown as `@_4`; the `_` indicates a *slot*, either +one of the input arguments or a local variable, and the 4 means the 4th one. +Together lines 4 and 5 correspond to `!(@_4 === nothing)`, where `@_4` has been assigned the +result of the call to `iterate` occurring on line 3. (In some Julia versions, this may be printed as `#temp#`, +similar to how we named it in our alternative implementation above.) + +Let's look at a couple of the fields of the `CodeInfo`. First, the statements themselves: + +```julia +julia> code.code +16-element Array{Any,1}: + :(_3 = Main.zero($(Expr(:static_parameter, 1)))) + :(_2) + :(_4 = Base.iterate(%2)) + :(_4 === nothing) + :(Base.not_int(%4)) + :(unless %5 goto %16) + :(_4) + :(_5 = Core.getfield(%7, 1)) + :(Core.getfield(%7, 2)) + :(_3 = _3 + _5) + :(_4 = Base.iterate(%2, %9)) + :(_4 === nothing) + :(Base.not_int(%12)) + :(unless %13 goto %16) + :(goto %7) + :(return _3) +``` + +You can see directly that the SSA assignments are implicit; they are not directly +present in the statement list. +The most noteworthy change here is the appearance of more objects like `_3`, which are +references that index into local variable slots: + +```julia +julia> code.slotnames +5-element Array{Any,1}: + Symbol("#self#") + :A + :s + Symbol("") + :a +``` + +When printing the whole `CodeInfo` object, these `slotnames` are substituted in +(unless they are empty, as was the case for `@_4` above). diff --git a/packages/JuliaInterpreter/docs/src/dev_reference.md b/packages/JuliaInterpreter/docs/src/dev_reference.md new file mode 100644 index 0000000..071318b --- /dev/null +++ b/packages/JuliaInterpreter/docs/src/dev_reference.md @@ -0,0 +1,117 @@ +# Function reference + +## Running the interpreter + +```@docs +@interpret +``` + +## Frame creation + +```@docs +Frame(mod::Module, ex::Expr) +ExprSplitter +JuliaInterpreter.enter_call +JuliaInterpreter.enter_call_expr +JuliaInterpreter.prepare_frame +JuliaInterpreter.determine_method_for_expr +JuliaInterpreter.prepare_args +JuliaInterpreter.prepare_call +JuliaInterpreter.get_call_framecode +JuliaInterpreter.optimize! +``` + +## Frame traversal + +```@docs +root +leaf +``` + +## Frame execution + +```@docs +JuliaInterpreter.Compiled +JuliaInterpreter.step_expr! +JuliaInterpreter.finish! +JuliaInterpreter.finish_and_return! +JuliaInterpreter.finish_stack! +JuliaInterpreter.get_return +JuliaInterpreter.next_until! +JuliaInterpreter.maybe_next_until! +JuliaInterpreter.through_methoddef_or_done! +JuliaInterpreter.evaluate_call! +JuliaInterpreter.evaluate_foreigncall +JuliaInterpreter.maybe_evaluate_builtin +JuliaInterpreter.next_call! +JuliaInterpreter.maybe_next_call! +JuliaInterpreter.next_line! +JuliaInterpreter.until_line! +JuliaInterpreter.maybe_reset_frame! +JuliaInterpreter.maybe_step_through_wrapper! +JuliaInterpreter.maybe_step_through_kwprep! +JuliaInterpreter.handle_err +JuliaInterpreter.debug_command +``` + +## Breakpoints + +```@docs +@breakpoint +@bp +breakpoint +enable +disable +remove +toggle +break_on +break_off +breakpoints +JuliaInterpreter.dummy_breakpoint +``` + +## Types + +```@docs +Frame +JuliaInterpreter.FrameCode +JuliaInterpreter.FrameData +JuliaInterpreter._INACTIVE_EXCEPTION +JuliaInterpreter.FrameInstance +JuliaInterpreter.BreakpointState +JuliaInterpreter.BreakpointRef +JuliaInterpreter.AbstractBreakpoint +JuliaInterpreter.BreakpointSignature +JuliaInterpreter.BreakpointFileLocation +``` + +## Internal storage + +```@docs +JuliaInterpreter.framedict +JuliaInterpreter.genframedict +JuliaInterpreter.compiled_methods +JuliaInterpreter.compiled_modules +JuliaInterpreter.interpreted_methods +``` + +## Utilities + +```@docs +JuliaInterpreter.eval_code +JuliaInterpreter.@lookup +JuliaInterpreter.is_wrapper_call +JuliaInterpreter.is_doc_expr +JuliaInterpreter.is_global_ref +CodeTracking.whereis +JuliaInterpreter.linenumber +JuliaInterpreter.Variable +JuliaInterpreter.locals +JuliaInterpreter.whichtt +``` + +## Hooks +```@docs +JuliaInterpreter.on_breakpoints_updated +JuliaInterpreter.firehooks +``` diff --git a/packages/JuliaInterpreter/docs/src/index.md b/packages/JuliaInterpreter/docs/src/index.md new file mode 100644 index 0000000..0b67a04 --- /dev/null +++ b/packages/JuliaInterpreter/docs/src/index.md @@ -0,0 +1,220 @@ +# JuliaInterpreter + +This package implements an [interpreter](https://en.wikipedia.org/wiki/Interpreter_(computing)) for Julia code. +Normally, Julia compiles your code when you first execute it; using JuliaInterpreter you can +avoid compilation and execute the expressions that define your code directly. +Interpreters have a number of applications, including support for stepping debuggers. + +## Use as an interpreter + +Using this package as an interpreter is straightforward: + +```jldoctest demo1 +julia> using JuliaInterpreter + +julia> list = [1, 2, 5] +3-element Vector{Int64}: + 1 + 2 + 5 + +julia> sum(list) +8 + +julia> @interpret sum(list) +8 +``` + +## Breakpoints + +You can interrupt execution by setting breakpoints. +You can set breakpoints via packages that explicitly target debugging, +like [Juno](https://junolab.org/), [Debugger](https://github.com/JuliaDebug/Debugger.jl), and +[Rebugger](https://github.com/timholy/Rebugger.jl). +But all of these just leverage the core functionality defined in JuliaInterpreter, +so here we'll illustrate it without using any of these other packages. + +Let's set a conditional breakpoint, to be triggered any time one of the elements in the +argument to `sum` is bigger than 4: + +```jldoctest demo1; filter = r"in Base at .*$" +julia> bp = @breakpoint sum([1, 2]) any(x->x>4, a); +``` + +Note that in writing the condition, we used `a`, the name of the argument to the relevant +method of `sum`. Conditionals should be written using a combination of argument and parameter +names of the method into which you're inserting a breakpoint; you can also use any +globally-available name (as used here with the `any` function). + +Now let's see what happens: + +```jldoctest demo1; filter = [r"in Base at .*$", r"[^\d]\d\d\d[^\d]"] +julia> @interpret sum([1,2,3]) # no element bigger than 4, breakpoint should not trigger +6 + +julia> frame, bpref = @interpret sum([1,2,5]) # should trigger breakpoint +(Frame for sum(a::AbstractArray; dims, kw...) in Base at reducedim.jl:873 +c 1* 873 1 ─ nothing + 2 873 │ %2 = ($(QuoteNode(NamedTuple)))() + 3 873 │ %3 = Base.pairs(%2) +⋮ +a = [1, 2, 5], breakpoint(sum(a::AbstractArray; dims, kw...) in Base at reducedim.jl:873, line 873)) +``` + +`frame` is described in more detail on the next page; for now, suffice it to say +that the `c` in the leftmost column indicates the presence of a conditional breakpoint +upon entry to `sum`. `bpref` is a reference to the breakpoint of type [`BreakpointRef`](@ref). +The breakpoint `bp` we created can be manipulated at the command line + +```jldoctest demo1; filter = [r"in Base at .*$", r"[^\d]\d\d\d[^\d]"] +julia> disable(bp) + +julia> @interpret sum([1,2,5]) +8 + +julia> enable(bp) + +julia> @interpret sum([1,2,5]) +(Frame for sum(a::AbstractArray; dims, kw...) in Base at reducedim.jl:873 +c 1* 873 1 ─ nothing + 2 873 │ %2 = ($(QuoteNode(NamedTuple)))() + 3 873 │ %3 = Base.pairs(%2) +⋮ +a = [1, 2, 5], breakpoint(sum(a::AbstractArray; dims, kw...) in Base at reducedim.jl:873, line 873)) +``` + +[`disable`](@ref) and [`enable`](@ref) allow you to turn breakpoints off and on without losing any +conditional statements you may have provided; [`remove`](@ref) allows a permanent removal of +the breakpoint. You can use `remove()` to remove all breakpoints in all methods. + +[`@breakpoint`](@ref) allows you to optionally specify a line number at which the breakpoint +is to be set. You can also use a functional form, [`breakpoint`](@ref), to specify file/line +combinations or that you want to break on entry to *any* method of a particular function. +At present, note that some of this functionality requires that you be running +[Revise.jl](https://github.com/timholy/Revise.jl). + +It is, in addition, possible to halt execution when otherwise an error would be thrown. +This functionality is enabled using [`break_on`](@ref) and disabled with [`break_off`](@ref): + +```jldoctest demo1; filter=r"none:\d" +julia> function f_outer() + println("before error") + f_inner() + println("after error") + end; + +julia> f_inner() = error("inner error"); + +julia> break_on(:error) + +julia> fr, pc = @interpret f_outer() +before error +(Frame for f_outer() in Main at none:1 + 1 2 1 ─ Base.println("before error") + 2* 3 │ f_inner() + 3 4 │ %3 = Base.println("after error") + 4 4 └── return %3 +callee: f_inner() in Main at none:1, breakpoint(error(s::AbstractString) in Base at error.jl:35, line 35, ErrorException("inner error"))) + +julia> leaf(fr) +Frame for error(s::AbstractString) in Base at error.jl:35 + 1 35 1 ─ %1 = ($(QuoteNode(ErrorException)))(s) + 2* 35 │ %2 = Core.throw(%1) + 3 35 └── return %2 +s = "inner error" +caller: f_inner() in Main at none:1 + +julia> typeof(pc) +BreakpointRef + +julia> pc.err +ErrorException("inner error") + +julia> break_off(:error) + +julia> @interpret f_outer() +before error +ERROR: inner error +Stacktrace: +[...] +``` + +Finally, you can set breakpoints using [`@bp`](@ref): + +```jldoctest demo1; filter=r"none:\d" +julia> function myfunction(x, y) + a = 1 + b = 2 + x > 3 && @bp + return a + b + x + y + end +myfunction (generic function with 1 method) + +julia> @interpret myfunction(1, 2) +6 + +julia> @interpret myfunction(5, 6) +(Frame for myfunction(x, y) in Main at none:1 +⋮ + 3 4 │ %3 = x > 3 + 4 4 └── goto #3 if not %3 +b 5* 4 2 ─ nothing + 6 4 └── goto #3 + 7 5 3 ┄ %7 = a + b + x + y +⋮ +x = 5 +y = 6 +b = 2 +a = 1, breakpoint(myfunction(x, y) in Main at none:1, line 4)) +``` + +Here the breakpoint is marked with a `b` indicating that it is an unconditional breakpoint. +Because we placed it inside the condition `x > 3`, we've achieved a conditional outcome. + +When using `@bp` in source-code files, the use of Revise is recommended, +since it allows you to add breakpoints, test code, and then remove the breakpoints from the +code without restarting Julia. + +## `debug_command` + +You can control execution of frames via [`debug_command`](@ref). +Authors of debugging applications should target `debug_command` for their interaction +with JuliaInterpreter. + +## Hooks +Consider if you were building a debugging application with a GUI component which displays a dot in the text editor margin where a breakpoint is. +If a user creates a breakpoint not via your GUI, but rather via a command in the REPL etc. +then you still wish to keep your GUI up to date. +How to do this? The answer is hooks. + + +JuliaInterpreter has experimental support for having a hook, or callback function invoked +whenever the set of all breakpoints is changed. +Hook functions are setup by invoking the [`JuliaInterpreter.on_breakpoints_updated`](@ref) function. + +To return to our example of keeping GUI up to date, the hooks would look something like this: +```julia +using JuliaInterpreter +using JuliaInterpreter: AbstractBreakpoint, update_states!, on_breakpoints_updated + +breakpoint_gui_elements = Dict{AbstractBreakpoint, MarginDot}() +# ... +function breakpoint_gui_hook(::typeof(breakpoint), bp::AbstractBreakpoint) + bp_dot = MarginDot(bp) + draw(bp_dot) + breakpoint_gui_elements[bp] = bp_dot +end + +function breakpoint_gui_hook(::typeof(remove), bp::AbstractBreakpoint) + bp_dot = pop!(breakpoint_gui_elements, bp) + undraw(bp_dot) +end + +function breakpoint_gui_hook(::typeof(update_states!), bp::AbstractBreakpoint) + is_enabled = bp.enabled[] + bp_dot = breakpoint_gui_elements[bp] + set_fill!(bp_dot, is_enabled ? :blue : :grey) +end + +on_breakpoints_updated(breakpoint_gui_hook) +``` diff --git a/packages/JuliaInterpreter/docs/src/internals.md b/packages/JuliaInterpreter/docs/src/internals.md new file mode 100644 index 0000000..32a8473 --- /dev/null +++ b/packages/JuliaInterpreter/docs/src/internals.md @@ -0,0 +1,256 @@ +# Internals + +## Basic usage + +The process of executing code in the interpreter is to prepare a `frame` and then +evaluate these statements one-by-one, branching via the `goto` statements as appropriate. +Using the `summer` example described in [Lowered representation](@ref), +let's build a frame: + +```julia +julia> frame = JuliaInterpreter.enter_call(summer, A) +Frame for summer(A::AbstractArray{T,N} where N) where T in Main at REPL[2]:2 + 1* 2 1 ─ s = (zero)($(Expr(:static_parameter, 1))) + 2 3 │ %2 = A + 3 3 │ #temp# = (iterate)(%2) +⋮ +A = [1, 2, 5] +T = Int64 +``` + +This is a [`Frame`](@ref). Only a portion of the `CodeInfo` is shown, a small region surrounding +the current statement (marked with `*` or in yellow text). The full `CodeInfo` can be extracted +as `code = frame.framecode.src`. (It's a slightly modified form of one returned by `@code_lowered`, +in that it has been processed by [`JuliaInterpreter.optimize!`](@ref) to speed up run-time execution.) + +`frame` has another field, `framedata`, that holds values needed for or generated by execution. +The input arguments and local variables are in `locals`: + +```julia +julia> frame.framedata.locals +5-element Array{Union{Nothing, Some{Any}},1}: + Some(summer) + Some([1, 2, 5]) + nothing + nothing + nothing +``` + +These correspond to the `code.slotnames`; the first is the `#self#` argument and the second +is the input array. The remaining local variables (e.g., `s` and `a`), have not yet been assigned---we've +only built the frame, but we haven't yet begun to execute it. +The static parameter, `T`, is stored in `frame.framedata.sparams`: + +```julia +julia> frame.framedata.sparams +1-element Array{Any,1}: + Int64 +``` + +The `Expr(:static_parameter, 1)` statement refers to this value. + +The other main storage is for the generated SSA values: + +```julia +julia> frame.framedata.ssavalues +16-element Array{Any,1}: + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef +``` + +Since we haven't executed any statements yet, these are all undefined. + +The other main entity is the so-called [program counter](https://en.wikipedia.org/wiki/Program_counter), +which just indicates the next statement to be executed: + +```julia +julia> frame.pc +1 +``` + +Let's try executing the first statement: + +```julia +julia> JuliaInterpreter.step_expr!(frame) +2 +``` + +This indicates that it ran statement 1 and is prepared to run statement 2. +(It's worth noting that the first line included a `call` to `zero`, so behind the scenes +JuliaInterpreter created a new frame for `zero`, executed all the statements, and then popped +back to `frame`.) +Since the first statement is an assignment of a local variable, let's check the +locals again: + +```julia +julia> frame.framedata.locals +5-element Array{Union{Nothing, Some{Any}},1}: + Some(summer) + Some([1, 2, 5]) + Some(0) + nothing + nothing +``` + +You can see that the entry corresponding to `s` has been initialized. + +The next statement just retrieves one of the slots (the input argument `A`) and stores +it in an SSA value: + +```julia +julia> JuliaInterpreter.step_expr!(frame) +3 + +julia> frame.framedata.ssavalues +16-element Array{Any,1}: + #undef + [1, 2, 5] + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef +``` + +One can easily continue this until execution completes, which is indicated when `step_expr!` +returns `nothing`. Alternatively, use the higher-level `JuliaInterpreter.finish!(frame)` +to step through the entire frame, +or `JuliaInterpreter.finish_and_return!(frame)` to also obtain the return value. + +## More complex expressions + +Sometimes you might have a whole sequence of expressions you want to run. +In such cases, your first thought should be to construct the `Frame` manually. +Here's a demonstration: + +```jldoctest; setup=(using JuliaInterpreter; JuliaInterpreter.clear_caches()) +using Test + +ex = quote + x, y = 1, 2 + @test x + y == 3 +end + +frame = Frame(Main, ex) +JuliaInterpreter.finish_and_return!(frame) + +# output + +Test Passed +``` + +## Toplevel code and world age + +Code that defines new `struct`s, new methods, or new modules is a bit more complicated +and requires special handling. In such cases, calling `finish_and_return!` on a frame that +defines these new objects and then calls them can trigger a +[world age error](https://docs.julialang.org/en/v1/manual/methods/#Redefining-Methods-1), +in which the method is considered to be too new to be run by the currently compiled code. +While one can resolve this by using `Base.invokelatest`, we'd have to use that strategy +throughout the entire package. This would cause a major reduction in performance. +To resolve this issue without leading to performance problems, care is required to +return to "top level" after defining such objects. This leads to altered syntax for executing +such expressions. + +Here's a demonstration of the problem: + +```julia +ex = :(map(x->x^2, [1, 2, 3])) +frame = Frame(Main, ex) +julia> JuliaInterpreter.finish_and_return!(frame) +ERROR: this frame needs to be run a top level +``` + +The reason for this error becomes clearer if we examine `frame` or look directly at the lowered code: + +```julia +julia> Meta.lower(Main, ex) +:($(Expr(:thunk, CodeInfo( + @ none within `top-level scope` +1 ─ $(Expr(:thunk, CodeInfo( + @ none within `top-level scope` +1 ─ global var"#3#4" +│ const var"#3#4" +│ %3 = Core._structtype(Main, Symbol("#3#4"), Core.svec(), Core.svec(), Core.svec(), false, 0) +│ var"#3#4" = %3 +│ Core._setsuper!(var"#3#4", Core.Function) +│ Core._typebody!(var"#3#4", Core.svec()) +└── return nothing +))) +│ %2 = Core.svec(var"#3#4", Core.Any) +│ %3 = Core.svec() +│ %4 = Core.svec(%2, %3, $(QuoteNode(:(#= REPL[18]:1 =#)))) +│ $(Expr(:method, false, :(%4), CodeInfo( + @ REPL[18]:1 within `none` +1 ─ %1 = Core.apply_type(Base.Val, 2) +│ %2 = (%1)() +│ %3 = Base.literal_pow(^, x, %2) +└── return %3 +))) +│ #3 = %new(var"#3#4") +│ %7 = #3 +│ %8 = Base.vect(1, 2, 3) +│ %9 = map(%7, %8) +└── return %9 +)))) +``` + +All of the code before the `%7` line is devoted to defining the anonymous function `x->x^2`: +it creates a new "anonymous type" (here written as `var"#3#4"`), and then defines a "call +function" for this type, equivalent to `(var"#3#4")(x) = x^2`. + +In some cases one can fix this simply by indicating that we want to run this frame at top level: + +```julia +julia> JuliaInterpreter.finish_and_return!(frame, true) +3-element Array{Int64,1}: + 1 + 4 + 9 +``` + +In other cases, such as nested calls of new methods, you may need to allow the world age to update +between evaluations. In such cases you want to use `ExprSplitter`: + +```julia +for (mod, e) in ExprSplitter(Main, ex) + frame = Frame(mod, e) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + JuliaInterpreter.get_return(frame) +end +``` + +This splits the expression into a sequence of frames (here just one, but more complex blocks may be split up into many). +Then, each frame is executed until it finishes defining a new method, then returns to top level. +The return to top level causes an update in the world age. +If the frame hasn't been finished yet (if the return value wasn't `nothing`), +this continues executing where it left off. + +(Incidentally, `JuliaInterpreter.enter_call(map, x->x^2, [1, 2, 3])` works fine on its own, +because the anonymous function is defined by the caller---you'll see that the created frame +is very simple.) diff --git a/packages/JuliaInterpreter/src/JuliaInterpreter.jl b/packages/JuliaInterpreter/src/JuliaInterpreter.jl new file mode 100644 index 0000000..c781f46 --- /dev/null +++ b/packages/JuliaInterpreter/src/JuliaInterpreter.jl @@ -0,0 +1,16 @@ +module JuliaInterpreter + +# We use a code structure where all `using` and `import` +# statements in the package that load anything other than +# a Julia base or stdlib package are located in this file here. +# Nothing else should appear in this file here, apart from +# the `include("packagedef.jl")` statement, which loads what +# we would normally consider the bulk of the package code. +# This somewhat unusual structure is in place to support +# the VS Code extension integration. + +using CodeTracking + +include("packagedef.jl") + +end # module diff --git a/packages/JuliaInterpreter/src/breakpoints.jl b/packages/JuliaInterpreter/src/breakpoints.jl new file mode 100644 index 0000000..1ea011b --- /dev/null +++ b/packages/JuliaInterpreter/src/breakpoints.jl @@ -0,0 +1,438 @@ +using Base: Callable + +const _breakpoints = AbstractBreakpoint[] + +""" + breakpoints()::Vector{AbstractBreakpoint} + +Return an array with all breakpoints. +""" +breakpoints() = _breakpoints + + +const breakpoint_update_hooks = [] +""" + on_breakpoints_updated(f) + +Register a one-argument function to be called after any update to the set of all +breakpoints. This includes their creation, deletion, enabling and disabling. + +The function `f` should take two inputs: + - First argument is the function doing to update, this is provided to allow to dispatch + on its type. It will be one: + - `::typeof(breakpoint)` for the creation, + - `::typeof(remove)` for the deletion. + - `::typeof(update_states)` for disable/enable/toggleing + - Second argument is the breakpoint object that was changed. + +If only desiring to handle some kinds of update, `f` should have fallback methods +to do nothing in the general case. + +!!! warning + This feature is experimental, and may be modified or removed in a minor release. +""" +on_breakpoints_updated(f) = push!(breakpoint_update_hooks, f) + + +""" + firehooks(hooked_fun, bp::AbstractBreakpoint) + +Trigger all hooks that were registered with [`on_breakpoints_updated`](@ref), +passing them the `hooked_fun` and the `bp`. +This should be called whenever the set of breakpoints is updated. +`hooked_fun` is the function doing the update, and `bp` is the relevent breakpoint being +updated _after_ the update is applied. + +!!! warning + This feature is experimental, and may be modified or removed in a minor release. +""" +function firehooks(hooked_fun, bp::AbstractBreakpoint) + for hook in breakpoint_update_hooks + try + hook(hooked_fun, bp) + catch err + @warn "Hook function errored" hook hooked_fun bp exception=err + end + end +end + +function add_to_existing_framecodes(bp::AbstractBreakpoint) + for framecode in values(framedict) + add_breakpoint_if_match!(framecode, bp) + end +end + +function add_breakpoint_if_match!(framecode::FrameCode, bp::BreakpointSignature) + if framecode_matches_breakpoint(framecode, bp) + scope = framecode.scope + matching_file = if scope isa Method + scope.file + else + # TODO: make more precise? + first(framecode.src.linetable).file + end + stmtidxs = bp.line === 0 ? [1] : statementnumbers(framecode, bp.line, matching_file::Symbol) + stmtidxs === nothing && return + breakpoint!(framecode, stmtidxs, bp.condition, bp.enabled[]) + foreach(stmtidx -> push!(bp.instances, BreakpointRef(framecode, stmtidx)), stmtidxs) + return + end +end + +function framecode_matches_breakpoint(framecode::FrameCode, bp::BreakpointSignature) + function extract_function_from_method(m::Method) + sig = Base.unwrap_unionall(m.sig) + ft0 = sig.parameters[1] + ft = Base.unwrap_unionall(ft0) + if ft <: Function && isa(ft, DataType) && isdefined(ft, :instance) + return ft.instance + elseif isa(ft, DataType) && ft.name === Type.body.name + return ft.parameters[1] + else + return ft + end + end + + meth = framecode.scope + meth isa Method || return false + bp.f isa Method && return meth === bp.f + f = extract_function_from_method(meth) + if !(bp.f === f || Core.kwfunc(bp.f) === f) + return false + end + bp.sig === nothing && return true + return bp.sig <: meth.sig +end + +""" + breakpoint(f, [sig], [line], [condition]) + +Add a breakpoint to `f` with the specified argument types `sig`.¨ +If `sig` is not given, the breakpoint will apply to all methods of `f`. +If `f` is a method, the breakpoint will only apply to that method. +Optionally specify an absolute line number `line` in the source file; the default +is to break upon entry at the first line of the body. +Without `condition`, the breakpoint will be triggered every time it is encountered; +the second only if `condition` evaluates to `true`. +`condition` should be written in terms of the arguments and local variables of `f`. + +# Example +```julia +function radius2(x, y) + return x^2 + y^2 +end + +breakpoint(radius2, Tuple{Int,Int}, :(y > x)) +``` +""" +function breakpoint(f::Union{Method, Callable}, sig=nothing, line::Integer=0, condition::Condition=nothing) + if sig !== nothing && f isa Callable + sig = Base.to_tuple_type(sig) + sig = Tuple{_Typeof(f), sig.parameters...} + end + bp = BreakpointSignature(f, sig, line, condition, Ref(true), BreakpointRef[]) + add_to_existing_framecodes(bp) + idx = findfirst(bp2 -> same_location(bp, bp2), _breakpoints) + if idx === nothing # creating new + push!(_breakpoints, bp) + else #Replace existing breakpoint + old_bp = _breakpoints[idx] + _breakpoints[idx] = bp + firehooks(remove, old_bp) + end + firehooks(breakpoint, bp) + return bp +end +breakpoint(f::Union{Method, Callable}, sig, condition::Condition) = breakpoint(f, sig, 0, condition) +breakpoint(f::Union{Method, Callable}, line::Integer, condition::Condition=nothing) = breakpoint(f, nothing, line, condition) +breakpoint(f::Union{Method, Callable}, condition::Condition) = breakpoint(f, nothing, 0, condition) + + +""" + breakpoint(file, line, [condition]) + +Set a breakpoint in `file` at `line`. The argument `file` can be a filename, a partial path or absolute path. +For example, `file = foo.jl` will match against all files with the name `foo.jl`, +`file = src/foo.jl` will match against all paths containing `src/foo.jl`, e.g. both `Foo/src/foo.jl` and `Bar/src/foo.jl`. +Absolute paths only matches against the file with that exact absolute path. +""" +function breakpoint(file::AbstractString, line::Integer, condition::Condition=nothing) + file = normpath(file) + apath = CodeTracking.maybe_fix_path(abspath(file)) + ispath(apath) && (apath = realpath(apath)) + bp = BreakpointFileLocation(file, apath, line, condition, Ref(true), BreakpointRef[]) + add_to_existing_framecodes(bp) + idx = findfirst(bp2 -> same_location(bp, bp2), _breakpoints) + idx === nothing ? push!(_breakpoints, bp) : (_breakpoints[idx] = bp) + firehooks(breakpoint, bp) + return bp +end + +function add_breakpoint_if_match!(framecode::FrameCode, bp::BreakpointFileLocation) + framecode_contains_file = false + matching_file = nothing + for file in framecode.unique_files + filepath = CodeTracking.maybe_fix_path(String(file)) + if Base.samefile(bp.abspath, filepath) || endswith(filepath, bp.path) + framecode_contains_file = true + matching_file = file + break + end + end + framecode_contains_file || return nothing + + stmtidxs = bp.line === 0 ? [1] : statementnumbers(framecode, bp.line, matching_file::Symbol) + stmtidxs === nothing && return + breakpoint!(framecode, stmtidxs, bp.condition, bp.enabled[]) + foreach(stmtidx -> push!(bp.instances, BreakpointRef(framecode, stmtidx)), stmtidxs) + return +end + +function shouldbreak(frame::Frame, pc::Int) + bps = frame.framecode.breakpoints + isassigned(bps, pc) || return false + bp = bps[pc] + bp.isactive || return false + return Base.invokelatest(bp.condition, frame)::Bool +end + +function prepare_slotfunction(framecode::FrameCode, body::Union{Symbol,Expr}) + framename, dataname = gensym("frame"), gensym("data") + assignments = Expr[:($dataname = $framename.framedata)] + default = Unassigned() + for slotname in unique(framecode.src.slotnames) + list = framecode.slotnamelists[slotname] + if length(list) == 1 + maxexpr = :($dataname.last_reference[$(list[1])] > 0 ? $(list[1]) : 0) + else + maxcounter, maxidx = gensym("maxcounter"), gensym("maxidx") + maxexpr = quote + begin + $maxcounter, $maxidx = 0, 0 + for l in $list + counter = $dataname.last_reference[l] + if counter > $maxcounter + $maxcounter, $maxidx = counter, l + end + end + $maxidx + end + end + end + maxexsym = gensym("slotid") + push!(assignments, :($maxexsym = $maxexpr)) + push!(assignments, :($slotname = $maxexsym > 0 ? something($dataname.locals[$maxexsym]) : $default)) + end + scope = framecode.scope + if isa(scope, Method) + syms = sparam_syms(scope) + for i = 1:length(syms) + push!(assignments, Expr(:(=), syms[i], :($dataname.sparams[$i]))) + end + end + funcname = isa(scope, Method) ? gensym("slotfunction") : gensym(Symbol(scope, "_slotfunction")) + return Expr(:function, Expr(:call, funcname, framename), Expr(:block, assignments..., body)) +end + +_unpack(condition) = isa(condition, Expr) ? (Main, condition) : condition + +## The fundamental implementations of breakpoint-setting +function breakpoint!(framecode::FrameCode, pc, condition::Condition=nothing, enabled=true) + stmtidx = pc + if condition === nothing + framecode.breakpoints[stmtidx] = BreakpointState(enabled) + else + mod, cond = _unpack(condition) + fex = prepare_slotfunction(framecode, cond) + framecode.breakpoints[stmtidx] = BreakpointState(enabled, Core.eval(mod, fex)) + end +end +breakpoint!(framecode::FrameCode, pcs::AbstractArray, condition::Condition=nothing, enabled=true) = + foreach(pc -> breakpoint!(framecode, pc, condition, enabled), pcs) +breakpoint!(frame::Frame, pc=frame.pc, condition::Condition=nothing) = + breakpoint!(frame.framecode, pc, condition) + +function update_states!(bp::AbstractBreakpoint) + foreach(bpref -> update_state!(bpref, bp.enabled[]), bp.instances) + firehooks(update_states!, bp) +end +update_state!(bp::BreakpointRef, v::Bool) = bp[] = v + +""" + enable(bp::AbstractBreakpoint) + +Enable breakpoint `bp`. +""" +enable(bp::AbstractBreakpoint) = (bp.enabled[] = true; update_states!(bp)) +enable(bp::BreakpointRef) = update_state!(bp, true) + + +""" + disable(bp::AbstractBreakpoint) + +Disable breakpoint `bp`. Disabled breakpoints can be re-enabled with [`enable`](@ref). +""" +disable(bp::AbstractBreakpoint) = (bp.enabled[] = false; update_states!(bp)) +disable(bp::BreakpointRef) = update_state!(bp, false) + +""" + remove(bp::AbstractBreakpoint) + +Remove (delete) breakpoint `bp`. Removed breakpoints cannot be re-enabled. +""" +function remove(bp::AbstractBreakpoint) + idx = findfirst(isequal(bp), _breakpoints) + if idx !== nothing + bp = _breakpoints[idx] + deleteat!(_breakpoints, idx) + firehooks(remove, bp) + end + foreach(remove, bp.instances) +end +function remove(bp::BreakpointRef) + bp.framecode.breakpoints[bp.stmtidx] = BreakpointState(false, falsecondition) + return nothing +end + +""" + toggle(bp::AbstractBreakpoint) + +Toggle breakpoint `bp`. +""" +toggle(bp::AbstractBreakpoint) = (bp.enabled[] = !bp.enabled[]; update_states!(bp)) +toggle(bp::BreakpointRef) = update_state!(bp, !bp[].isactive) + +""" + enable() + +Enable all breakpoints. +""" +enable() = foreach(enable, _breakpoints) + +""" + disable() + +Disable all breakpoints. +""" +disable() = foreach(disable, _breakpoints) + +""" + remove() + +Remove all breakpoints. +""" +function remove() + for bp in _breakpoints + foreach(remove, bp.instances) + end + empty!(_breakpoints) +end + +""" + break_on(states...) + +Turn on automatic breakpoints when any of the conditions described in `states` occurs. +The supported states are: + +- `:error`: trigger a breakpoint any time an uncaught exception is thrown +- `:throw` : trigger a breakpoint any time a throw is executed (even if it will eventually be caught) +""" +function break_on(states::Vararg{Symbol}) + for state in states + if state === :error + break_on_error[] = true + elseif state === :throw + break_on_throw[] = true + else + throw(ArgumentError(string("unsupported state :", state))) + end + end +end + +""" + break_off(states...) + +Turn off automatic breakpoints when any of the conditions described in `states` occurs. +See [`break_on`](@ref) for a description of valid states. +""" +function break_off(states::Vararg{Symbol}) + for state in states + if state === :error + break_on_error[] = false + elseif state === :throw + break_on_throw[] = false + else + throw(ArgumentError(string("unsupported state :", state))) + end + end +end + + +""" + @breakpoint f(args...) condition=nothing + @breakpoint f(args...) line condition=nothing + +Break upon entry, or at the specified line number, in the method called by `f(args...)`. +Optionally supply a condition expressed in terms of the arguments and internal variables +of the method. +If `line` is supplied, it must be a literal integer. + +# Example + +Suppose a method `mysum` is defined as follows, where the numbers to the left are the line +number in the file: + +``` +12 function mysum(A) +13 s = zero(eltype(A)) +14 for a in A +15 s += a +16 end +17 return s +18 end +``` + +Then + +``` +@breakpoint mysum(A) 15 s>10 +``` + +would cause execution of the loop to break whenever `s>10`. +""" +macro breakpoint(call_expr, args...) + whichexpr = InteractiveUtils.gen_call_with_extracted_types(__module__, :which, call_expr) + haveline, line, condition = false, 0, nothing + while !isempty(args) + arg = first(args) + if isa(arg, Integer) + haveline, line = true, arg + else + condition = arg + end + args = Base.tail(args) + end + condexpr = condition === nothing ? nothing : esc(Expr(:quote, condition)) + if haveline + return quote + local method = $whichexpr + $breakpoint(method, $line, $condexpr) + end + else + return quote + local method = $whichexpr + $breakpoint(method, $condexpr) + end + end +end + +const __BREAKPOINT_MARKER__ = nothing + +""" + @bp + +Insert a breakpoint at a location in the source code. +""" +macro bp() + return esc(:($(JuliaInterpreter).__BREAKPOINT_MARKER__)) +end diff --git a/packages/JuliaInterpreter/src/builtins.jl b/packages/JuliaInterpreter/src/builtins.jl new file mode 100644 index 0000000..90fb6ad --- /dev/null +++ b/packages/JuliaInterpreter/src/builtins.jl @@ -0,0 +1,360 @@ +# This file is generated by `generate_builtins.jl`. Do not edit by hand. + +function getargs(args, frame) + nargs = length(args)-1 # skip f + callargs = resize!(frame.framedata.callargs, nargs) + for i = 1:nargs + callargs[i] = @lookup(frame, args[i+1]) + end + return callargs +end + +const kwinvoke = Core.kwfunc(Core.invoke) + +function maybe_recurse_expanded_builtin(frame, new_expr) + f = new_expr.args[1] + if isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction) + return maybe_evaluate_builtin(frame, new_expr, true) + else + return new_expr + end +end + +""" + ret = maybe_evaluate_builtin(frame, call_expr, expand::Bool) + +If `call_expr` is to a builtin function, evaluate it, returning the result inside a `Some` wrapper. +Otherwise, return `call_expr`. + +If `expand` is true, `Core._apply_iterate` calls will be resolved as a call to the applied function. +""" +function maybe_evaluate_builtin(frame, call_expr, expand::Bool) + args = call_expr.args + nargs = length(args) - 1 + fex = args[1] + if isa(fex, QuoteNode) + f = fex.value + else + f = @lookup(frame, fex) + end + + @static if isdefined(Core, :OpaqueClosure) + if f isa Core.OpaqueClosure + return Some{Any}(f(args...)) + end + end + if !(isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction)) + return call_expr + end + # By having each call appearing statically in the "switch" block below, + # each gets call-site optimized. + if f === <: + if nargs == 2 + return Some{Any}(<:(@lookup(frame, args[2]), @lookup(frame, args[3]))) + else + return Some{Any}(<:(getargs(args, frame)...)) + end + elseif f === === + if nargs == 2 + return Some{Any}(===(@lookup(frame, args[2]), @lookup(frame, args[3]))) + else + return Some{Any}(===(getargs(args, frame)...)) + end + elseif f === Core._abstracttype + return Some{Any}(Core._abstracttype(getargs(args, frame)...)) + elseif f === Core._apply_iterate + argswrapped = getargs(args, frame) + if !expand + return Some{Any}(Core._apply_iterate(argswrapped...)) + end + aw1 = argswrapped[1]::Function + @assert aw1 === Core.iterate || aw1 === Core.Compiler.iterate || aw1 === Base.iterate "cannot handle `_apply_iterate` with non iterate as first argument, got $(aw1), $(typeof(aw1))" + new_expr = Expr(:call, argswrapped[2]) + popfirst!(argswrapped) # pop the iterate + popfirst!(argswrapped) # pop the function + argsflat = append_any(argswrapped...) + for x in argsflat + push!(new_expr.args, QuoteNode(x)) + end + return maybe_recurse_expanded_builtin(frame, new_expr) + elseif f === Core._apply_pure + return Some{Any}(Core._apply_pure(getargs(args, frame)...)) + elseif f === Core._call_in_world + return Some{Any}(Core._call_in_world(getargs(args, frame)...)) + elseif @static isdefined(Core, :_call_in_world_total) && f === Core._call_in_world_total + return Some{Any}(Core._call_in_world_total(getargs(args, frame)...)) + elseif f === Core._call_latest + args = getargs(args, frame) + if !expand + return Some{Any}(Core._call_latest(args...)) + end + new_expr = Expr(:call, args[1]) + popfirst!(args) + for x in args + push!(new_expr.args, QuoteNode(x)) + end + return maybe_recurse_expanded_builtin(frame, new_expr) + elseif f === Core._call_latest + return Some{Any}(Core._call_latest(getargs(args, frame)...)) + elseif @static isdefined(Core, :_compute_sparams) && f === Core._compute_sparams + return Some{Any}(Core._compute_sparams(getargs(args, frame)...)) + elseif f === Core._equiv_typedef + return Some{Any}(Core._equiv_typedef(getargs(args, frame)...)) + elseif f === Core._expr + return Some{Any}(Core._expr(getargs(args, frame)...)) + elseif f === Core._primitivetype + return Some{Any}(Core._primitivetype(getargs(args, frame)...)) + elseif f === Core._setsuper! + return Some{Any}(Core._setsuper!(getargs(args, frame)...)) + elseif f === Core._structtype + return Some{Any}(Core._structtype(getargs(args, frame)...)) + elseif @static isdefined(Core, :_svec_ref) && f === Core._svec_ref + return Some{Any}(Core._svec_ref(getargs(args, frame)...)) + elseif f === Core._typebody! + return Some{Any}(Core._typebody!(getargs(args, frame)...)) + elseif f === Core._typevar + if nargs == 3 + return Some{Any}(Core._typevar(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + else + return Some{Any}(Core._typevar(getargs(args, frame)...)) + end + elseif f === Core.apply_type + return Some{Any}(Core.apply_type(getargs(args, frame)...)) + elseif f === Core.arrayref + if nargs == 3 + return Some{Any}(Core.arrayref(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + elseif nargs == 4 + return Some{Any}(Core.arrayref(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]))) + elseif nargs == 5 + return Some{Any}(Core.arrayref(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6]))) + else + return Some{Any}(Core.arrayref(getargs(args, frame)...)) + end + elseif f === Core.arrayset + if nargs == 4 + return Some{Any}(Core.arrayset(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]))) + elseif nargs == 5 + return Some{Any}(Core.arrayset(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6]))) + elseif nargs == 6 + return Some{Any}(Core.arrayset(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6]), @lookup(frame, args[7]))) + else + return Some{Any}(Core.arrayset(getargs(args, frame)...)) + end + elseif f === Core.arraysize + if nargs == 2 + return Some{Any}(Core.arraysize(@lookup(frame, args[2]), @lookup(frame, args[3]))) + else + return Some{Any}(Core.arraysize(getargs(args, frame)...)) + end + elseif @static isdefined(Core, :compilerbarrier) && f === Core.compilerbarrier + if nargs == 2 + return Some{Any}(Core.compilerbarrier(@lookup(frame, args[2]), @lookup(frame, args[3]))) + else + return Some{Any}(Core.compilerbarrier(getargs(args, frame)...)) + end + elseif f === Core.const_arrayref + return Some{Any}(Core.const_arrayref(getargs(args, frame)...)) + elseif @static isdefined(Core, :donotdelete) && f === Core.donotdelete + return Some{Any}(Core.donotdelete(getargs(args, frame)...)) + elseif @static isdefined(Core, :finalizer) && f === Core.finalizer + if nargs == 2 + return Some{Any}(Core.finalizer(@lookup(frame, args[2]), @lookup(frame, args[3]))) + elseif nargs == 3 + return Some{Any}(Core.finalizer(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + elseif nargs == 4 + return Some{Any}(Core.finalizer(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]))) + else + return Some{Any}(Core.finalizer(getargs(args, frame)...)) + end + elseif @static isdefined(Core, :get_binding_type) && f === Core.get_binding_type + if nargs == 2 + return Some{Any}(Core.get_binding_type(@lookup(frame, args[2]), @lookup(frame, args[3]))) + else + return Some{Any}(Core.get_binding_type(getargs(args, frame)...)) + end + elseif f === Core.ifelse + if nargs == 3 + return Some{Any}(Core.ifelse(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + else + return Some{Any}(Core.ifelse(getargs(args, frame)...)) + end + elseif @static isdefined(Core, :set_binding_type!) && f === Core.set_binding_type! + return Some{Any}(Core.set_binding_type!(getargs(args, frame)...)) + elseif f === Core.sizeof + if nargs == 1 + return Some{Any}(Core.sizeof(@lookup(frame, args[2]))) + else + return Some{Any}(Core.sizeof(getargs(args, frame)...)) + end + elseif f === Core.svec + return Some{Any}(Core.svec(getargs(args, frame)...)) + elseif f === applicable + return Some{Any}(applicable(getargs(args, frame)...)) + elseif f === fieldtype + if nargs == 2 + return Some{Any}(fieldtype(@lookup(frame, args[2]), @lookup(frame, args[3]))::Type) + elseif nargs == 3 + return Some{Any}(fieldtype(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))::Type) + else + return Some{Any}(fieldtype(getargs(args, frame)...)::Type) + end + elseif f === getfield + if nargs == 2 + return Some{Any}(getfield(@lookup(frame, args[2]), @lookup(frame, args[3]))) + elseif nargs == 3 + return Some{Any}(getfield(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + elseif nargs == 4 + return Some{Any}(getfield(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]))) + else + return Some{Any}(getfield(getargs(args, frame)...)) + end + elseif @static isdefined(Core, :getglobal) && f === getglobal + if nargs == 2 + return Some{Any}(getglobal(@lookup(frame, args[2]), @lookup(frame, args[3]))) + elseif nargs == 3 + return Some{Any}(getglobal(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + else + return Some{Any}(getglobal(getargs(args, frame)...)) + end + elseif f === invoke + if !expand + argswrapped = getargs(args, frame) + return Some{Any}(invoke(argswrapped...)) + end + # This uses the original arguments to avoid looking them up twice + # See #442 + return Expr(:call, invoke, args[2:end]...) + elseif f === isa + if nargs == 2 + return Some{Any}(isa(@lookup(frame, args[2]), @lookup(frame, args[3]))) + else + return Some{Any}(isa(getargs(args, frame)...)) + end + elseif f === isdefined + if nargs == 2 + return Some{Any}(isdefined(@lookup(frame, args[2]), @lookup(frame, args[3]))) + elseif nargs == 3 + return Some{Any}(isdefined(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + else + return Some{Any}(isdefined(getargs(args, frame)...)) + end + elseif @static isdefined(Core, :modifyfield!) && f === modifyfield! + if nargs == 4 + return Some{Any}(modifyfield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]))) + elseif nargs == 5 + return Some{Any}(modifyfield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6]))) + else + return Some{Any}(modifyfield!(getargs(args, frame)...)) + end + elseif f === nfields + if nargs == 1 + return Some{Any}(nfields(@lookup(frame, args[2]))) + else + return Some{Any}(nfields(getargs(args, frame)...)) + end + elseif @static isdefined(Core, :replacefield!) && f === replacefield! + if nargs == 4 + return Some{Any}(replacefield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]))) + elseif nargs == 5 + return Some{Any}(replacefield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6]))) + elseif nargs == 6 + return Some{Any}(replacefield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6]), @lookup(frame, args[7]))) + else + return Some{Any}(replacefield!(getargs(args, frame)...)) + end + elseif f === setfield! + if nargs == 3 + return Some{Any}(setfield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + elseif nargs == 4 + return Some{Any}(setfield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]))) + else + return Some{Any}(setfield!(getargs(args, frame)...)) + end + elseif @static isdefined(Core, :setglobal!) && f === setglobal! + if nargs == 3 + return Some{Any}(setglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + elseif nargs == 4 + return Some{Any}(setglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]))) + else + return Some{Any}(setglobal!(getargs(args, frame)...)) + end + elseif @static isdefined(Core, :swapfield!) && f === swapfield! + if nargs == 3 + return Some{Any}(swapfield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]))) + elseif nargs == 4 + return Some{Any}(swapfield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]))) + else + return Some{Any}(swapfield!(getargs(args, frame)...)) + end + elseif f === throw + if nargs == 1 + return Some{Any}(throw(@lookup(frame, args[2]))) + else + return Some{Any}(throw(getargs(args, frame)...)) + end + elseif f === tuple + return Some{Any}(ntupleany(i->@lookup(frame, args[i+1]), length(args)-1)) + elseif f === typeassert + if nargs == 2 + return Some{Any}(typeassert(@lookup(frame, args[2]), @lookup(frame, args[3]))) + else + return Some{Any}(typeassert(getargs(args, frame)...)) + end + elseif f === typeof + if nargs == 1 + return Some{Any}(typeof(@lookup(frame, args[2]))) + else + return Some{Any}(typeof(getargs(args, frame)...)) + end + # Intrinsics + elseif f === Base.cglobal + if nargs == 1 + call_expr = copy(call_expr) + args2 = args[2] + call_expr.args[2] = isa(args2, QuoteNode) ? args2 : @lookup(frame, args2) + return Some{Any}(Core.eval(moduleof(frame), call_expr)) + elseif nargs == 2 + call_expr = copy(call_expr) + args2 = args[2] + call_expr.args[2] = isa(args2, QuoteNode) ? args2 : @lookup(frame, args2) + call_expr.args[3] = @lookup(frame, args[3]) + return Some{Any}(Core.eval(moduleof(frame), call_expr)) + end + elseif f === Core.Intrinsics.llvmcall + return Some{Any}(Core.Intrinsics.llvmcall(getargs(args, frame)...)) + end + if isa(f, Core.IntrinsicFunction) + cargs = getargs(args, frame) + @static if isdefined(Core.Intrinsics, :have_fma) + if f === Core.Intrinsics.have_fma && length(cargs) == 1 + cargs1 = cargs[1] + if cargs1 == Float64 + return Some{Any}(FMA_FLOAT64[]) + elseif cargs1 == Float32 + return Some{Any}(FMA_FLOAT32[]) + elseif cargs1 == Float16 + return Some{Any}(FMA_FLOAT16[]) + end + end + end + if f === Core.Intrinsics.muladd_float && length(cargs) == 3 + a, b, c = cargs + Ta, Tb, Tc = typeof(a), typeof(b), typeof(c) + if !(Ta == Tb == Tc) + error("muladd_float: types of a, b, and c must match") + end + if Ta == Float64 && FMA_FLOAT64[] + f = Core.Intrinsics.fma_float + elseif Ta == Float32 && FMA_FLOAT32[] + f = Core.Intrinsics.fma_float + elseif Ta == Float16 && FMA_FLOAT16[] + f = Core.Intrinsics.fma_float + end + end + return Some{Any}(ccall(:jl_f_intrinsic_call, Any, (Any, Ptr{Any}, UInt32), f, cargs, length(cargs))) + end + if isa(f, typeof(kwinvoke)) + return Some{Any}(kwinvoke(getargs(args, frame)...)) + end + return call_expr +end diff --git a/packages/JuliaInterpreter/src/commands.jl b/packages/JuliaInterpreter/src/commands.jl new file mode 100644 index 0000000..3d6b542 --- /dev/null +++ b/packages/JuliaInterpreter/src/commands.jl @@ -0,0 +1,522 @@ +""" + pc = finish!(recurse, frame, istoplevel=false) + pc = finish!(frame, istoplevel=false) + +Run `frame` until execution terminates. `pc` is either `nothing` (if execution terminates +when it hits a `return` statement) or a reference to a breakpoint. +In the latter case, `leaf(frame)` returns the frame in which it hit the breakpoint. + +`recurse` controls call evaluation; `recurse = Compiled()` evaluates :call expressions +by normal dispatch, whereas the default `recurse = finish_and_return!` uses recursive interpretation. +""" +function finish!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) + while true + pc = step_expr!(recurse, frame, istoplevel) + pc === nothing && return pc + isa(pc, BreakpointRef) && return pc + shouldbreak(frame, pc) && return BreakpointRef(frame.framecode, pc) + end +end +finish!(frame::Frame, istoplevel::Bool=false) = finish!(finish_and_return!, frame, istoplevel) + +""" + ret = finish_and_return!(recurse, frame, istoplevel::Bool=false) + ret = finish_and_return!(frame, istoplevel::Bool=false) + +Call [`JuliaInterpreter.finish!`](@ref) and pass back the return value `ret`. If execution +pauses at a breakpoint, `ret` is the reference to the breakpoint. +""" +function finish_and_return!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) + pc = finish!(recurse, frame, istoplevel) + isa(pc, BreakpointRef) && return pc + return get_return(frame) +end +finish_and_return!(frame::Frame, istoplevel::Bool=false) = finish_and_return!(finish_and_return!, frame, istoplevel) + +""" + bpref = dummy_breakpoint(recurse, frame::Frame, istoplevel) + +Return a fake breakpoint. `dummy_breakpoint` can be useful as the `recurse` argument to +`evaluate_call!` (or any of the higher-order commands) to ensure that you return immediately +after stepping into a call. +""" +dummy_breakpoint(@nospecialize(recurse), frame::Frame, istoplevel) = BreakpointRef(frame.framecode, 0) + +""" + ret = finish_stack!(recurse, frame, rootistoplevel=false) + ret = finish_stack!(frame, rootistoplevel=false) + +Unwind the callees of `frame`, finishing each before returning to the caller. +`frame` itself is also finished. `rootistoplevel` should be true if the root frame is top-level. + +`ret` is typically the returned value. If execution hits a breakpoint, `ret` will be a +reference to the breakpoint. +""" +function finish_stack!(@nospecialize(recurse), frame::Frame, rootistoplevel::Bool=false) + frame0 = frame + frame = leaf(frame) + while true + istoplevel = rootistoplevel && frame.caller === nothing + ret = finish_and_return!(recurse, frame, istoplevel) + isa(ret, BreakpointRef) && return ret + frame === frame0 && return ret + frame = return_from(frame) + frame === nothing && return ret + pc = frame.pc + if isassign(frame, pc) + lhs = SSAValue(pc) + do_assignment!(frame, lhs, ret) + else + stmt = pc_expr(frame, pc) + if isexpr(stmt, :(=)) + lhs = stmt.args[1] + do_assignment!(frame, lhs, ret) + end + end + pc += 1 + frame.pc = pc + shouldbreak(frame, pc) && return BreakpointRef(frame.framecode, pc) + end +end +finish_stack!(frame::Frame, istoplevel::Bool=false) = finish_stack!(finish_and_return!, frame, istoplevel) + +""" + pc = next_until!(predicate, recurse, frame, istoplevel=false) + pc = next_until!(predicate, frame, istoplevel=false) + +Execute the current statement. Then step through statements of `frame` until the next +statement satisfies `predicate(frame)`. `pc` will be the index of the statement at which +evaluation terminates, `nothing` (if the frame reached a `return`), or a `BreakpointRef`. +""" +function next_until!(@nospecialize(predicate), @nospecialize(recurse), frame::Frame, istoplevel::Bool=false) + pc = step_expr!(recurse, frame, istoplevel) + while pc !== nothing && !isa(pc, BreakpointRef) + shouldbreak(frame, pc) && return BreakpointRef(frame.framecode, pc) + predicate(frame) && return pc + pc = step_expr!(recurse, frame, istoplevel) + end + return pc +end +next_until!(predicate, frame::Frame, istoplevel::Bool=false) = + next_until!(predicate, finish_and_return!, frame, istoplevel) + +""" + pc = maybe_next_until!(predicate, recurse, frame, istoplevel=false) + pc = maybe_next_until!(predicate, frame, istoplevel=false) + +Like [`next_until!`](@ref) except checks `predicate` before executing the current statment. + +""" +function maybe_next_until!(@nospecialize(predicate), @nospecialize(recurse), frame::Frame, istoplevel::Bool=false) + predicate(frame) && return frame.pc + return next_until!(predicate, recurse, frame, istoplevel) +end +maybe_next_until!(@nospecialize(predicate), frame::Frame, istoplevel::Bool=false) = + maybe_next_until!(predicate, finish_and_return!, frame, istoplevel) + +""" + pc = next_call!(recurse, frame, istoplevel=false) + pc = next_call!(frame, istoplevel=false) + +Execute the current statement. Continue stepping through `frame` until the next +`:return` or `:call` expression. +""" +next_call!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) = + next_until!(frame -> is_call_or_return(pc_expr(frame)), recurse, frame, istoplevel) +next_call!(frame::Frame, istoplevel::Bool=false) = next_call!(finish_and_return!, frame, istoplevel) + +""" + pc = maybe_next_call!(recurse, frame, istoplevel=false) + pc = maybe_next_call!(frame, istoplevel=false) + +Return the current program counter of `frame` if it is a `:return` or `:call` expression. +Otherwise, step through the statements of `frame` until the next `:return` or `:call` expression. +""" +maybe_next_call!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) = + maybe_next_until!(frame -> is_call_or_return(pc_expr(frame)), recurse, frame, istoplevel) +maybe_next_call!(frame::Frame, istoplevel::Bool=false) = maybe_next_call!(finish_and_return!, frame, istoplevel) + +""" + pc = through_methoddef_or_done!(recurse, frame) + pc = through_methoddef_or_done!(frame) + +Runs `frame` at top level until it either finishes (e.g., hits a `return` statement) +or defines a new method. +""" +function through_methoddef_or_done!(@nospecialize(recurse), frame::Frame) + predicate(frame) = (stmt = pc_expr(frame); isexpr(stmt, :method, 3) || isexpr(stmt, :thunk)) + pc = next_until!(predicate, recurse, frame, true) + (pc === nothing || isa(pc, BreakpointRef)) && return pc + return step_expr!(recurse, frame, true) # define the method and return +end +through_methoddef_or_done!(@nospecialize(recurse), t::Tuple{Module,Expr,Frame}) = + through_methoddef_or_done!(recurse, t[end]) +through_methoddef_or_done!(@nospecialize(recurse), modex::Tuple{Module,Expr,Expr}) = Core.eval(modex[1], modex[3]) +through_methoddef_or_done!(@nospecialize(recurse), ::Nothing) = nothing +through_methoddef_or_done!(arg) = through_methoddef_or_done!(finish_and_return!, arg) + +# Sentinel to see if the call was a wrapper call +struct Wrapper end + +""" + pc = next_line!(recurse, frame, istoplevel=false) + pc = next_line!(frame, istoplevel=false) + +Execute until reaching the first call of the next line of the source code. +Upon return, `pc` is either the new program counter, `nothing` if a `return` is reached, +or a `BreakpointRef` if it encountered a wrapper call. In the latter case, call `leaf(frame)` +to obtain the new execution frame. +""" +function next_line!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) + pc = frame.pc + initialline, initialfile = linenumber(frame, pc), getfile(frame, pc) + if initialline === nothing || initialfile === nothing + return step_expr!(recurse, frame, istoplevel) + end + return _next_line!(recurse, frame, istoplevel, initialline, initialfile) # avoid boxing +end +function _next_line!(@nospecialize(recurse), frame, istoplevel, initialline::Int, initialfile::String) + predicate(frame) = is_return(pc_expr(frame)) || (linenumber(frame) != initialline || getfile(frame) != initialfile) + + pc = next_until!(predicate, recurse, frame, istoplevel) + (pc === nothing || isa(pc, BreakpointRef)) && return pc + maybe_step_through_kwprep!(recurse, frame, istoplevel) + maybe_next_call!(recurse, frame, istoplevel) +end +next_line!(frame::Frame, istoplevel::Bool=false) = next_line!(finish_and_return!, frame, istoplevel) + +""" + pc = until_line!(recurse, frame, line=nothing istoplevel=false) + pc = until_line!(frame, line=nothing, istoplevel=false) + +Execute until the current frame reaches a line greater than `line`. If `line == nothing` +execute until the current frame reaches any line greater than the current line. +""" +function until_line!(@nospecialize(recurse), frame::Frame, line::Union{Nothing, Integer}=nothing, istoplevel::Bool=false) + pc = frame.pc + initialline, initialfile = linenumber(frame, pc), getfile(frame, pc) + line === nothing && (line = initialline + 1) + predicate(frame) = is_return(pc_expr(frame)) || (linenumber(frame) >= line && getfile(frame) == initialfile) + pc = next_until!(predicate, frame, istoplevel) + (pc === nothing || isa(pc, BreakpointRef)) && return pc + maybe_step_through_kwprep!(recurse, frame, istoplevel) + maybe_next_call!(recurse, frame, istoplevel) +end +until_line!(frame::Frame, line::Union{Nothing, Integer}=nothing, istoplevel::Bool=false) = until_line!(finish_and_return!, frame, line, istoplevel) + +""" + cframe = maybe_step_through_wrapper!(recurse, frame) + cframe = maybe_step_through_wrapper!(frame) + +Return the new frame of execution, potentially stepping through "wrapper" methods like those +that supply default positional arguments or handle keywords. `cframe` is the leaf frame from +which execution should start. +""" +function maybe_step_through_wrapper!(@nospecialize(recurse), frame::Frame) + code = frame.framecode + stmts, scope = code.src.code, code.scope::Method + length(stmts) < 2 && return frame + last = stmts[end-1] + isexpr(last, :(=)) && (last = last.args[2]) + + is_kw = false + if isa(scope, Method) + unwrap1 = Base.unwrap_unionall(scope.sig) + if unwrap1 isa DataType + param1 = Base.unwrap_unionall(unwrap1.parameters[1]) + if param1 isa DataType + is_kw = endswith(String(param1.name.name), "#kw") + end + end + end + + has_selfarg = isexpr(last, :call) && any(@nospecialize(x) -> isa(x, SlotNumber) && x.id == 1, last.args) # isequal(SlotNumber(1)) vulnerable to invalidation + issplatcall, _callee = unpack_splatcall(last) + if is_kw || has_selfarg || (issplatcall && is_bodyfunc(_callee)) + # If the last expr calls #self# or passes it to an implementation method, + # this is a wrapper function that we might want to step through + while frame.pc != length(stmts)-1 + pc = next_call!(recurse, frame, false) # since we're in a Method we're not at toplevel + if pc === nothing || isa(pc, BreakpointRef) + return frame + end + end + ret = evaluate_call!(dummy_breakpoint, frame, last) + if !isa(ret, BreakpointRef) # Happens if next call is Compiled + return frame + end + frame.framedata.ssavalues[frame.pc] = Wrapper() + return maybe_step_through_wrapper!(recurse, callee(frame)) + end + maybe_step_through_nkw_meta!(frame) + return frame +end +maybe_step_through_wrapper!(frame::Frame) = maybe_step_through_wrapper!(finish_and_return!, frame) + + +""" + frame = maybe_step_through_kwprep!(recurse, frame) + frame = maybe_step_through_kwprep!(frame) + +If `frame.pc` points to the beginning of preparatory work for calling a keyword-argument +function, advance forward until the actual call. +""" +function maybe_step_through_kwprep!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) + pc, src = frame.pc, frame.framecode.src + n = length(src.code) + stmt = pc_expr(frame, pc) + if isa(stmt, Tuple{Symbol,Vararg{Symbol}}) + # Check to see if we're creating a NamedTuple followed by kwfunc call + pccall = pc + 5 + if pccall <= n + stmt1 = src.code[pc+1] + # We deliberately check isexpr(stmt, :call) rather than is_call(stmt): if it's + # assigned to a local, it's *not* kwarg preparation. + if isexpr(stmt1, :call) && is_quotenode_egal(stmt1.args[1], Core.apply_type) && is_quoted_type(stmt1.args[2], :NamedTuple) + stmt4, stmt5 = src.code[pc+4], src.code[pc+5] + if isexpr(stmt4, :call) && is_quotenode_egal(stmt4.args[1], Core.kwfunc) + while pc < pccall + pc = step_expr!(recurse, frame, istoplevel) + end + return frame + elseif isexpr(stmt5, :call) && is_quotenode_egal(stmt5.args[1], Core.kwfunc) && pccall+1 <= n + # This happens when the call is scoped by a module + pccall += 1 + while pc < pccall + pc = step_expr!(recurse, frame, istoplevel) + end + maybe_next_call!(recurse, frame, istoplevel) + return frame + end + end + end + elseif isexpr(stmt, :call) && is_quoted_type(stmt.args[1], :NamedTuple) && length(stmt.args) == 1 + # Creating an empty NamedTuple, now split by type (no supplied kwargs vs kwargs...) + if pc + 1 <= n + stmt1 = src.code[pc+1] + if isexpr(stmt1, :call) + f = stmt1.args[1] + if is_quotenode_egal(f, Base.pairs) + # No supplied kwargs + pcsplat = pc + 3 + if pcsplat <= n + issplatcall, callee = unpack_splatcall(src.code[pcsplat]) + if issplatcall && is_bodyfunc(callee) + while pc < pcsplat + pc = step_expr!(recurse, frame, istoplevel) + end + return frame + end + end + pccall = pc + 2 + if pccall <= n + stmt2 = src.code[pccall] + if isa(stmt2, Expr) + if stmt2.head === :call && length(stmt2.args) >= 3 && stmt2.args[2] === SSAValue(pc+1) && stmt2.args[3] === SlotNumber(1) + while pc < pccall + pc = step_expr!(recurse, frame, istoplevel) + end + end + end + end + elseif is_quotenode_egal(f, Base.merge) && ((pccall = pc + 7) <= n) + stmtk = src.code[pccall-1] + if isexpr(stmtk, :call) && is_quotenode_egal(stmtk.args[1], Core.kwfunc) + for i = 1:4 + pc = step_expr!(recurse, frame, istoplevel) + end + stmti = src.code[pc] + if isexpr(stmti, :call) && is_quotenode_egal(stmti.args[1], Core.kwfunc) + pc = step_expr!(recurse, frame, istoplevel) + end + end + end + end + end + end + return frame +end +maybe_step_through_kwprep!(frame::Frame, istoplevel::Bool=false) = + maybe_step_through_kwprep!(finish_and_return!, frame, istoplevel) + +""" + ret = maybe_reset_frame!(recurse, frame, pc, rootistoplevel) + +Perform a return to the caller, or descend to the level of a breakpoint. +`pc` is the return state from the previous command (e.g., `next_call!` or similar). +`rootistoplevel` should be true if the root frame is top-level. + +`ret` will be `nothing` if we have just completed a top-level frame. Otherwise, + + cframe, cpc = ret + +where `cframe` is the frame from which execution should continue and `cpc` is the state +of `cframe` (the program counter, a `BreakpointRef`, or `nothing`). +""" +function maybe_reset_frame!(@nospecialize(recurse), frame::Frame, @nospecialize(pc), rootistoplevel::Bool) + isa(pc, BreakpointRef) && return leaf(frame), pc + if pc === nothing + val = get_return(frame) + frame = return_from(frame) + frame === nothing && return nothing + ssavals = frame.framedata.ssavalues + is_wrapper = isassigned(ssavals, frame.pc) && ssavals[frame.pc] === Wrapper() + maybe_assign!(frame, val) + frame.pc >= nstatements(frame.framecode) && return maybe_reset_frame!(recurse, frame, nothing, rootistoplevel) + frame.pc += 1 + if is_wrapper + return maybe_reset_frame!(recurse, frame, finish!(recurse, frame), rootistoplevel) + end + pc = maybe_next_call!(recurse, frame, rootistoplevel && frame.caller===nothing) + return maybe_reset_frame!(recurse, frame, pc, rootistoplevel) + end + return frame, pc +end +maybe_reset_frame!(frame::Frame, @nospecialize(pc), rootistoplevel::Bool) = + maybe_reset_frame!(finish_and_return!, frame, pc, rootistoplevel) + +# Unwind the stack until an exc is eventually caught, thereby +# returning the frame that caught the exception at the pc of the catch +# or rethrow the error +function unwind_exception(frame::Frame, exc) + while frame !== nothing + if !isempty(frame.framedata.exception_frames) + # Exception caught + frame.pc = frame.framedata.exception_frames[end] + frame.framedata.last_exception[] = exc + return frame + end + frame = return_from(frame) + end + rethrow(exc) +end + +function maybe_step_through_nkw_meta!(frame) + stmt = pc_expr(frame) + if stmt === nothing || (isexpr(stmt, :meta) && (stmt::Expr).args[1] === :nkw) + @assert frame.pc == 1 + frame.pc += 1 + end +end + + +function more_calls_on_current_line(frame) + _, curr_line = whereis(frame) + curr_pc = frame.pc + 1 + while curr_pc <= length(frame.framecode.src.code) + _, new_line = whereis(frame, curr_pc) + new_line == curr_line || return false + is_call(pc_expr(frame, curr_pc)) && return true + curr_pc += 1 + end + return false +end + +""" + ret = debug_command(recurse, frame, cmd, rootistoplevel=false; line=nothing) + ret = debug_command(frame, cmd, rootistoplevel=false; line=nothing) + +Perform one "debugger" command. The keyword arguments are not used for all debug commands. +`cmd` should be one of: + +- `:n`: advance to the next line +- `:s`: step into the next call +- `:sl` step into the last call on the current line (e.g. steps into `f` if the line is `f(g(h(x)))`). +- `:sr` step until the current function will return +- `:until`: advance the frame to line `line` if given, otherwise advance to the line after the current line +- `:c`: continue execution until termination or reaching a breakpoint +- `:finish`: finish the current frame and return to the parent + +or one of the 'advanced' commands + +- `:nc`: step forward to the next call +- `:se`: execute a single statement +- `:si`: execute a single statement, stepping in if it's a call +- `:sg`: step into the generator of a generated function + +`rootistoplevel` and `ret` are as described for [`JuliaInterpreter.maybe_reset_frame!`](@ref). +""" +function debug_command(@nospecialize(recurse), frame::Frame, cmd::Symbol, rootistoplevel::Bool=false; line=nothing) + function nicereturn!(@nospecialize(recurse), frame, pc, rootistoplevel) + if pc === nothing || isa(pc, BreakpointRef) + return maybe_reset_frame!(recurse, frame, pc, rootistoplevel) + end + maybe_step_through_kwprep!(recurse, frame, rootistoplevel && frame.caller === nothing) + return frame, frame.pc + end + + istoplevel = rootistoplevel && frame.caller === nothing + cmd0 = cmd + is_si = false + if cmd === :si + stmt = pc_expr(frame) + cmd = is_call(stmt) ? :s : :se + is_si = true + end + try + cmd === :nc && return nicereturn!(recurse, frame, next_call!(recurse, frame, istoplevel), rootistoplevel) + cmd === :n && return maybe_reset_frame!(recurse, frame, next_line!(recurse, frame, istoplevel), rootistoplevel) + cmd === :se && return maybe_reset_frame!(recurse, frame, step_expr!(recurse, frame, istoplevel), rootistoplevel) + cmd === :until && return maybe_reset_frame!(recurse, frame, until_line!(recurse, frame, line, istoplevel), rootistoplevel) + if cmd === :sl + while more_calls_on_current_line(frame) + next_call!(recurse, frame, istoplevel) + end + return debug_command(recurse, frame, :s, rootistoplevel; line) + end + if cmd === :sr + maybe_next_until!(frame -> is_return(pc_expr(frame)), recurse, frame, istoplevel) + return frame, frame.pc + end + enter_generated = false + if cmd === :sg + enter_generated = true + cmd = :s + end + if cmd === :s + pc = maybe_next_call!(recurse, frame, istoplevel) + (isa(pc, BreakpointRef) || pc === nothing) && return maybe_reset_frame!(recurse, frame, pc, rootistoplevel) + is_si || maybe_step_through_kwprep!(recurse, frame, istoplevel) + pc = frame.pc + stmt0 = stmt = pc_expr(frame, pc) + is_return(stmt0) && return maybe_reset_frame!(recurse, frame, nothing, rootistoplevel) + if isexpr(stmt, :(=)) + stmt = stmt.args[2] + end + local ret + try + ret = evaluate_call!(dummy_breakpoint, frame, stmt; enter_generated=enter_generated) + catch err + ret = handle_err(recurse, frame, err) + return isa(ret, BreakpointRef) ? (leaf(frame), ret) : ret + end + if isa(ret, BreakpointRef) + newframe = leaf(frame) + cmd0 === :si && return newframe, ret + is_si || (newframe = maybe_step_through_wrapper!(recurse, newframe)) + is_si || maybe_step_through_kwprep!(recurse, newframe, istoplevel) + return newframe, BreakpointRef(newframe.framecode, 0) + end + # if we got here, the call returned a value + maybe_assign!(frame, stmt0, ret) + frame.pc += 1 + return frame, frame.pc + end + if cmd === :c + r = root(frame) + ret = finish_stack!(recurse, r, rootistoplevel) + return isa(ret, BreakpointRef) ? (leaf(r), ret) : nothing + end + cmd === :finish && return maybe_reset_frame!(recurse, frame, finish!(recurse, frame, istoplevel), rootistoplevel) + catch err + frame = unwind_exception(frame, err) + if cmd === :c + return debug_command(recurse, frame, :c, istoplevel) + else + return debug_command(recurse, frame, :nc, istoplevel) + end + end + throw(ArgumentError("command $cmd not recognized")) +end +debug_command(frame::Frame, cmd::Symbol, rootistoplevel::Bool=false; kwargs...) = + debug_command(finish_and_return!, frame, cmd, rootistoplevel; kwargs...) diff --git a/packages/JuliaInterpreter/src/construct.jl b/packages/JuliaInterpreter/src/construct.jl new file mode 100644 index 0000000..6ebd5d1 --- /dev/null +++ b/packages/JuliaInterpreter/src/construct.jl @@ -0,0 +1,757 @@ +""" +`framedict[method]` returns the `FrameCode` for `method`. For `@generated` methods, +see [`genframedict`](@ref). +""" +const framedict = Dict{Method,FrameCode}() # essentially a method table for lowered code + +""" +`genframedict[(method,argtypes)]` returns the `FrameCode` for a `@generated` method `method`, +for the particular argument types `argtypes`. + +The framecodes stored in `genframedict` are for the code returned by the generator +(i.e, what will run when you call the method on particular argument types); +for the generator itself, its framecode would be stored in [`framedict`](@ref). +""" +const genframedict = Dict{Tuple{Method,Type},FrameCode}() # the same for @generated functions + +""" +`meth ∈ compiled_methods` indicates that `meth` should be run using [`Compiled`](@ref) +rather than recursed into via the interpreter. +""" +const compiled_methods = Set{Method}() + +""" +`meth ∈ interpreted_methods` indicates that `meth` should *not* be run using [`Compiled`](@ref) +and recursed into via the interpreter. This takes precedence over [`compiled_methods`](@ref) and +[`compiled_modules`](@ref). +""" +const interpreted_methods = Set{Method}() + +""" +`mod ∈ compiled_modules` indicates that any method in `mod` should be run using [`Compiled`](@ref) +rather than recursed into via the interpreter. +""" +const compiled_modules = Set{Module}() + +const junk_framedata = FrameData[] # to allow re-use of allocated memory (this is otherwise a bottleneck) +const junk_frames = Frame[] +debug_mode() = false +@noinline function _check_frame_not_in_junk(frame) + @assert frame.framedata ∉ junk_framedata + @assert frame ∉ junk_frames +end + +@inline function recycle(frame) + debug_mode() && _check_frame_not_in_junk(frame) + push!(junk_framedata, frame.framedata) + push!(junk_frames, frame) +end + +function return_from(frame::Frame) + recycle(frame) + frame = caller(frame) + frame === nothing || (frame.callee = nothing) + return frame +end + +function clear_caches() + empty!(junk_framedata) + empty!(framedict) + empty!(genframedict) + empty!(junk_frames) + for bp in breakpoints() + empty!(bp.instances) + end +end + +const empty_svec = Core.svec() + +function namedtuple(kwargs) + names, types, vals = Symbol[], [], [] + for pr in kwargs + if isa(pr, Expr) + push!(names, pr.args[1]) + val = pr.args[2] + push!(types, typeof(val)) + push!(vals, val) + elseif isa(pr, Pair) + push!(names, pr.first) + val = pr.second + push!(types, typeof(val)) + push!(vals, val) + else + error("unhandled entry type ", typeof(pr)) + end + end + return NamedTuple{(names...,), Tuple{types...}}(vals) +end + +get_source(meth::Method) = Base.uncompressed_ast(meth) + +function get_source(g::GeneratedFunctionStub, env) + b = g(env..., g.argnames...) + b isa CodeInfo && return b + return eval(b) +end + +""" + frun, allargs = prepare_args(fcall, fargs, kwargs) + +Prepare the complete argument sequence for a call to `fcall`. `fargs = [fcall, args...]` is a list +containing both `fcall` (the `#self#` slot in lowered code) and the positional +arguments supplied to `fcall`. `kwargs` is a list of keyword arguments, supplied either as +list of expressions `:(kwname=kwval)` or pairs `:kwname=>kwval`. + +For non-keyword methods, `frun === fcall`, but for methods with keywords `frun` will be the +keyword-sorter function for `fcall`. + +# Example + +```jldoctest +julia> mymethod(x) = 1 +mymethod (generic function with 1 method) + +julia> mymethod(x, y; verbose=false) = nothing +mymethod (generic function with 2 methods) + +julia> JuliaInterpreter.prepare_args(mymethod, [mymethod, 15], ()) +(mymethod, Any[mymethod, 15]) + +julia> JuliaInterpreter.prepare_args(mymethod, [mymethod, 1, 2], [:verbose=>true]) +(var"#mymethod##kw"(), Any[var"#mymethod##kw"(), (verbose = true,), mymethod, 1, 2]) +``` +""" +function prepare_args(@nospecialize(f), allargs, kwargs) + if !isempty(kwargs) + f = Core.kwfunc(f) + allargs = Any[f, namedtuple(kwargs), allargs...] + end + return f, allargs +end + +function prepare_framecode(method::Method, @nospecialize(argtypes); enter_generated=false) + sig = method.sig + if (method.module ∈ compiled_modules || method ∈ compiled_methods) && !(method ∈ interpreted_methods) + return Compiled() + end + # Get static parameters + (ti, lenv::SimpleVector) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), + argtypes, sig)::SimpleVector + enter_generated &= is_generated(method) + if is_generated(method) && !enter_generated + framecode = get(genframedict, (method, argtypes::Type), nothing) + else + framecode = get(framedict, method, nothing) + end + if framecode === nothing + if is_generated(method) && !enter_generated + # If we're stepping into a staged function, we need to use + # the specialization, rather than stepping through the + # unspecialized method. + code = Core.Compiler.get_staged(Core.Compiler.specialize_method(method, argtypes, lenv)) + code === nothing && return nothing + generator = false + else + if is_generated(method) + code = get_source(method.generator, lenv) + generator = true + else + code = get_source(method) + generator = false + end + end + code = code::CodeInfo + # Currenly, our strategy to deal with llvmcall can't handle parametric functions + # (the "mini interpreter" runs in module scope, not method scope) + if (!isempty(lenv) && (hasarg(isidentical(:llvmcall), code.code) || + hasarg(isidentical(Base.llvmcall), code.code) || + hasarg(a->is_global_ref(a, Base, :llvmcall), code.code))) || + hasarg(isidentical(:iolock_begin), code.code) + return Compiled() + end + framecode = FrameCode(method, code; generator=generator) + if is_generated(method) && !enter_generated + genframedict[(method, argtypes)] = framecode + else + framedict[method] = framecode + end + end + return framecode, lenv +end + +function get_framecode(method) + framecode = get(framedict, method, nothing) + if framecode === nothing + @assert !is_generated(method) + code = get_source(method) + framecode = FrameCode(method, code; generator=false) + framedict[method] = framecode + end + return framecode +end + +""" + framecode, frameargs, lenv, argtypes = prepare_call(f, allargs; enter_generated=false) + +Prepare all the information needed to execute lowered code for `f` given arguments `allargs`. +`f` and `allargs` are the outputs of [`prepare_args`](@ref). +For `@generated` methods, set `enter_generated=true` if you want to extract the lowered code +of the generator itself. + +On return `framecode` is the [`FrameCode`](@ref) of the method. +`frameargs` contains the actual arguments needed for executing this frame (for generators, +this will be the types of `allargs`); +`lenv` is the "environment", i.e., the static parameters for `f` given `allargs`. +`argtypes` is the `Tuple`-type for this specific call (equivalent to the signature of the `MethodInstance`). + +# Example + +```jldoctest +julia> mymethod(x::Vector{T}) where T = 1 +mymethod (generic function with 1 method) + +julia> framecode, frameargs, lenv, argtypes = JuliaInterpreter.prepare_call(mymethod, [mymethod, [1.0,2.0]]); + +julia> framecode + 1 1 1 ─ return 1 + +julia> frameargs +2-element Vector{Any}: + mymethod (generic function with 1 method) + [1.0, 2.0] + +julia> lenv +svec(Float64) + +julia> argtypes +Tuple{typeof(mymethod), Vector{Float64}} +``` +""" +function prepare_call(@nospecialize(f), allargs; enter_generated = false) + # Can happen for thunks created by generated functions + if isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction) + return nothing + elseif any(is_vararg_type, allargs) + return nothing # https://github.com/JuliaLang/julia/issues/30995 + end + argtypesv = Any[_Typeof(a) for a in allargs] + argtypes = Tuple{argtypesv...} + method = whichtt(argtypes) + if method === nothing + # Call it to generate the exact error + return f(allargs[2:end]...) + end + ret = prepare_framecode(method, argtypes; enter_generated=enter_generated) + # Exceptional returns + if ret === nothing + # The generator threw an error. Let's generate the same error by calling it. + return f(allargs[2:end]...) + end + isa(ret, Compiled) && return ret, argtypes + # Typical return + framecode, lenv = ret + if is_generated(method) && enter_generated + allargs = Any[_Typeof(a) for a in allargs] + end + return framecode, allargs, lenv, argtypes +end + +function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=empty_svec, caller_will_catch_err::Bool=false) + src = framecode.src + slotnames = src.slotnames + ssavt = src.ssavaluetypes + ng, ns = isa(ssavt, Int) ? ssavt : length(ssavt::Vector{Any}), length(src.slotflags) + if length(junk_framedata) > 0 + olddata = pop!(junk_framedata) + locals, ssavalues, sparams = olddata.locals, olddata.ssavalues, olddata.sparams + exception_frames, last_reference = olddata.exception_frames, olddata.last_reference + last_exception = olddata.last_exception + callargs = olddata.callargs + resize!(locals, ns) + fill!(locals, nothing) + resize!(ssavalues, 0) + resize!(ssavalues, ng) + # for check_isdefined to work properly, we need sparams to start out unassigned + resize!(sparams, 0) + empty!(exception_frames) + resize!(last_reference, ns) + last_exception[] = _INACTIVE_EXCEPTION.instance + else + locals = Vector{Union{Nothing,Some{Any}}}(nothing, ns) + ssavalues = Vector{Any}(undef, ng) + sparams = Vector{Any}(undef, 0) + exception_frames = Int[] + last_reference = Vector{Int}(undef, ns) + callargs = Any[] + last_exception = Ref{Any}(_INACTIVE_EXCEPTION.instance) + end + fill!(last_reference, 0) + if isa(framecode.scope, Method) + meth = framecode.scope::Method + nargs, meth_nargs = length(argvals), Int(meth.nargs) + islastva = meth.isva && nargs >= meth_nargs + for i = 1:meth_nargs-islastva + if nargs >= i + locals[i], last_reference[i] = Some{Any}(argvals[i]), 1 + else + locals[i] = Some{Any}(()) + end + end + if islastva + locals[meth_nargs] = (let i=meth_nargs; Some{Any}(ntupleany(k->argvals[i+k-1], nargs-i+1)); end) + last_reference[meth_nargs] = 1 + end + end + resize!(sparams, length(lenv)) + # Add static parameters to environment + for i = 1:length(lenv) + T = lenv[i] + isa(T, TypeVar) && continue # only fill concrete types + sparams[i] = T + end + FrameData(locals, ssavalues, sparams, exception_frames, last_exception, caller_will_catch_err, last_reference, callargs) +end + +""" + frame = prepare_frame(framecode::FrameCode, frameargs, lenv) + +Construct a new `Frame` for `framecode`, given lowered-code arguments `frameargs` and +static parameters `lenv`. See [`JuliaInterpreter.prepare_call`](@ref) for information about how to prepare the inputs. +""" +function prepare_frame(framecode::FrameCode, args::Vector{Any}, lenv::SimpleVector, caller_will_catch_err::Bool=false) + framedata = prepare_framedata(framecode, args, lenv, caller_will_catch_err) + return Frame(framecode, framedata) +end + +function prepare_frame_caller(caller::Frame, framecode::FrameCode, args::Vector{Any}, lenv::SimpleVector) + caller_will_catch_err = !isempty(caller.framedata.exception_frames) || caller.framedata.caller_will_catch_err + caller.callee = frame = prepare_frame(framecode, args, lenv, caller_will_catch_err) + frame.caller = caller + return frame +end + +""" + ExprSplitter(mod::Module, ex::Expr; lnn=nothing) + +Create an iterable that returns individual expressions together with their module of evaluation. +Optionally supply an initial `LineNumberNode` `lnn`. + +# Example + +``` +julia> expr = quote + public(x::Integer) = true + module Private + private(y::String) = false + end + const threshold = 0.1 + end; + +julia> for (mod, ex) in ExprSplitter(Main, expr) + @show mod ex + end +mod = Main +ex = quote + #= REPL[7]:2 =# + public(x::Integer) = begin + #= REPL[7]:2 =# + true + end +end +mod = Main.Private +ex = quote + #= REPL[7]:4 =# + private(y::String) = begin + #= REPL[7]:4 =# + false + end +end +mod = Main +ex = :($(Expr(:toplevel, :(#= REPL[7]:6 =#), :(const threshold = 0.1)))) +``` + +Note that `Main.Private` was created for you so that its internal expressions could be evaluated. +`ExprSplitter` will check to see whether the module already exists and if so return it rather than +try to create a new module with the same name. + +In general each returned expression is a block with two parts: a `LineNumberNode` followed by a single expression. +In some cases the returned expression may be `:toplevel`, as shown in the `const` declaration, +but otherwise it will be a `:block`. + +# World age, frame creation, and evaluation + +The primary purpose of `ExprSplitter` is to allow sequential return to top-level (e.g., the REPL) +after evaluation of each expression. Returning to top-level allows the world age to update, and hence allows one to call +methods and use types defined in earlier expressions in a block. + +For evaluation by JuliaInterpreter, the returned module/expression pairs can be passed directly to +the `Frame` constructor. However, some expressions cannot be converted into `Frame`s and may need +special handling: + +```julia +julia> for (mod, ex) in ExprSplitter(Main, expr) + if ex.head === :global + # global declarations can't be lowered to a CodeInfo. + # In this demo we choose to evaluate them, but you can do something else. + Core.eval(mod, ex) + continue + end + frame = Frame(mod, ex) + debug_command(frame, :c, true) + end + +julia> threshold +0.1 + +julia> public(3) +true +``` + +If you're parsing package code, `ex` might be a docstring-expression; you may wish +to check for such expressions and take distinct actions. + +See [`Frame(mod::Module, ex::Expr)`](@ref) for more information about frame creation. +""" +mutable struct ExprSplitter + # Non-mutating fields + stack::Vector{Tuple{Module,Expr}} # mod[i] is module of evaluation for + index::Vector{Int} # next-to-handle argument index for :block or :toplevel exprs + # Mutating fields + lnn::Union{LineNumberNode,Nothing} +end +function ExprSplitter(mod::Module, ex::Expr; lnn=nothing) + iter = ExprSplitter(Tuple{Module,Expr}[], Int[], lnn) + push_modex!(iter, mod, ex) + queuenext!(iter) + return iter +end + +Base.IteratorSize(::Type{ExprSplitter}) = Base.SizeUnknown() +Base.eltype(::Type{ExprSplitter}) = Tuple{Module,Expr} + +function push_modex!(iter::ExprSplitter, mod::Module, ex::Expr) + push!(iter.stack, (mod, ex)) + if ex.head === :toplevel || ex.head === :block + # Issue #427 + modifies_scope = false + if ex.head === :block + for a in ex.args + if isa(a, Expr) && a.head ∈ (:local, :global) + modifies_scope = true + break + end + end + end + push!(iter.index, modifies_scope ? 0 : 1) + end + return iter +end + +function pop_modex!(iter) + mod, ex = pop!(iter.stack) + if ex.head === :toplevel || ex.head === :block + pop!(iter.index) + end + return mod, ex +end + +# Load the next-to-evaluate expression into `iter.stack[end]`. +function queuenext!(iter::ExprSplitter) + isempty(iter.stack) && return nothing + mod, ex = iter.stack[end] + head = ex.head + if head === :module + # Find or create the module + newname = ex.args[2]::Symbol + if isdefined(mod, newname) + newmod = getfield(mod, newname) + newmod isa Module || throw(ErrorException("invalid redefinition of constant $(newname)")) + mod = newmod + else + newnamestr = String(newname) + id = Base.identify_package(mod, newnamestr) + # If we're in a test environment and Julia's internal stdlibs are not a declared dependency of the package, + # we might fail to find it. Try really hard to find it. + if id === nothing && mod === Base.__toplevel__ + for loaded_id in keys(Base.loaded_modules) + if loaded_id.name == newnamestr + id = loaded_id + break + end + end + end + if id !== nothing && haskey(Base.loaded_modules, id) + mod = Base.root_module(id)::Module + else + loc = firstline(ex) + mod = Core.eval(mod, Expr(:module, ex.args[1], ex.args[2], Expr(:block, loc, loc)))::Module + end + end + # We've handled the module declaration, remove it and queue the body + pop!(iter.stack) + ex = ex.args[3]::Expr + push_modex!(iter, mod, ex) + return queuenext!(iter) + elseif head === :macrocall + iter.lnn = ex.args[2]::LineNumberNode + elseif head === :block || head === :toplevel + # Container expression + idx = iter.index[end] + if idx == 0 + # return the whole block (issue #427) + return nothing + end + while idx <= length(ex.args) + a = ex.args[idx] + if isa(a, LineNumberNode) + iter.lnn = a + elseif isa(a, Expr) + iter.index[end] = idx + 1 + push_modex!(iter, mod, a) + return queuenext!(iter) + end + idx += 1 + end + # We exhausted the expression without returning anything to evaluate + pop!(iter.stack) + pop!(iter.index) + return queuenext!(iter) + end + return nothing # mod, ex will be returned by iterate +end + +function Base.iterate(iter::ExprSplitter, state=nothing) + isempty(iter.stack) && return nothing + mod, ex = pop_modex!(iter) + lnn = iter.lnn + if is_doc_expr(ex) + body = ex.args[4] + if isa(body, Expr) && body.head === :module + # Just document the module itself and push the module def onto the stack + excopy = Expr(ex.head, ex.args[1], ex.args[2], ex.args[3]) + push!(excopy.args, body.args[2]) + append!(excopy.args, ex.args[5:end]) # there should only be at most a 5th, but just for robustness + ex = excopy + push_modex!(iter, mod, body) + end + end + if ex.head === :block || ex.head === :toplevel + # This was a block that we couldn't safely descend into (issue #427) + if !isempty(iter.index) && iter.index[end] > length(iter.stack[end][2].args) + pop!(iter.stack) + pop!(iter.index) + queuenext!(iter) + end + return (mod, ex), nothing + end + queuenext!(iter) + # :global expressions can't be lowered. For debugging it might be nice + # to still return the lnn, but then we have to work harder on detecting them. + ex.head === :global && return (mod, ex), nothing + return (mod, Expr(:block, lnn, ex)), nothing +end + +""" + framecode, frameargs, lenv, argtypes = determine_method_for_expr(expr; enter_generated = false) + +Prepare all the information needed to execute a particular `:call` expression `expr`. +For example, try `JuliaInterpreter.determine_method_for_expr(:(\$sum([1,2])))`. +See [`JuliaInterpreter.prepare_call`](@ref) for information about the outputs. +""" +function determine_method_for_expr(expr; enter_generated = false) + f = to_function(expr.args[1]) + allargs = expr.args + # Extract keyword args + kwargs = Expr(:parameters) + if length(allargs) > 1 && isexpr(allargs[2], :parameters) + kwargs = splice!(allargs, 2)::Expr + end + f, allargs = prepare_args(f, allargs, kwargs.args) + return prepare_call(f, allargs; enter_generated=enter_generated) +end + +""" + frame = enter_call_expr(expr; enter_generated=false) + +Build a `Frame` ready to execute the expression `expr`. Set `enter_generated=true` +if you want to execute the generator of a `@generated` function, rather than the code that +would be created by the generator. + +# Example + +```jldoctest +julia> mymethod(x) = x+1 +mymethod (generic function with 1 method) + +julia> JuliaInterpreter.enter_call_expr(:(\$mymethod(1))) +Frame for mymethod(x) in Main at none:1 + 1* 1 1 ─ %1 = x + 1 + 2 1 └── return %1 +x = 1 + +julia> mymethod(x::Vector{T}) where T = 1 +mymethod (generic function with 2 methods) + +julia> a = [1.0, 2.0] +2-element Vector{Float64}: + 1.0 + 2.0 + +julia> JuliaInterpreter.enter_call_expr(:(\$mymethod(\$a))) +Frame for mymethod(x::Vector{T}) where T in Main at none:1 + 1* 1 1 ─ return 1 +x = [1.0, 2.0] +T = Float64 +``` + +See [`enter_call`](@ref) for a similar approach not based on expressions. +""" +function enter_call_expr(expr; enter_generated = false) + clear_caches() + r = determine_method_for_expr(expr; enter_generated = enter_generated) + if r !== nothing && !isa(r[1], Compiled) + return prepare_frame(Base.front(r)...) + end + nothing +end + +""" + frame = enter_call(f, args...; kwargs...) + +Build a `Frame` ready to execute `f` with the specified positional and keyword arguments. + +# Example + +```jldoctest +julia> mymethod(x) = x+1 +mymethod (generic function with 1 method) + +julia> JuliaInterpreter.enter_call(mymethod, 1) +Frame for mymethod(x) in Main at none:1 + 1* 1 1 ─ %1 = x + 1 + 2 1 └── return %1 +x = 1 + +julia> mymethod(x::Vector{T}) where T = 1 +mymethod (generic function with 2 methods) + +julia> JuliaInterpreter.enter_call(mymethod, [1.0, 2.0]) +Frame for mymethod(x::Vector{T}) where T in Main at none:1 + 1* 1 1 ─ return 1 +x = [1.0, 2.0] +T = Float64 +``` + +For a `@generated` function you can use `enter_call((f, true), args...; kwargs...)` +to execute the generator of a `@generated` function, rather than the code that +would be created by the generator. + +See [`enter_call_expr`](@ref) for a similar approach based on expressions. +""" +function enter_call(@nospecialize(finfo), @nospecialize(args...); kwargs...) + clear_caches() + if isa(finfo, Tuple) + f = finfo[1] + enter_generated = finfo[2]::Bool + else + f = finfo + enter_generated = false + end + f, allargs = prepare_args(f, Any[f, args...], kwargs) + # Can happen for thunks created by generated functions + if isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction) + error(f, " is a builtin or intrinsic") + end + r = prepare_call(f, allargs; enter_generated=enter_generated) + if r !== nothing && !isa(r[1], Compiled) + return prepare_frame(Base.front(r)...) + end + return nothing +end + +# This is a version of InteractiveUtils.gen_call_with_extracted_types, except that is passes back the +# call expression for further processing. +function extract_args(__module__, ex0) + if isa(ex0, Expr) + if any(a->(isexpr(a, :kw) || isexpr(a, :parameters)), ex0.args) + arg1, args, kwargs = gensym("arg1"), gensym("args"), gensym("kwargs") + return quote + $arg1 = $(ex0.args[1]) + $args, $kwargs = $separate_kwargs($(ex0.args[2:end]...)) + tuple(Core.kwfunc($arg1), $kwargs, $arg1, $args...) + end + elseif ex0.head === :. + return Expr(:tuple, :getproperty, ex0.args...) + elseif ex0.head === :(<:) + return Expr(:tuple, :(<:), ex0.args...) + else + return Expr(:tuple, + mapany(x->isexpr(x,:parameters) ? QuoteNode(x) : x, ex0.args)...) + end + end + if isexpr(ex0, :macrocall) # Make @edit @time 1+2 edit the macro by using the types of the *expressions* + return error("Macros are not supported in @enter") + end + ex = Meta.lower(__module__, ex0) + if !isa(ex, Expr) + return error("expression is not a function call or symbol") + elseif ex.head === :call + return Expr(:tuple, + mapany(x->isexpr(x, :parameters) ? QuoteNode(x) : x, ex.args)...) + elseif ex.head === :body + a1 = ex.args[1] + if isexpr(a1, :call) + a11 = a1.args[1] + if a11 === :setindex! + return Expr(:tuple, + mapany(x->isexpr(x, :parameters) ? QuoteNode(x) : x, arg.args)...) + end + end + end + return error("expression is not a function call, " + * "or is too complex for @enter to analyze; " + * "break it down to simpler parts if possible") +end + +""" + @interpret f(args; kwargs...) + +Evaluate `f` on the specified arguments using the interpreter. + +# Example + +```jldoctest +julia> a = [1, 7] +2-element Vector{Int64}: + 1 + 7 + +julia> sum(a) +8 + +julia> @interpret sum(a) +8 +``` +""" +macro interpret(arg) + args = try + extract_args(__module__, arg) + catch e + return :(throw($e)) + end + quote + local theargs = $(esc(args)) + local frame = JuliaInterpreter.enter_call_expr(Expr(:call, theargs...)) + if frame === nothing + eval(Expr(:call, map(QuoteNode, theargs)...)) + elseif shouldbreak(frame, 1) + frame, BreakpointRef(frame.framecode, 1) + else + local ret = finish_and_return!(frame) + # We deliberately return the top frame here; future debugging commands + # via debug_command may alter the leaves, we want the top frame so we can + # ultimately do `get_return`. + isa(ret, BreakpointRef) ? (frame, ret) : ret + end + end +end diff --git a/packages/JuliaInterpreter/src/interpret.jl b/packages/JuliaInterpreter/src/interpret.jl new file mode 100644 index 0000000..250f634 --- /dev/null +++ b/packages/JuliaInterpreter/src/interpret.jl @@ -0,0 +1,657 @@ +isassign(frame) = isassign(frame, frame.pc) +isassign(frame, pc) = (pc in frame.framecode.used) + +lookup_var(frame, val::SSAValue) = frame.framedata.ssavalues[val.id] +lookup_var(frame, ref::GlobalRef) = getfield(ref.mod, ref.name) +function lookup_var(frame, slot::SlotNumber) + val = frame.framedata.locals[slot.id] + val !== nothing && return val.value + throw(UndefVarError(frame.framecode.src.slotnames[slot.id])) +end + +function lookup_expr(frame, e::Expr) + head = e.head + head === :the_exception && return frame.framedata.last_exception[] + if head === :static_parameter + arg = e.args[1]::Int + if isassigned(frame.framedata.sparams, arg) + return frame.framedata.sparams[arg] + else + syms = sparam_syms(frame.framecode.scope::Method) + throw(UndefVarError(syms[arg])) + end + end + head === :boundscheck && length(e.args) == 0 && return true + error("invalid lookup expr ", e) +end + +""" + rhs = @lookup(frame, node) + rhs = @lookup(mod, frame, node) + +This macro looks up previously-computed values referenced as SSAValues, SlotNumbers, +GlobalRefs, QuoteNode, sparam or exception reference expression. +It will also lookup symbols in `moduleof(frame)`; this can be supplied ahead-of-time via +the 3-argument version. +If none of the above apply, the value of `node` will be returned. +""" +macro lookup(args...) + length(args) == 2 || length(args) == 3 || error("invalid number of arguments ", length(args)) + havemod = length(args) == 3 + local mod + if havemod + mod, frame, node = args + else + frame, node = args + end + nodetmp = gensym(:node) # used to hoist, e.g., args[4] + if havemod + fallback = quote + isa($nodetmp, Symbol) ? getfield($(esc(mod)), $nodetmp) : + $nodetmp + end + else + fallback = quote + $nodetmp + end + end + quote + $nodetmp = $(esc(node)) + isa($nodetmp, SSAValue) ? lookup_var($(esc(frame)), $nodetmp) : + isa($nodetmp, GlobalRef) ? lookup_var($(esc(frame)), $nodetmp) : + isa($nodetmp, SlotNumber) ? lookup_var($(esc(frame)), $nodetmp) : + isa($nodetmp, QuoteNode) ? $nodetmp.value : + isa($nodetmp, Symbol) ? getfield(moduleof($(esc(frame))), $nodetmp) : + isa($nodetmp, Expr) ? lookup_expr($(esc(frame)), $nodetmp) : + $fallback + end +end + +# This is used only for new struct/abstract/primitive nodes. +# The most important issue is that in these expressions, :call Exprs can be nested, +# and hence our re-use of the `callargs` field of Frame would introduce +# bugs. Since these nodes use a very limited repertoire of calls, we can special-case +# this quite easily. +function lookup_or_eval(@nospecialize(recurse), frame, @nospecialize(node)) + if isa(node, SSAValue) + return lookup_var(frame, node) + elseif isa(node, SlotNumber) + return lookup_var(frame, node) + elseif isa(node, GlobalRef) + return lookup_var(frame, node) + elseif isa(node, Symbol) + return getfield(moduleof(frame), node) + elseif isa(node, QuoteNode) + return node.value + elseif isa(node, Expr) + ex = Expr(node.head) + for arg in node.args + push!(ex.args, lookup_or_eval(recurse, frame, arg)) + end + if ex.head === :call + f = ex.args[1] + if f === Core.svec + return Core.svec(ex.args[2:end]...) + elseif f === Core.apply_type + return Core.apply_type(ex.args[2:end]...) + elseif f === Core.typeof + return Core.typeof(ex.args[2]) + elseif f === Base.getproperty + return Base.getproperty(ex.args[2], ex.args[3]) + else + error("unknown call f ", f) + end + else + error("unknown expr ", ex) + end + elseif isa(node, Int) || isa(node, Number) # Number is slow, requires subtyping + return node + elseif isa(node, Type) + return node + end + return eval_rhs(recurse, frame, node) +end + +function resolvefc(frame, @nospecialize(expr)) + if isa(expr, SlotNumber) + expr = lookup_var(frame, expr) + elseif isa(expr, SSAValue) + expr = lookup_var(frame, expr) + isa(expr, Symbol) && return QuoteNode(expr) + end + (isa(expr, Symbol) || isa(expr, String) || isa(expr, Ptr) || isa(expr, QuoteNode)) && return expr + isa(expr, Tuple{Symbol,Symbol}) && return expr + isa(expr, Tuple{String,String}) && return expr + isa(expr, Tuple{Symbol,String}) && return expr + isa(expr, Tuple{String,Symbol}) && return expr + if isexpr(expr, :call) + a = (expr::Expr).args[1] + (isa(a, QuoteNode) && a.value === Core.tuple) || error("unexpected ccall to ", expr) + return Expr(:call, GlobalRef(Core, :tuple), (expr::Expr).args[2:end]...) + end + error("unexpected ccall to ", expr) +end + +function collect_args(@nospecialize(recurse), frame::Frame, call_expr::Expr; isfc::Bool=false) + args = frame.framedata.callargs + resize!(args, length(call_expr.args)) + mod = moduleof(frame) + args[1] = isfc ? resolvefc(frame, call_expr.args[1]) : @lookup(mod, frame, call_expr.args[1]) + for i = 2:length(args) + if isexpr(call_expr.args[i], :call) + args[i] = lookup_or_eval(recurse, frame, call_expr.args[i]) + else + args[i] = @lookup(mod, frame, call_expr.args[i]) + end + end + return args +end + +""" + ret = evaluate_foreigncall(recurse, frame::Frame, call_expr) + +Evaluate a `:foreigncall` (from a `ccall`) statement `callexpr` in the context of `frame`. +""" +function evaluate_foreigncall(@nospecialize(recurse), frame::Frame, call_expr::Expr) + head = call_expr.head + args = collect_args(recurse, frame, call_expr; isfc = head === :foreigncall) + for i = 2:length(args) + arg = args[i] + args[i] = isa(arg, Symbol) ? QuoteNode(arg) : arg + end + head === :cfunction && (args[2] = QuoteNode(args[2])) + scope = frame.framecode.scope + data = frame.framedata + if !isempty(data.sparams) && scope isa Method + sig = scope.sig + args[2] = instantiate_type_in_env(args[2], sig, data.sparams) + arg3 = args[3] + if (@static VERSION < v"1.7.0" && arg3 isa Core.SimpleVector) || + head === :foreigncall + args[3] = Core.svec(map(arg3) do arg + instantiate_type_in_env(arg, sig, data.sparams) + end...) + else + args[3] = instantiate_type_in_env(arg3, sig, data.sparams) + args[4] = Core.svec(map(args[4]::Core.SimpleVector) do arg + instantiate_type_in_env(arg, sig, data.sparams) + end...) + end + end + return Core.eval(moduleof(frame), Expr(head, args...)) +end + +# We have to intercept ccalls / llvmcalls before we try it as a builtin +function bypass_builtins(@nospecialize(recurse), frame, call_expr, pc) + if isassigned(frame.framecode.methodtables, pc) + tme = frame.framecode.methodtables[pc] + if isa(tme, Compiled) + fargs = collect_args(recurse, frame, call_expr) + f = to_function(fargs[1]) + fmod = parentmodule(f)::Module + if fmod === JuliaInterpreter.CompiledCalls || fmod === Core.Compiler + # Fixing https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/432. + @static if VERSION >= v"1.7.0" + return Some{Any}(Base.invoke_in_world(get_world_counter(), f, fargs[2:end]...)) + else + return Some{Any}(Base.invokelatest(f, fargs[2:end]...)) + end + else + return Some{Any}(f(fargs[2:end]...)) + end + end + end + return nothing +end + +function evaluate_call_compiled!(::Compiled, frame::Frame, call_expr::Expr; enter_generated::Bool=false) + # @assert !enter_generated + pc = frame.pc + ret = bypass_builtins(Compiled(), frame, call_expr, pc) + isa(ret, Some{Any}) && return ret.value + ret = maybe_evaluate_builtin(frame, call_expr, false) + isa(ret, Some{Any}) && return ret.value + fargs = collect_args(Compiled(), frame, call_expr) + f = fargs[1] + popfirst!(fargs) # now it's really just `args` + return f(fargs...) +end + +function evaluate_call_recurse!(@nospecialize(recurse), frame::Frame, call_expr::Expr; enter_generated::Bool=false) + pc = frame.pc + ret = bypass_builtins(recurse, frame, call_expr, pc) + isa(ret, Some{Any}) && return ret.value + ret = maybe_evaluate_builtin(frame, call_expr, true) + isa(ret, Some{Any}) && return ret.value + call_expr = ret + fargs = collect_args(recurse, frame, call_expr) + if fargs[1] === Core.eval + return Core.eval(fargs[2], fargs[3]) # not a builtin, but worth treating specially + elseif fargs[1] === Base.rethrow + err = length(fargs) > 1 ? fargs[2] : frame.framedata.last_exception[] + throw(err) + end + if fargs[1] === Core.invoke # invoke needs special handling + f_invoked = which(fargs[2], fargs[3])::Method + fargs_pruned = [fargs[2]; fargs[4:end]] + sig = Tuple{mapany(_Typeof, fargs_pruned)...} + ret = prepare_framecode(f_invoked, sig; enter_generated=enter_generated) + isa(ret, Compiled) && return invoke(fargs[2:end]...) + @assert ret !== nothing + framecode, lenv = ret + lenv === nothing && return framecode # this was a Builtin + fargs = fargs_pruned + else + framecode, lenv = get_call_framecode(fargs, frame.framecode, frame.pc; enter_generated=enter_generated) + if lenv === nothing + if isa(framecode, Compiled) + f = popfirst!(fargs) # now it's really just `args` + return Base.invokelatest(f, fargs...) + end + return framecode # this was a Builtin + end + end + newframe = prepare_frame_caller(frame, framecode, fargs, lenv) + npc = newframe.pc + shouldbreak(newframe, npc) && return BreakpointRef(newframe.framecode, npc) + # if the following errors, handle_err will pop the stack and recycle newframe + if recurse === finish_and_return! + # Optimize this case to avoid dynamic dispatch + ret = finish_and_return!(finish_and_return!, newframe, false) + else + ret = recurse(recurse, newframe, false) + end + isa(ret, BreakpointRef) && return ret + frame.callee = nothing + return_from(newframe) + return ret +end + +""" + ret = evaluate_call!(Compiled(), frame::Frame, call_expr) + ret = evaluate_call!(recurse, frame::Frame, call_expr) + +Evaluate a `:call` expression `call_expr` in the context of `frame`. +The first causes it to be executed using Julia's normal dispatch (compiled code), +whereas the second recurses in via the interpreter. +`recurse` has a default value of [`JuliaInterpreter.finish_and_return!`](@ref). +""" +evaluate_call!(::Compiled, frame::Frame, call_expr::Expr; kwargs...) = evaluate_call_compiled!(Compiled(), frame, call_expr; kwargs...) +evaluate_call!(@nospecialize(recurse), frame::Frame, call_expr::Expr; kwargs...) = evaluate_call_recurse!(recurse, frame, call_expr; kwargs...) +evaluate_call!(frame::Frame, call_expr::Expr; kwargs...) = evaluate_call!(finish_and_return!, frame, call_expr; kwargs...) + +# The following come up only when evaluating toplevel code +function evaluate_methoddef(frame, node) + f = node.args[1] + if isa(f, Symbol) + mod = moduleof(frame) + if Base.isbindingresolved(mod, f) && isdefined(mod, f) # `isdefined` accesses the binding, making it impossible to create a new one + f = getfield(mod, f) + else + f = Core.eval(moduleof(frame), Expr(:function, f)) # create a new function + end + elseif isa(f, GlobalRef) + f = getfield(f.mod, f.name) + end + length(node.args) == 1 && return f + sig = @lookup(frame, node.args[2])::SimpleVector + body = @lookup(frame, node.args[3]) + # branching on https://github.com/JuliaLang/julia/pull/41137 + @static if isdefined(Core.Compiler, :OverlayMethodTable) + ccall(:jl_method_def, Cvoid, (Any, Ptr{Cvoid}, Any, Any), sig, C_NULL, body, moduleof(frame)) + else + ccall(:jl_method_def, Cvoid, (Any, Any, Any), sig, body, moduleof(frame)) + end + return f +end + +function structname(frame, node) + name = node.args[1] + if isa(name, GlobalRef) + mod = name.mod + name = name.name + else + mod = moduleof(frame) + name = name::Symbol + end + return name, mod +end + +function set_structtype_const(mod::Module, name::Symbol) + dt = Base.unwrap_unionall(getfield(mod, name)) + ccall(:jl_set_const, Cvoid, (Any, Any, Any), mod, dt.name.name, dt.name.wrapper) +end + +function inplace_lookup!(ex, i, frame) + a = ex.args[i] + if isa(a, SSAValue) || isa(a, SlotNumber) + ex.args[i] = lookup_var(frame, a) + elseif isexpr(a, :call) + for j = 1:length((a::Expr).args) + inplace_lookup!(a, j, frame) + end + end + return ex +end + +function do_assignment!(frame, @nospecialize(lhs), @nospecialize(rhs)) + code, data = frame.framecode, frame.framedata + if isa(lhs, SSAValue) + data.ssavalues[lhs.id] = rhs + elseif isa(lhs, SlotNumber) + counter = (frame.assignment_counter += 1) + data.locals[lhs.id] = Some{Any}(rhs) + data.last_reference[lhs.id] = counter + elseif isa(lhs, GlobalRef) + @static if @isdefined setglobal! + setglobal!(lhs.mod, lhs.name, rhs) + else + ccall(:jl_set_global, Cvoid, (Any, Any, Any), lhs.mod, lhs.name, rhs) + end + elseif isa(lhs, Symbol) + @static if @isdefined setglobal! + setglobal!(moduleof(code), lhs, rhs) + else + ccall(:jl_set_global, Cvoid, (Any, Any, Any), moduleof(code), lhs, rhs) + end + end +end + +function maybe_assign!(frame, @nospecialize(stmt), @nospecialize(val)) + pc = frame.pc + if isexpr(stmt, :(=)) + lhs = stmt.args[1] + do_assignment!(frame, lhs, val) + elseif isassign(frame, pc) + lhs = SSAValue(pc) + do_assignment!(frame, lhs, val) + end + return nothing +end +maybe_assign!(frame, @nospecialize(val)) = maybe_assign!(frame, pc_expr(frame), val) + + +function eval_rhs(@nospecialize(recurse), frame, node::Expr) + head = node.head + if head === :new + mod = moduleof(frame) + args = let mod=mod + Any[@lookup(mod, frame, arg) for arg in node.args] + end + T = popfirst!(args) + rhs = ccall(:jl_new_structv, Any, (Any, Ptr{Any}, UInt32), T, args, length(args)) + return rhs + elseif head === :splatnew # Julia 1.2+ + mod = moduleof(frame) + rhs = ccall(:jl_new_structt, Any, (Any, Any), @lookup(mod, frame, node.args[1]), @lookup(mod, frame, node.args[2])) + return rhs + elseif head === :isdefined + return check_isdefined(frame, node.args[1]) + elseif head === :call + # here it's crucial to avoid dynamic dispatch + isa(recurse, Compiled) && return evaluate_call_compiled!(recurse, frame, node) + return evaluate_call_recurse!(recurse, frame, node) + elseif head === :foreigncall || head === :cfunction + return evaluate_foreigncall(recurse, frame, node) + elseif head === :copyast + val = (node.args[1]::QuoteNode).value + return isa(val, Expr) ? copy(val) : val + elseif head === :enter + return length(frame.framedata.exception_frames) + elseif head === :boundscheck + return true + elseif head === :meta || head === :inbounds || head === :loopinfo || + head === :gc_preserve_begin || head === :gc_preserve_end + return nothing + elseif head === :method && length(node.args) == 1 + return evaluate_methoddef(frame, node) + end + return lookup_expr(frame, node) +end + +function check_isdefined(frame, @nospecialize(node)) + data = frame.framedata + if isa(node, SlotNumber) + return data.locals[node.id] !== nothing + elseif isa(node, Core.Compiler.Argument) # just to be safe, since base handles this + return data.locals[node.n] !== nothing + elseif isexpr(node, :static_parameter) + return isassigned(data.sparams, node.args[1]::Int) + elseif isa(node, GlobalRef) + return isdefined(node.mod, node.name) + elseif isa(node, Symbol) + return isdefined(moduleof(frame), node) + else # QuoteNode or other implicitly quoted object + return true + end +end + +function coverage_visit_line!(frame::Frame) + pc, code = frame.pc, frame.framecode + code.report_coverage || return + src = code.src + codeloc = src.codelocs[pc] + if codeloc != frame.last_codeloc + linetable = src.linetable::Vector{Any} + lineinfo = linetable[codeloc]::Core.LineInfoNode + file, line = String(lineinfo.file), lineinfo.line + ccall(:jl_coverage_visit_line, Cvoid, (Cstring, Csize_t, Cint), file, sizeof(file), line) + frame.last_codeloc = codeloc + end +end + +# For "profiling" where JuliaInterpreter spends its time. See the commented-out block +# in `step_expr!` +const _location = Dict{Tuple{Method,Int},Int}() + +function step_expr!(@nospecialize(recurse), frame, @nospecialize(node), istoplevel::Bool) + pc, code, data = frame.pc, frame.framecode, frame.framedata + # if !is_leaf(frame) + # show_stackloc(frame) + # @show node + # end + @assert is_leaf(frame) + @static VERSION >= v"1.8.0-DEV.370" && coverage_visit_line!(frame) + local rhs + # For debugging: + # show_stackloc(frame) + # @show node + # For profiling: + # location_key = (scopeof(frame), pc) + # _location[location_key] = get(_location, location_key, 0) + 1 + try + if isa(node, Expr) + if node.head === :(=) + lhs, rhs = node.args + if isa(rhs, Expr) + rhs = eval_rhs(recurse, frame, rhs) + else + rhs = istoplevel ? @lookup(moduleof(frame), frame, rhs) : @lookup(frame, rhs) + end + isa(rhs, BreakpointRef) && return rhs + do_assignment!(frame, lhs, rhs) + elseif node.head === :gotoifnot + arg = @lookup(frame, node.args[1]) + if !isa(arg, Bool) + throw(TypeError(nameof(frame), "if", Bool, arg)) + end + if !arg + return (frame.pc = node.args[2]::Int) + end + elseif node.head === :enter + rhs = node.args[1]::Int + push!(data.exception_frames, rhs) + elseif node.head === :leave + for _ = 1:node.args[1]::Int + pop!(data.exception_frames) + end + elseif node.head === :pop_exception + n = lookup_var(frame, node.args[1]::SSAValue)::Int + deleteat!(data.exception_frames, n+1:length(data.exception_frames)) + elseif node.head === :return + return nothing + elseif istoplevel + if node.head === :method && length(node.args) > 1 + evaluate_methoddef(frame, node) + elseif node.head === :module + error("this should have been handled by split_expressions") + elseif node.head === :using || node.head === :import || node.head === :export + Core.eval(moduleof(frame), node) + elseif node.head === :const + g = node.args[1] + if isa(g, GlobalRef) + mod, name = g.mod, g.name + else + mod, name = moduleof(frame), g::Symbol + end + Core.eval(mod, Expr(:const, name)) + elseif node.head === :thunk + newframe = Frame(moduleof(frame), node.args[1]::CodeInfo) + if isa(recurse, Compiled) + finish!(recurse, newframe, true) + else + newframe.caller = frame + frame.callee = newframe + finish!(recurse, newframe, true) + frame.callee = nothing + end + return_from(newframe) + elseif node.head === :global + Core.eval(moduleof(frame), node) + elseif node.head === :toplevel + mod = moduleof(frame) + iter = ExprSplitter(mod, node) + rhs = Core.eval(mod, Expr(:toplevel, + :(for (mod, ex) in $iter + if ex.head === :toplevel + Core.eval(mod, ex) + continue + end + newframe = ($Frame)(mod, ex) + while true + ($through_methoddef_or_done!)($recurse, newframe) === nothing && break + end + $return_from(newframe) + end))) + elseif node.head === :error + error("unexpected error statement ", node) + elseif node.head === :incomplete + error("incomplete statement ", node) + else + rhs = eval_rhs(recurse, frame, node) + end + elseif node.head === :thunk || node.head === :toplevel + error("this frame needs to be run at top level") + else + rhs = eval_rhs(recurse, frame, node) + end + elseif isa(node, GotoNode) + return (frame.pc = node.label) + elseif is_GotoIfNot(node) + node = node::Core.GotoIfNot + arg = @lookup(frame, node.cond) + if !isa(arg, Bool) + throw(TypeError(nameof(frame), "if", Bool, arg)) + end + if !arg + return (frame.pc = node.dest) + end + elseif is_ReturnNode(node) + return nothing + elseif isa(node, NewvarNode) + # FIXME: undefine the slot? + elseif istoplevel && isa(node, LineNumberNode) + elseif istoplevel && isa(node, Symbol) + rhs = getfield(moduleof(frame), node) + else + rhs = @lookup(frame, node) + end + catch err + return handle_err(recurse, frame, err) + end + @isdefined(rhs) && isa(rhs, BreakpointRef) && return rhs + if isassign(frame, pc) + # if !@isdefined(rhs) + # @show frame node + # end + lhs = SSAValue(pc) + do_assignment!(frame, lhs, rhs) + end + return (frame.pc = pc + 1) +end + +""" + pc = step_expr!(recurse, frame, istoplevel=false) + pc = step_expr!(frame, istoplevel=false) + +Execute the next statement in `frame`. `pc` is the new program counter, or `nothing` +if execution terminates, or a [`BreakpointRef`](@ref) if execution hits a breakpoint. + +`recurse` controls call evaluation; `recurse = Compiled()` evaluates :call expressions +by normal dispatch. The default value `recurse = finish_and_return!` will use recursive +interpretation. + +If you are evaluating `frame` at module scope you should pass `istoplevel=true`. +""" +step_expr!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) = + step_expr!(recurse, frame, pc_expr(frame), istoplevel) +step_expr!(frame::Frame, istoplevel::Bool=false) = + step_expr!(finish_and_return!, frame, istoplevel) + +""" + loc = handle_err(recurse, frame, err) + +Deal with an error `err` that arose while evaluating `frame`. There are one of three +behaviors: + +- if `frame` catches the error, `loc` is the program counter at which to resume + evaluation of `frame`; +- if `frame` doesn't catch the error, but `break_on_error[]` is `true`, + `loc` is a `BreakpointRef`; +- otherwise, `err` gets rethrown. +""" +function handle_err(@nospecialize(recurse), frame, err) + data = frame.framedata + err_will_be_thrown_to_top_level = isempty(data.exception_frames) && !data.caller_will_catch_err + if break_on_throw[] || (break_on_error[] && err_will_be_thrown_to_top_level) + return BreakpointRef(frame.framecode, frame.pc, err) + end + if isempty(data.exception_frames) + if !err_will_be_thrown_to_top_level + return_from(frame) + end + # Check for world age errors, which generally indicate a failure to go back to toplevel + if isa(err, MethodError) + is_arg_types = isa(err.args, DataType) + arg_types = is_arg_types ? err.args : Base.typesof(err.args...) + if (err.world != typemax(UInt) && + hasmethod(err.f, arg_types) && + !hasmethod(err.f, arg_types, world = err.world)) + @warn "likely failure to return to toplevel, try `ExprSplitter`" + end + end + rethrow(err) + end + data.last_exception[] = err + return (frame.pc = data.exception_frames[end]) +end + +if isdefined(Core, :ReturnNode) + lookup_return(frame, node::Core.ReturnNode) = @lookup(frame, node.val) +else + lookup_return(frame, node::Expr) = @lookup(frame, node.args[1]) +end + +""" + ret = get_return(frame) + +Get the return value of `frame`. Throws an error if `frame.pc` does not point to a `return` expression. +`frame` must have already been executed so that the return value has been computed (see, +e.g., [`JuliaInterpreter.finish!`](@ref)). +""" +function get_return(frame) + node = pc_expr(frame) + is_return(node) || error("expected return statement, got ", node) + return lookup_return(frame, node) +end +get_return(t::Tuple{Module,Expr,Frame}) = get_return(t[end]) diff --git a/packages/JuliaInterpreter/src/localmethtable.jl b/packages/JuliaInterpreter/src/localmethtable.jl new file mode 100644 index 0000000..f535635 --- /dev/null +++ b/packages/JuliaInterpreter/src/localmethtable.jl @@ -0,0 +1,96 @@ +const max_methods = 4 # maximum number of MethodInstances tracked for a particular :call statement + +""" + framecode, lenv = get_call_framecode(fargs, parentframe::FrameCode, idx::Int) + +Return the framecode and environment for a call specified by `fargs = [f, args...]` (see [`prepare_args`](@ref)). +`parentframecode` is the caller, and `idx` is the program-counter index. +If possible, `framecode` will be looked up from the local method tables of `parentframe`. +""" +function get_call_framecode(fargs::Vector{Any}, parentframe::FrameCode, idx::Int; enter_generated::Bool=false) + nargs = length(fargs) # includes f as the first "argument" + # Determine whether we can look up the appropriate framecode in the local method table + if isassigned(parentframe.methodtables, idx) # if this is the first call, this may not yet be set + # The case where `methodtables[idx]` is a `Compiled` has already been handled in `bypass_builtins` + d_meth = d_meth1 = parentframe.methodtables[idx]::DispatchableMethod + local d_methprev + depth = 1 + while true + # TODO: consider using world age bounds to handle cache invalidation + # Determine whether the argument types match the signature + sig = d_meth.sig.parameters::SimpleVector + if length(sig) == nargs + # If this is generated, match only if `enter_generated` also matches + fi = d_meth.frameinstance + if fi isa FrameInstance + matches = !is_generated(scopeof(fi.framecode)::Method) || enter_generated == fi.enter_generated + else + matches = !enter_generated + end + if matches + for i = 1:nargs + if !isa(fargs[i], sig[i]) + matches = false + break + end + end + end + if matches + # Rearrange the list to place this method first + # (if we're in a loop, we'll likely match this one again on the next iteration) + if depth > 1 + parentframe.methodtables[idx] = d_meth + d_methprev.next = d_meth.next + d_meth.next = d_meth1 + end + if fi isa Compiled + return Compiled(), nothing + else + fi = fi::FrameInstance + return fi.framecode, fi.sparam_vals + end + end + end + depth += 1 + d_methprev = d_meth + d_meth = d_meth.next + d_meth === nothing && break + d_meth = d_meth::DispatchableMethod + end + end + # We haven't yet encountered this argtype combination and need to look it up by dispatch + fargs[1] = f = to_function(fargs[1]) + ret = prepare_call(f, fargs; enter_generated=enter_generated) + ret === nothing && return f(fargs[2:end]...), nothing + is_compiled = isa(ret[1], Compiled) + local framecode + if is_compiled + d_meth = DispatchableMethod(nothing, Compiled(), ret[2]) + else + framecode, args, env, argtypes = ret + # Store the results of the method lookup in the local method table + fi = FrameInstance(framecode, env, is_generated(scopeof(framecode::FrameCode)::Method) && enter_generated) + d_meth = DispatchableMethod(nothing, fi, argtypes) + end + if isassigned(parentframe.methodtables, idx) + # Drop the oldest d_meth, if necessary + d_methtmp = d_meth.next = parentframe.methodtables[idx]::DispatchableMethod + depth = 2 + while d_methtmp.next !== nothing + depth += 1 + depth >= max_methods && break + d_methtmp = d_methtmp.next::DispatchableMethod + end + if depth >= max_methods + d_methtmp.next = nothing + end + else + d_meth.next = nothing + end + parentframe.methodtables[idx] = d_meth + if is_compiled + return Compiled(), nothing + else + return framecode, env + end +end diff --git a/packages/JuliaInterpreter/src/optimize.jl b/packages/JuliaInterpreter/src/optimize.jl new file mode 100644 index 0000000..7a2dda2 --- /dev/null +++ b/packages/JuliaInterpreter/src/optimize.jl @@ -0,0 +1,439 @@ +const calllike = (:call, :foreigncall) + +const compiled_calls = Dict{Any,Any}() + +function extract_inner_call!(stmt::Expr, idx, once::Bool=false) + (stmt.head === :toplevel || stmt.head === :thunk) && return nothing + once |= stmt.head ∈ calllike + for (i, a) in enumerate(stmt.args) + isa(a, Expr) || continue + # Make sure we don't "damage" special syntax that requires literals + if i == 1 && stmt.head === :foreigncall + continue + end + if i == 2 && stmt.head === :call && stmt.args[1] === :cglobal + continue + end + ret = extract_inner_call!(a, idx, once) # doing this first extracts innermost calls + ret !== nothing && return ret + iscalllike = a.head ∈ calllike + if once && iscalllike + stmt.args[i] = NewSSAValue(idx) + return a + end + end + return nothing +end + +function replace_ssa(@nospecialize(stmt), ssalookup) + isa(stmt, Expr) || return stmt + return Expr(stmt.head, Any[ + if isa(a, SSAValue) + SSAValue(ssalookup[a.id]) + elseif isa(a, NewSSAValue) + SSAValue(a.id) + else + replace_ssa(a, ssalookup) + end + for a in stmt.args + ]...) +end + +function renumber_ssa!(stmts::Vector{Any}, ssalookup) + # When updating jumps, when lines get split into multiple lines + # (see "Un-nest :call expressions" below), we need to jump to the first of them. + # Consequently we use the previous "old-code" offset and add one. + # Fixes #455. + jumplookup(l, idx) = idx > 1 ? l[idx-1] + 1 : idx + + for (i, stmt) in enumerate(stmts) + if isa(stmt, GotoNode) + stmts[i] = GotoNode(jumplookup(ssalookup, stmt.label)) + elseif isa(stmt, SSAValue) + stmts[i] = SSAValue(ssalookup[stmt.id]) + elseif isa(stmt, NewSSAValue) + stmts[i] = SSAValue(stmt.id) + elseif isa(stmt, Expr) + stmt = replace_ssa(stmt, ssalookup) + if (stmt.head === :gotoifnot || stmt.head === :enter) && isa(stmt.args[end], Int) + stmt.args[end] = jumplookup(ssalookup, stmt.args[end]) + end + stmts[i] = stmt + elseif is_GotoIfNot(stmt) + cond = (stmt::Core.GotoIfNot).cond + if isa(cond, SSAValue) + cond = SSAValue(ssalookup[cond.id]) + end + stmts[i] = Core.GotoIfNot(cond, jumplookup(ssalookup, stmt.dest)) + elseif is_ReturnNode(stmt) + val = (stmt::Core.ReturnNode).val + if isa(val, SSAValue) + stmts[i] = Core.ReturnNode(SSAValue(ssalookup[val.id])) + end + end + end + return stmts +end + +function compute_ssa_mapping_delete_statements!(code::CodeInfo, stmts::Vector{Int}) + stmts = unique!(sort!(stmts)) + ssalookup = collect(1:length(codelocs(code))) + cnt = 1 + for i in 1:length(stmts) + start = stmts[i] + 1 + stop = i == length(stmts) ? length(codelocs(code)) : stmts[i+1] + ssalookup[start:stop] .-= cnt + cnt += 1 + end + return ssalookup +end + +# Pre-frame-construction lookup +function lookup_stmt(stmts, arg) + if isa(arg, SSAValue) + arg = stmts[arg.id] + end + if isa(arg, QuoteNode) + arg = arg.value + end + return arg +end + +function smallest_ref(stmts, arg, idmin) + if isa(arg, SSAValue) + idmin = min(idmin, arg.id) + return smallest_ref(stmts, stmts[arg.id], idmin) + elseif isa(arg, Expr) + for a in arg.args + idmin = smallest_ref(stmts, a, idmin) + end + end + return idmin +end + +function lookup_global_ref(a::GlobalRef) + if isdefined(a.mod, a.name) && isconst(a.mod, a.name) + r = getfield(a.mod, a.name) + return QuoteNode(r) + else + return a + end +end + +function lookup_global_refs!(ex::Expr) + (ex.head === :isdefined || ex.head === :thunk || ex.head === :toplevel) && return nothing + for (i, a) in enumerate(ex.args) + ex.head === :(=) && i == 1 && continue # Don't look up globalrefs on the LHS of an assignment (issue #98) + if isa(a, GlobalRef) + ex.args[i] = lookup_global_ref(a) + elseif isa(a, Expr) + lookup_global_refs!(a) + end + end + return nothing +end + +function lookup_getproperties(a::Expr) + if a.head === :call && length(a.args) == 3 && + a.args[1] isa QuoteNode && a.args[1].value === Base.getproperty && + a.args[2] isa QuoteNode && a.args[2].value isa Module && + a.args[3] isa QuoteNode && a.args[3].value isa Symbol + return lookup_global_ref(Core.GlobalRef(a.args[2].value, a.args[3].value)) + end + return a +end + +""" + optimize!(code::CodeInfo, mod::Module) + +Perform minor optimizations on the lowered AST in `code` to reduce execution time +of the interpreter. +Currently it looks up `GlobalRef`s (for which it needs `mod` to know the scope in +which this will run) and ensures that no statement includes nested `:call` expressions +(splitting them out into multiple SSA-form statements if needed). +""" +function optimize!(code::CodeInfo, scope) + mod = moduleof(scope) + evalmod = mod == Core.Compiler ? Core.Compiler : CompiledCalls + sparams = scope isa Method ? sparam_syms(scope) : Symbol[] + code.inferred && error("optimization of inferred code not implemented") + replace_coretypes!(code) + # TODO: because of builtins.jl, for CodeInfos like + # %1 = Core.apply_type + # %2 = (%1)(args...) + # it would be best to *not* resolve the GlobalRef at %1 + ## Replace GlobalRefs with QuoteNodes + for (i, stmt) in enumerate(code.code) + if isa(stmt, GlobalRef) + code.code[i] = lookup_global_ref(stmt) + elseif isa(stmt, Expr) + if stmt.head === :call && stmt.args[1] === :cglobal # cglobal requires literals + continue + else + lookup_global_refs!(stmt) + code.code[i] = lookup_getproperties(stmt) + end + end + end + + # Replace :llvmcall and :foreigncall with compiled variants. See + # https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/13#issuecomment-464880123 + foreigncalls_idx = Int[] + for (idx, stmt) in enumerate(code.code) + # Foregincalls can be rhs of assignments + if isexpr(stmt, :(=)) + stmt = (stmt::Expr).args[2] + end + if isa(stmt, Expr) + if stmt.head === :call + # Check for :llvmcall + arg1 = stmt.args[1] + if (arg1 === :llvmcall || lookup_stmt(code.code, arg1) === Base.llvmcall) && isempty(sparams) && scope isa Method + # Call via `invokelatest` to avoid compiling it until we need it + Base.invokelatest(build_compiled_llvmcall!, stmt, code, idx, evalmod) + push!(foreigncalls_idx, idx) + end + elseif stmt.head === :foreigncall && scope isa Method + # Call via `invokelatest` to avoid compiling it until we need it + Base.invokelatest(build_compiled_foreigncall!, stmt, code, sparams, evalmod) + push!(foreigncalls_idx, idx) + end + end + end + + ## Un-nest :call expressions (so that there will be only one :call per line) + # This will allow us to re-use args-buffers rather than having to allocate new ones each time. + old_code, old_codelocs = code.code, codelocs(code) + code.code = new_code = eltype(old_code)[] + code.codelocs = new_codelocs = Int32[] + ssainc = fill(1, length(old_code)) + for (i, stmt) in enumerate(old_code) + loc = old_codelocs[i] + if isa(stmt, Expr) + inner = extract_inner_call!(stmt, length(new_code)+1) + while inner !== nothing + push!(new_code, inner) + push!(new_codelocs, loc) + ssainc[i] += 1 + inner = extract_inner_call!(stmt, length(new_code)+1) + end + end + push!(new_code, stmt) + push!(new_codelocs, loc) + end + # Fix all the SSAValues and GotoNodes + ssalookup = cumsum(ssainc) + renumber_ssa!(new_code, ssalookup) + code.ssavaluetypes = length(new_code) + + # Insert the foreigncall wrappers at the updated idxs + methodtables = Vector{Union{Compiled,DispatchableMethod}}(undef, length(code.code)) + for idx in foreigncalls_idx + methodtables[ssalookup[idx]] = Compiled() + end + + return code, methodtables +end + +function parametric_type_to_expr(@nospecialize(t::Type)) + t isa Core.TypeofBottom && return t + while t isa UnionAll + t = t.body + end + t = t::DataType + if Base.isvarargtype(t) + return Expr(:(...), t.parameters[1]) + end + if Base.has_free_typevars(t) + params = map(t.parameters) do @nospecialize(p) + isa(p, TypeVar) ? p.name : + isa(p, DataType) && Base.has_free_typevars(p) ? parametric_type_to_expr(p) : p + end + return Expr(:curly, scopename(t.name), params...)::Expr + end + return t +end + +function build_compiled_llvmcall!(stmt::Expr, code, idx, evalmod) + # Run a mini-interpreter to extract the types + framecode = FrameCode(CompiledCalls, code; optimize=false) + frame = Frame(framecode, prepare_framedata(framecode, [])) + idxstart = idx + for i = 2:4 + idxstart = smallest_ref(code.code, stmt.args[i], idxstart) + end + frame.pc = idxstart + if idxstart < idx + while true + pc = step_expr!(Compiled(), frame) + pc === idx && break + pc === nothing && error("this should never happen") + end + end + llvmir, RetType, ArgType = @lookup(frame, stmt.args[2]), @lookup(frame, stmt.args[3]), @lookup(frame, stmt.args[4])::DataType + args = stmt.args[5:end] + argnames = Any[Symbol(:arg, i) for i = 1:length(args)] + cc_key = (llvmir, RetType, ArgType, evalmod) # compiled call key + f = get(compiled_calls, cc_key, nothing) + if f === nothing + methname = gensym("compiled_llvmcall") + def = :( + function $methname($(argnames...)) + return $(Base.llvmcall)($llvmir, $RetType, $ArgType, $(argnames...)) + end) + f = Core.eval(evalmod, def) + compiled_calls[cc_key] = f + end + + stmt.args[1] = QuoteNode(f) + stmt.head = :call + deleteat!(stmt.args, 2:length(stmt.args)) + append!(stmt.args, args) +end + + +# Handle :llvmcall & :foreigncall (issue #28) +function build_compiled_foreigncall!(stmt::Expr, code, sparams::Vector{Symbol}, evalmod) + TVal = evalmod == Core.Compiler ? Core.Compiler.Val : Val + cfunc, RetType, ArgType = lookup_stmt(code.code, stmt.args[1]), stmt.args[2], stmt.args[3]::SimpleVector + + dynamic_ccall = false + oldcfunc = nothing + if isa(cfunc, Expr) # specification by tuple, e.g., (:clock, "libc") + cfunc = something(static_eval(cfunc), cfunc) + end + if isa(cfunc, Symbol) + cfunc = QuoteNode(cfunc) + elseif isa(cfunc, String) || isa(cfunc, Ptr) || isa(cfunc, Tuple) + # do nothing + else + dynamic_ccall = true + oldcfunc = cfunc + cfunc = gensym("ptr") + end + if isa(RetType, SimpleVector) + @assert length(RetType) == 1 + RetType = RetType[1] + end + args = stmt.args[6:end] + # When the ccall is dynamic we pass the pointer as an argument so can reuse the function + cc_key = ((dynamic_ccall ? :ptr : cfunc), RetType, ArgType, evalmod, length(sparams), length(args)) # compiled call key + f = get(compiled_calls, cc_key, nothing) + if f === nothing + ArgType = Expr(:tuple, Any[parametric_type_to_expr(t) for t in ArgType::SimpleVector]...) + RetType = parametric_type_to_expr(RetType) + # #285: test whether we can evaluate an type constraints on parametric expressions + # this essentially comes down to having the names be available in CompiledCalls, + # if they are not then executing the method will fail + try + isa(RetType, Expr) && Core.eval(CompiledCalls, wrap_params(RetType, sparams)) + isa(ArgType, Expr) && Core.eval(CompiledCalls, wrap_params(ArgType, sparams)) + catch + return nothing + end + argnames = Any[Symbol(:arg, i) for i = 1:length(args)] + wrapargs = copy(argnames) + for sparam in sparams + push!(wrapargs, :(::$TVal{$sparam})) + end + if dynamic_ccall + pushfirst!(wrapargs, cfunc) + end + methname = gensym("compiled_ccall") + def = :( + function $methname($(wrapargs...)) where {$(sparams...)} + return $(Expr(:foreigncall, cfunc, RetType, stmt.args[3:5]..., argnames...)) + end) + f = Core.eval(evalmod, def) + compiled_calls[cc_key] = f + end + stmt.args[1] = QuoteNode(f) + stmt.head = :call + deleteat!(stmt.args, 2:length(stmt.args)) + if dynamic_ccall + push!(stmt.args, oldcfunc) + end + append!(stmt.args, args) + for i in 1:length(sparams) + push!(stmt.args, :($TVal($(Expr(:static_parameter, i))))) + end + return nothing +end + +function replace_coretypes!(src; rev::Bool=false) + if isa(src, CodeInfo) + replace_coretypes_list!(src.code; rev=rev) + elseif isa(src, Expr) + replace_coretypes_list!(src.args; rev=rev) + end + return src +end + +function replace_coretypes_list!(list::AbstractVector; rev::Bool) + function rep(@nospecialize(x), rev::Bool) + if rev + if isa(x, SSAValue) + return Core.SSAValue(x.id) + elseif isa(x, SlotNumber) + return Core.SlotNumber(x.id) + end + return x + end + if isa(x, Core.SSAValue) + return SSAValue(x.id) + elseif isa(x, Core.SlotNumber) || isa(x, Core.TypedSlot) + return SlotNumber(x.id) + end + return x + end + + for (i, stmt) in enumerate(list) + rstmt = rep(stmt, rev) + if rstmt !== stmt + list[i] = rstmt + elseif is_GotoIfNot(stmt) + stmt = stmt::Core.GotoIfNot + cond = stmt.cond + rcond = rep(cond, rev) + if rcond !== cond + list[i] = Core.GotoIfNot(rcond, stmt.dest) + end + elseif is_ReturnNode(stmt) + stmt = stmt::Core.ReturnNode + val = stmt.val + rval = rep(val, rev) + if rval !== val + list[i] = Core.ReturnNode(rval) + end + elseif isa(stmt, Expr) + replace_coretypes!(stmt; rev=rev) + end + end + return nothing +end + +function reverse_lookup_globalref!(list) + # This only handles the function in calls + for (i, stmt) in enumerate(list) + if isexpr(stmt, :(=)) + stmt = (stmt::Expr).args[2] + end + if isexpr(stmt, :call) + stmt = stmt::Expr + f = stmt.args[1] + if isa(f, QuoteNode) + f = f.value + if isa(f, Function) && !isa(f, Core.IntrinsicFunction) + ft = typeof(f) + tn = ft.name::Core.TypeName + name = String(tn.name) + if startswith(name, '#') + name = name[2:end] + end + stmt.args[1] = GlobalRef(tn.module, Symbol(name)) + end + end + end + end + return list +end diff --git a/packages/JuliaInterpreter/src/packagedef.jl b/packages/JuliaInterpreter/src/packagedef.jl new file mode 100644 index 0000000..f5e01c3 --- /dev/null +++ b/packages/JuliaInterpreter/src/packagedef.jl @@ -0,0 +1,162 @@ +using Base.Meta +import Base: +, -, convert, isless, get_world_counter +using Core: CodeInfo, SimpleVector, LineInfoNode, GotoNode, Slot, + GeneratedFunctionStub, MethodInstance, NewvarNode, TypeName + +using UUIDs +using Random +# The following are for circumventing #28, memcpy invalid instruction error, +# in Base and stdlib +using Random.DSFMT +using InteractiveUtils + +export @interpret, Compiled, Frame, root, leaf, ExprSplitter, + BreakpointRef, breakpoint, @breakpoint, breakpoints, enable, disable, remove, toggle, + debug_command, @bp, break_on, break_off, on_breakpoints_updated + +module CompiledCalls +# This module is for handling intrinsics that must be compiled (llvmcall) as well as ccalls +end + +const SlotNamesType = Vector{Symbol} + +append_any(@nospecialize x...) = append!([], Core.svec((x...)...)) + +if isdefined(Base, :mapany) + const mapany = Base.mapany +else + mapany(f, itr) = map!(f, Vector{Any}(undef, length(itr)::Int), itr) # convenient for Expr.args +end + +if isdefined(Base, :ntupleany) + const ntupleany = Base.ntupleany +else + @noinline function ntupleany(f, n) + (n >= 0) || throw(ArgumentError(string("tuple length should be ≥ 0, got ", n))) + (Any[f(i) for i = 1:n]...,) + end +end + +if !isdefined(Base, Symbol("@something")) + macro something(x...) + :(something($(map(esc, x)...))) + end +end + +include("types.jl") +include("utils.jl") +include("construct.jl") +include("localmethtable.jl") +include("interpret.jl") +include("builtins.jl") +include("optimize.jl") +include("commands.jl") +include("breakpoints.jl") + +function set_compiled_methods() + ########### + # Methods # + ########### + # Work around #28 by preventing interpretation of all Base methods that have a ccall to memcpy + push!(compiled_methods, which(vcat, (Vector,))) + push!(compiled_methods, first(methods(Base._getindex_ra))) + push!(compiled_methods, first(methods(Base._setindex_ra!))) + push!(compiled_methods, which(Base.decompose, (BigFloat,))) + push!(compiled_methods, which(DSFMT.dsfmt_jump, (DSFMT.DSFMT_state, DSFMT.GF2X))) + @static if Sys.iswindows() + push!(compiled_methods, which(InteractiveUtils.clipboard, (AbstractString,))) + end + # issue #76 + push!(compiled_methods, which(unsafe_store!, (Ptr{Any}, Any, Int))) + push!(compiled_methods, which(unsafe_store!, (Ptr, Any, Int))) + # issue #92 + push!(compiled_methods, which(objectid, Tuple{Any})) + # issue #106 --- anything that uses sigatomic_(begin|end) + push!(compiled_methods, which(flush, Tuple{IOStream})) + push!(compiled_methods, which(disable_sigint, Tuple{Function})) + push!(compiled_methods, which(reenable_sigint, Tuple{Function})) + # Signal-handling in the `print` dispatch hierarchy + push!(compiled_methods, which(Base.unsafe_write, Tuple{Base.LibuvStream,Ptr{UInt8},UInt})) + push!(compiled_methods, which(print, Tuple{IO,Any})) + push!(compiled_methods, which(print, Tuple{IO,Any,Any})) + # Libc.GetLastError() + @static if Sys.iswindows() + push!(compiled_methods, which(Base.access_env, Tuple{Function,AbstractString})) + push!(compiled_methods, which(Base._hasenv, Tuple{Vector{UInt16}})) + end + # These are currently extremely slow to interpret (https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/193) + push!(compiled_methods, which(subtypes, Tuple{Module,Type})) + push!(compiled_methods, which(subtypes, Tuple{Type})) + push!(compiled_methods, which(match, Tuple{Regex,String,Int,UInt32})) + + # Anything that ccalls jl_typeinf_begin cannot currently be handled + for finf in (Core.Compiler.typeinf_code, Core.Compiler.typeinf_ext, Core.Compiler.typeinf_type) + for m in methods(finf) + push!(compiled_methods, m) + end + end + + # Does an atomic operation via llvmcall (this fixes #354) + if isdefined(Base, :load_state_acquire) + for m in methods(Base.load_state_acquire) + push!(compiled_methods, m) + end + end + + # This is about performance, not safety (issue #462) + push!(compiled_methods, which(nameof, (Module,))) + push!(compiled_methods, which(Base.binding_module, (Module, Symbol))) + push!(compiled_methods, which(Base.unsafe_pointer_to_objref, (Ptr,))) + push!(compiled_methods, which(Vector{Int}, (UndefInitializer, Int))) + push!(compiled_methods, which(fill!, (Vector{Int8}, Int))) + + ########### + # Modules # + ########### + push!(compiled_modules, Base.Threads) +end + +_have_fma_compiled(::Type{T}) where {T} = Core.Intrinsics.have_fma(T) + +const FMA_FLOAT64 = Ref(false) +const FMA_FLOAT32 = Ref(false) +const FMA_FLOAT16 = Ref(false) + +function __init__() + set_compiled_methods() + COVERAGE[] = Base.JLOptions().code_coverage + # If we interpret into Core.Compiler, we need to take precautions to avoid needing + # inference of JuliaInterpreter methods in the middle of a `ccall(:jl_typeinf_begin, ...)` + # block. + # for (sym, RT, AT) in ((:jl_typeinf_begin, Cvoid, ()), + # (:jl_typeinf_end, Cvoid, ()), + # (:jl_isa_compileable_sig, Int32, (Any, Any)), + # (:jl_compress_ast, Any, (Any, Any)), + # # (:jl_set_method_inferred, Ref{Core.CodeInstance}, (Any, Any, Any, Any, Int32, UInt, UInt)), + # (:jl_method_instance_add_backedge, Cvoid, (Any, Any)), + # (:jl_method_table_add_backedge, Cvoid, (Any, Any, Any)), + # (:jl_new_code_info_uninit, Ref{CodeInfo}, ()), + # (:jl_uncompress_argnames, Vector{Symbol}, (Any,)), + # (:jl_get_tls_world_age, UInt, ()), + # (:jl_call_in_typeinf_world, Any, (Ptr{Ptr{Cvoid}}, Cint)), + # (:jl_value_ptr, Any, (Ptr{Cvoid},)), + # (:jl_value_ptr, Ptr{Cvoid}, (Any,))) + # fname = Symbol(:ccall_, sym) + # qsym = QuoteNode(sym) + # argnames = [Symbol(:arg_, string(i)) for i = 1:length(AT)] + # TAT = Expr(:tuple, [parametric_type_to_expr(t) for t in AT]...) + # def = :($fname($(argnames...)) = ccall($qsym, $RT, $TAT, $(argnames...))) + # f = Core.eval(Core.Compiler, def) + # compiled_calls[(qsym, RT, Core.svec(AT...), Core.Compiler)] = f + # precompile(f, AT) + # end + + @static if isdefined(Base, :have_fma) + FMA_FLOAT64[] = _have_fma_compiled(Float64) + FMA_FLOAT32[] = _have_fma_compiled(Float32) + FMA_FLOAT16[] = _have_fma_compiled(Float16) + end +end + +include("precompile.jl") +_precompile_() diff --git a/packages/JuliaInterpreter/src/precompile.jl b/packages/JuliaInterpreter/src/precompile.jl new file mode 100644 index 0000000..f929cde --- /dev/null +++ b/packages/JuliaInterpreter/src/precompile.jl @@ -0,0 +1,19 @@ +module var"#Internal" +public(x::String) = false +end + +function _precompile_() + ccall(:jl_generating_output, Cint, ()) == 1 || return nothing + @interpret sum(rand(10)) + expr = quote + public(x::Integer) = true + module Private + private(y::String) = false + end + const threshold = 0.1 + end + for (mod, ex) in ExprSplitter(var"#Internal", expr) + frame = Frame(mod, ex) + debug_command(frame, :c, true) + end +end diff --git a/packages/JuliaInterpreter/src/types.jl b/packages/JuliaInterpreter/src/types.jl new file mode 100644 index 0000000..6bb16af --- /dev/null +++ b/packages/JuliaInterpreter/src/types.jl @@ -0,0 +1,488 @@ +""" +`Compiled` is a trait indicating that any `:call` expressions should be evaluated +using Julia's normal compiled-code evaluation. The alternative is to pass `stack=Frame[]`, +which will cause all calls to be evaluated via the interpreter. +""" +struct Compiled end +Base.similar(::Compiled, sz) = Compiled() # to support similar(stack, 0) + +# A type used transiently in renumbering CodeInfo SSAValues (to distinguish a new SSAValue from an old one) +struct NewSSAValue + id::Int +end + +# Our own replacements for Core types. We need to do this to ensure we can tell the difference +# between "data" (Core types) and "code" (our types) if we step into Core.Compiler +struct SSAValue + id::Int +end +struct SlotNumber + id::Int +end + +Base.show(io::IO, ssa::SSAValue) = print(io, "%J", ssa.id) +Base.show(io::IO, slot::SlotNumber) = print(io, "_J", slot.id) + +# Breakpoint support +truecondition(frame) = true +falsecondition(frame) = false +const break_on_error = Ref(false) +const break_on_throw = Ref(false) + +""" + BreakpointState(isactive=true, condition=JuliaInterpreter.truecondition) + +`BreakpointState` represents a breakpoint at a particular statement in +a `FrameCode`. `isactive` indicates whether the breakpoint is currently +[`enable`](@ref)d or [`disable`](@ref)d. `condition` is a function that accepts +a single `Frame`, and `condition(frame)` must return either +`true` or `false`. Execution will stop at a breakpoint only if `isactive` +and `condition(frame)` both evaluate as `true`. The default `condition` always +returns `true`. + +To create these objects, see [`breakpoint`](@ref). +""" +struct BreakpointState + isactive::Bool + condition::Function +end +BreakpointState(isactive::Bool) = BreakpointState(isactive, truecondition) +BreakpointState() = BreakpointState(true) + +function breakpointchar(bps::BreakpointState) + if bps.isactive + return bps.condition === truecondition ? 'b' : 'c' # unconditional : conditional + end + return bps.condition === falsecondition ? ' ' : 'd' # no breakpoint : disabled +end + +abstract type AbstractFrameInstance end +mutable struct DispatchableMethod + next::Union{Nothing,DispatchableMethod} # linked-list representation + frameinstance::Union{Compiled, AbstractFrameInstance} # really a Union{Compiled, FrameInstance} but we have a cyclic dependency + sig::Type # for speed of matching, this is a *concrete* signature. `sig <: frameinstance.framecode.scope.sig` +end + +# 0: none +# 1: user +# 2: all +const COVERAGE = Ref{Int8}() +function do_coverage(m::Module) + COVERAGE[] == 2 && return true + if COVERAGE[] == 1 + root = Base.moduleroot(m) + return root !== Base && root !== Core + end + return false +end + +""" +`FrameCode` holds static information about a method or toplevel code. +One `FrameCode` can be shared by many calling `Frame`s. + +Important fields: +- `scope`: the `Method` or `Module` in which this frame is to be evaluated. +- `src`: the `CodeInfo` object storing (optimized) lowered source code. +- `methodtables`: a vector, each entry potentially stores a "local method table" for the corresponding + `:call` expression in `src` (undefined entries correspond to statements that do not + contain `:call` expressions). +- `used`: a `BitSet` storing the list of SSAValues that get referenced by later statements. +""" +struct FrameCode + scope::Union{Method,Module} + src::CodeInfo + methodtables::Vector{Union{Compiled,DispatchableMethod}} # line-by-line method tables for generic-function :call Exprs + breakpoints::Vector{BreakpointState} + slotnamelists::Dict{Symbol,Vector{Int}} + used::BitSet + generator::Bool # true if this is for the expression-generator of a @generated function + report_coverage::Bool + unique_files::Set{Symbol} +end + +const BREAKPOINT_EXPR = :($(QuoteNode(getproperty))($JuliaInterpreter, :__BREAKPOINT_MARKER__)) +function is_breakpoint_expr(ex::Expr) + # Sadly, comparing QuoteNodes calls isequal(::Any, ::Any), and === seems not to work. + # To avoid invalidations, do it the hard way. + ex.head === :call || return false + length(ex.args) === 3 || return false + (q = ex.args[1]; isa(q, QuoteNode) && q.value === getproperty) || return false + ex.args[2] === JuliaInterpreter || return false + q = ex.args[3] + return isa(q, QuoteNode) && q.value === :__BREAKPOINT_MARKER__ +end +function FrameCode(scope, src::CodeInfo; generator=false, optimize=true) + if optimize + src, methodtables = optimize!(copy(src), scope) + else + src = replace_coretypes!(copy(src)) + methodtables = Vector{Union{Compiled,DispatchableMethod}}(undef, length(src.code)) + end + breakpoints = Vector{BreakpointState}(undef, length(src.code)) + for (i, pc_expr) in enumerate(src.code) + if isa(pc_expr, Expr) && is_breakpoint_expr(pc_expr) + breakpoints[i] = BreakpointState() + src.code[i] = nothing + end + end + slotnamelists = Dict{Symbol,Vector{Int}}() + for (i, sym) in enumerate(src.slotnames) + list = get(slotnamelists, sym, Int[]) + slotnamelists[sym] = push!(list, i) + end + used = find_used(src) + report_coverage = do_coverage(moduleof(scope)) + + lt = linetable(src) + unique_files = Set{Symbol}() + for entry in lt + push!(unique_files, entry.file) + end + + framecode = FrameCode(scope, src, methodtables, breakpoints, slotnamelists, used, generator, report_coverage, unique_files) + if scope isa Method + for bp in _breakpoints + # Manual union splitting + if bp isa BreakpointSignature + add_breakpoint_if_match!(framecode, bp) + elseif bp isa BreakpointFileLocation + add_breakpoint_if_match!(framecode, bp) + else + error("unhandled breakpoint type") + end + end + else + for bp in _breakpoints + if bp isa BreakpointFileLocation + add_breakpoint_if_match!(framecode, bp) + end + end + end + + return framecode +end + +nstatements(framecode::FrameCode) = length(framecode.src.code) + +Base.show(io::IO, framecode::FrameCode) = print_framecode(io, framecode) + +""" +`FrameInstance` represents a method specialized for particular argument types. + +Fields: +- `framecode`: the [`FrameCode`](@ref) for the method. +- `sparam_vals`: the static parameter values for the method. +""" +struct FrameInstance <: AbstractFrameInstance + framecode::FrameCode + sparam_vals::SimpleVector + enter_generated::Bool +end + +Base.show(io::IO, instance::FrameInstance) = + print(io, "FrameInstance(", scopeof(instance.framecode), ", ", instance.sparam_vals, ", ", instance.enter_generated, ')') + +""" +`FrameData` holds the arguments, local variables, and intermediate execution state +in a particular call frame. + +Important fields: +- `locals`: a vector containing the input arguments and named local variables for this frame. + The indexing corresponds to the names in the `slotnames` of the src. Use [`locals`](@ref) + to extract the current value of local variables. +- `ssavalues`: a vector containing the + [Static Single Assignment](https://en.wikipedia.org/wiki/Static_single_assignment_form) + values produced at the current state of execution. +- `sparams`: the static type parameters, e.g., for `f(x::Vector{T}) where T` this would store + the value of `T` given the particular input `x`. +- `exception_frames`: a list of indexes to `catch` blocks for handling exceptions within + the current frame. The active handler is the last one on the list. +- `last_exception`: the exception `throw`n by this frame or one of its callees. +""" +struct FrameData + locals::Vector{Union{Nothing,Some{Any}}} + ssavalues::Vector{Any} + sparams::Vector{Any} + exception_frames::Vector{Int} + last_exception::Base.RefValue{Any} + caller_will_catch_err::Bool + last_reference::Vector{Int} + callargs::Vector{Any} # a temporary for processing arguments of :call exprs +end + +""" + _INACTIVE_EXCEPTION + +Represents a case where no exceptions are thrown yet. +End users will not see this singleton type, otherwise it usually means there is missing +error handling in the interpretation process. +""" +struct _INACTIVE_EXCEPTION end + +""" +`Frame` represents the current execution state in a particular call frame. +Fields: +- `framecode`: the [`FrameCode`](@ref) for this frame. +- `framedata`: the [`FrameData`](@ref) for this frame. +- `pc`: the program counter (integer index of the next statment to be evaluated) for this frame. +- `caller`: the parent caller of this frame, or `nothing`. +- `callee`: the frame called by this one, or `nothing`. + +The `Base` functions `show_backtrace` and `display_error` are overloaded such that +`show_backtrace(io::IO, frame::Frame)` and `display_error(io::IO, er, frame::Frame)` +shows a backtrace or error, respectively, in a similar way as to how Base shows +them. +""" +mutable struct Frame + framecode::FrameCode + framedata::FrameData + pc::Int + assignment_counter::Int64 + caller::Union{Frame,Nothing} + callee::Union{Frame,Nothing} + last_codeloc::Int32 +end +function Frame(framecode::FrameCode, framedata::FrameData, pc=1, caller=nothing) + if length(junk_frames) > 0 + frame = pop!(junk_frames) + frame.framecode = framecode + frame.framedata = framedata + frame.pc = pc + frame.assignment_counter = 1 + frame.caller = caller + frame.callee = nothing + frame.last_codeloc = 0 + return frame + else + return Frame(framecode, framedata, pc, 1, caller, nothing, 0) + end +end +""" + frame = Frame(mod::Module, src::CodeInfo; kwargs...) + +Construct a `Frame` to evaluate `src` in module `mod`. +""" +function Frame(mod::Module, src::CodeInfo; kwargs...) + framecode = FrameCode(mod, src; kwargs...) + return Frame(framecode, prepare_framedata(framecode, [])) +end +""" + frame = Frame(mod::Module, ex::Expr) + +Construct a `Frame` to evaluate `ex` in module `mod`. + +This constructor can error, for example if lowering `ex` results in an `:error` or `:incomplete` +expression, or if it otherwise fails to return a `:thunk`. +""" +function Frame(mod::Module, ex::Expr) + lwr = Meta.lower(mod, ex) + isexpr(lwr, :thunk) && return Frame(mod, lwr.args[1]) + if isexpr(lwr, :error) || isexpr(lwr, :incomplete) + throw(ArgumentError("lowering returned an error, $lwr")) + end + throw(ArgumentError("lowering did not return a `:thunk` expression, got $lwr")) +end + +caller(frame) = frame.caller +callee(frame) = frame.callee + +function traverse(f, frame) + while f(frame) !== nothing + frame = f(frame) + end + return frame +end + +""" + rframe = root(frame) + +Return the initial frame in the call stack. +""" +root(frame) = traverse(caller, frame) + +""" + lframe = leaf(frame) + +Return the deepest callee in the call stack. +""" +leaf(frame) = traverse(callee, frame) + +function Base.show(io::IO, frame::Frame) + frame_loc = CodeTracking.replace_buildbot_stdlibpath(repr(scopeof(frame))) + println(io, "Frame for ", frame_loc) + pc = frame.pc + ns = nstatements(frame.framecode) + range = get(io, :limit, false) ? (max(1, pc-2):min(ns, pc+2)) : (1:ns) + first(range) > 1 && println(io, "⋮") + print_framecode(io, frame.framecode; pc=pc, range=range) + last(range) < ns && print(io, "\n⋮") + print_vars(IOContext(io, :limit=>true, :compact=>true), locals(frame)) + if caller(frame) !== nothing + print(io, "\ncaller: ", scopeof(caller(frame))) + end + if callee(frame) !== nothing + print(io, "\ncallee: ", scopeof(callee(frame))) + end +end + +""" +`Variable` is a struct representing a variable with an asigned value. +By calling the function [`locals`](@ref) on a [`Frame`](@ref) a +`Vector` of `Variable`'s is returned. + +Important fields: +- `value::Any`: the value of the local variable. +- `name::Symbol`: the name of the variable as given in the source code. +- `isparam::Bool`: if the variable is a type parameter, for example `T` in `f(x::T) where {T} = x`. +- `is_captured_closure::Bool`: if the variable has been captured by a closure +""" +struct Variable + value::Any + name::Symbol + isparam::Bool + is_captured_closure::Bool +end +Variable(value, name) = Variable(value, name, false, false) +Variable(value, name, isparam) = Variable(value, name, isparam, false) +Base.show(io::IO, var::Variable) = (print(io, var.name, " = "); show(io,var.value)) +Base.isequal(var1::Variable, var2::Variable) = + var1.value == var2.value && var1.name === var2.name && var1.isparam == var2.isparam && + var1.is_captured_closure == var2.is_captured_closure + +# A type that is unique to this package for which there are no valid operations +struct Unassigned end + +""" + BreakpointRef(framecode, stmtidx) + BreakpointRef(framecode, stmtidx, err) + +A reference to a breakpoint at a particular statement index `stmtidx` in `framecode`. +If the break was due to an error, supply that as well. + +Commands that execute complex control-flow (e.g., `next_line!`) may also return a +`BreakpointRef` to indicate that the execution stack switched frames, even when no +breakpoint has been set at the corresponding statement. +""" +struct BreakpointRef + framecode::FrameCode + stmtidx::Int + err +end +BreakpointRef(framecode, stmtidx) = BreakpointRef(framecode, stmtidx, nothing) +Base.getindex(bp::BreakpointRef) = bp.framecode.breakpoints[bp.stmtidx] +Base.setindex!(bp::BreakpointRef, isactive::Bool) = + bp.framecode.breakpoints[bp.stmtidx] = BreakpointState(isactive, bp[].condition) + +function Base.show(io::IO, bp::BreakpointRef) + if checkbounds(Bool, bp.framecode.breakpoints, bp.stmtidx) + lineno = linenumber(bp.framecode, bp.stmtidx) + print(io, "breakpoint(", bp.framecode.scope, ", line ", lineno) + else + print(io, "breakpoint(", bp.framecode.scope, ", %", bp.stmtidx) + end + if bp.err !== nothing + print(io, ", ", bp.err) + end + print(io, ')') +end + +# Possible types for breakpoint condition +const Condition = Union{Nothing,Expr,Tuple{Module,Expr}} + +""" +`AbstractBreakpoint` is the abstract type that is the supertype for breakpoints. Currently, +the concrete breakpoint types [`BreakpointSignature`](@ref) and [`BreakpointFileLocation`](@ref) +exist. + +Common fields shared by the concrete breakpoints: + +- `condition::Union{Nothing,Expr,Tuple{Module,Expr}}`: the condition when the breakpoint applies . + `nothing` means unconditionally, otherwise when the `Expr` (optionally in `Module`). +- `enabled::Ref{Bool}`: If the breakpoint is enabled (should not be directly modified, use [`enable()`](@ref) or [`disable()`](@ref)). +- `instances::Vector{BreakpointRef}`: All the [`BreakpointRef`](@ref) that the breakpoint has applied to. +- `line::Int` The line of the breakpoint (equal to 0 if unset). + +See [`BreakpointSignature`](@ref) and [`BreakpointFileLocation`](@ref) for additional fields in the concrete types. +""" +abstract type AbstractBreakpoint end + +same_location(::AbstractBreakpoint, ::AbstractBreakpoint) = false + +function print_bp_condition(io::IO, cond::Condition) + if cond !== nothing + if isa(cond, Tuple{Module, Expr}) && (expr = expr[2]) + cond = (cond[1], Base.remove_linenums!(copy(cond[2]))) + elseif isa(cond, Expr) + cond = Base.remove_linenums!(copy(cond)) + end + print(io, " ", cond) + end +end + +""" +A `BreakpointSignature` is a breakpoint that is set on methods or functions. + +Fields: + +- `f::Union{Method, Function, Type}`: A method or function that the breakpoint should apply to. +- `sig::Union{Nothing, Type}`: if `f` is a `Method`, always equal to `nothing`. Otherwise, contains the method signature + as a tuple type for what methods the breakpoint should apply to. + +For common fields shared by all breakpoints, see [`AbstractBreakpoint`](@ref). +""" +struct BreakpointSignature <: AbstractBreakpoint + f::Union{Method, Base.Callable} + sig::Union{Nothing, Type} + line::Int # 0 is a sentinel for first statement + condition::Condition + enabled::Ref{Bool} + instances::Vector{BreakpointRef} +end +same_location(bp2::BreakpointSignature, bp::BreakpointSignature) = + bp2.f == bp.f && bp2.sig == bp.sig && bp2.line == bp.line +function Base.show(io::IO, bp::BreakpointSignature) + print(io, bp.f) + bbsig = bp.sig + if bbsig !== nothing + print(io, '(', join("::" .* string.(bbsig.types), ", "), ')') + end + if bp.line !== 0 + print(io, ":", bp.line) + end + print_bp_condition(io, bp.condition) + if !bp.enabled[] + print(io, " [disabled]") + end +end + +""" +A `BreakpointFileLocation` is a breakpoint that is set on a line in a file. + +Fields: +- `path::String`: The literal string that was used to create the breakpoint, e.g. `"path/file.jl"`. +- `abspath`::String: The absolute path to the file when the breakpoint was created, e.g. `"/Users/Someone/path/file.jl"`. + +For common fields shared by all breakpoints, see [`AbstractBreakpoint`](@ref). +""" +struct BreakpointFileLocation <: AbstractBreakpoint + # Both the input path and the absolute path is stored to handle the case + # where a user sets a breakpoint on a relative path e.g. `../foo.jl`. The absolute path is needed + # to handle the case where the current working directory change, and + # the input path is needed to do "partial path matches", e.g match "src/foo.jl" against + # "Package/src/foo.jl". + path::String + abspath::String + line::Int + condition::Condition + enabled::Ref{Bool} + instances::Vector{BreakpointRef} +end +same_location(bp2::BreakpointFileLocation, bp::BreakpointFileLocation) = + bp2.path == bp.path && bp2.abspath == bp.abspath && bp2.line == bp.line +function Base.show(io::IO, bp::BreakpointFileLocation) + print(io, bp.path, ':', bp.line) + print_bp_condition(io, bp.condition) + if !bp.enabled[] + print(io, " [disabled]") + end +end diff --git a/packages/JuliaInterpreter/src/utils.jl b/packages/JuliaInterpreter/src/utils.jl new file mode 100644 index 0000000..e6166ca --- /dev/null +++ b/packages/JuliaInterpreter/src/utils.jl @@ -0,0 +1,791 @@ +## Simple utils + +# Note: to avoid dynamic dispatch, many of these are coded as a single method using isa statements + +function scopeof(@nospecialize(x))::Union{Method,Module} + (isa(x, Method) || isa(x, Module)) && return x + isa(x, FrameCode) && return x.scope + isa(x, Frame) && return x.framecode.scope + error("unknown scope for ", x) +end + +function moduleof(@nospecialize(x)) + s = scopeof(x) + return isa(s, Module) ? s : s.module +end + +function Base.nameof(frame::Frame) + s = frame.framecode.scope + isa(s, Method) ? s.name : nameof(s) +end + +_Typeof(x) = isa(x, Type) ? Type{x} : typeof(x) + +function to_function(@nospecialize(x)) + isa(x, GlobalRef) ? getfield(x.mod, x.name) : x +end + +""" + method = whichtt(tt) + +Like `which` except it operates on the complete tuple-type `tt`, +and doesn't throw when there is no matching method. +""" +function whichtt(@nospecialize(tt)) + # TODO: provide explicit control over world age? In case we ever need to call "old" methods. + @static if VERSION ≥ v"1.8-beta2" + # branch on https://github.com/JuliaLang/julia/pull/44515 + # for now, actual code execution doesn't ever need to consider overlayed method table + match, _ = Core.Compiler._findsup(tt, nothing, get_world_counter()) + match === nothing && return nothing + return match.method + else + m = ccall(:jl_gf_invoke_lookup, Any, (Any, UInt), tt, get_world_counter()) + m === nothing && return nothing + isa(m, Method) && return m + return m.func::Method + end +end + +instantiate_type_in_env(arg, spsig, spvals) = + ccall(:jl_instantiate_type_in_env, Any, (Any, Any, Ptr{Any}), arg, spsig, spvals) + +function sparam_syms(meth::Method) + s = Symbol[] + sig = meth.sig + while sig isa UnionAll + push!(s, Symbol(sig.var.name)) + sig = sig.body + end + return s +end + +separate_kwargs(args...; kwargs...) = (args, values(kwargs)) + +pc_expr(src::CodeInfo, pc) = src.code[pc] +pc_expr(framecode::FrameCode, pc) = pc_expr(framecode.src, pc) +pc_expr(frame::Frame, pc) = pc_expr(frame.framecode, pc) +pc_expr(frame::Frame) = pc_expr(frame, frame.pc) + +function find_used(code::CodeInfo) + used = BitSet() + stmts = code.code + for stmt in stmts + scan_ssa_use!(used, stmt) + end + return used +end + +function scan_ssa_use!(used::BitSet, @nospecialize(stmt)) + if isa(stmt, SSAValue) + push!(used, stmt.id) + end + iter = Core.Compiler.userefs(stmt) + iterval = Core.Compiler.iterate(iter) + while iterval !== nothing + useref, state = iterval + val = Core.Compiler.getindex(useref) + if isa(val, SSAValue) + push!(used, val.id) + end + iterval = Core.Compiler.iterate(iter, state) + end +end + +function hasarg(predicate, args) + predicate(args) && return true + for a in args + predicate(a) && return true + if isa(a, Expr) + hasarg(predicate, a.args) && return true + elseif isa(a, QuoteNode) + predicate(a.value) && return true + elseif isa(a, GlobalRef) + predicate(a.name) && return true + end + end + return false +end + +function wrap_params(expr, sparams::Vector{Symbol}) + isempty(sparams) && return expr + params = [] + for p in sparams + hasarg(isidentical(p), expr.args) && push!(params, p) + end + return isempty(params) ? expr : Expr(:where, expr, params...) +end + +function scopename(tn::TypeName) + modpath = Base.fullname(tn.module) + if isa(modpath, Tuple{Symbol}) + return Expr(:., modpath[1], QuoteNode(tn.name)) + end + ex = Expr(:., modpath[end-1], QuoteNode(modpath[end])) + for i = length(modpath)-2:-1:1 + ex = Expr(:., modpath[i], ex) + end + return Expr(:., ex, QuoteNode(tn.name)) +end + +## Predicates + +isidentical(x) = Base.Fix2(===, x) # recommended over isequal(::Symbol) since it cannot be invalidated + +# is_goto_node(@nospecialize(node)) = isa(node, GotoNode) || isexpr(node, :gotoifnot) + +if isdefined(Core, :GotoIfNot) + is_GotoIfNot(@nospecialize(node)) = isa(node, Core.GotoIfNot) + is_gotoifnot(@nospecialize(node)) = is_GotoIfNot(node) +else + is_GotoIfNot(@nospecialize(node)) = false + is_gotoifnot(@nospecialize(node)) = isexpr(node, :gotoifnot) +end + +if isdefined(Core, :ReturnNode) + is_ReturnNode(@nospecialize(node)) = isa(node, Core.ReturnNode) + is_return(@nospecialize(node)) = is_ReturnNode(node) + get_return_node(@nospecialize(node)) = (node::Core.ReturnNode).val +else + is_ReturnNode(@nospecialize(node)) = false + is_return(@nospecialize(node)) = isexpr(node, :return) + get_return_node(@nospecialize(node)) = node.args[1] +end + +is_loc_meta(@nospecialize(expr), @nospecialize(kind)) = isexpr(expr, :meta) && length(expr.args) >= 1 && expr.args[1] === kind + +""" + is_global_ref(g, mod, name) + +Tests whether `g` is equal to `GlobalRef(mod, name)`. +""" +is_global_ref(@nospecialize(g), mod::Module, name::Symbol) = isa(g, GlobalRef) && g.mod === mod && g.name == name + +is_quotenode(@nospecialize(q), @nospecialize(val)) = isa(q, QuoteNode) && q.value == val +is_quotenode_egal(@nospecialize(q), @nospecialize(val)) = isa(q, QuoteNode) && q.value === val + +function is_quoted_type(@nospecialize(a), name::Symbol) + if isa(a, QuoteNode) + T = a.value + if isa(T, UnionAll) + T = Base.unwrap_unionall(T) + end + isa(T, DataType) && return T.name.name === name + end + return false +end + +function is_function_def(@nospecialize(ex)) + (isexpr(ex, :(=)) && isexpr(ex.args[1], :call)) || isexpr(ex, :function) +end + +function is_call(@nospecialize(node)) + isexpr(node, :call) || + (isexpr(node, :(=)) && (isexpr(node.args[2], :call))) +end + +is_call_or_return(@nospecialize(node)) = is_call(node) || is_return(node) + +is_dummy(bpref::BreakpointRef) = bpref.stmtidx == 0 && bpref.err === nothing + +function unpack_splatcall(stmt) + if isexpr(stmt, :call) && length(stmt.args) >= 3 && is_quotenode_egal(stmt.args[1], Core._apply_iterate) + return true, stmt.args[3] + end + return false, nothing +end + +function is_bodyfunc(@nospecialize(arg)) + if isa(arg, QuoteNode) + arg = arg.value + end + if isa(arg, Function) + fname = String((typeof(arg).name::Core.TypeName).name) + return startswith(fname, "##") && match(r"#\d+$", fname) !== nothing + end + return false +end + +""" +Determine whether we are calling a function for which the current function +is a wrapper (either because of optional arguments or because of keyword arguments). +""" +function is_wrapper_call(@nospecialize(expr)) + isexpr(expr, :(=)) && (expr = expr.args[2]) + isexpr(expr, :call) && any(x->x==SlotNumber(1), expr.args) +end + +is_generated(meth::Method) = isdefined(meth, :generator) + +""" + is_doc_expr(ex) + +Test whether expression `ex` is a `@doc` expression. +""" +function is_doc_expr(@nospecialize(ex)) + docsym = Symbol("@doc") + if isexpr(ex, :macrocall) + ex::Expr + length(ex.args) == 4 || return false + a = ex.args[1] + is_global_ref(a, Core, docsym) && return true + isa(a, Symbol) && a == docsym && return true + if isexpr(a, :.) + mod, name = (a::Expr).args[1], (a::Expr).args[2] + return mod === :Core && isa(name, QuoteNode) && name.value === docsym + end + end + return false +end + +is_leaf(frame::Frame) = frame.callee === nothing + +function is_vararg_type(x) + @static if isa(Vararg, Type) + if isa(x, Type) + (x <: Vararg && !(x <: Union{})) && return true + if isa(x, UnionAll) + x = Base.unwrap_unionall(x) + end + return isa(x, DataType) && nameof(x) === :Vararg + end + else + return isa(x, typeof(Vararg)) + end + return false +end + +## Location info + +# These getters improve inference since fieldtype(CodeInfo, :linetable) +# and fieldtype(CodeInfo, :codelocs) are both Any +const LineTypes = Union{LineNumberNode,Core.LineInfoNode} +function linetable(arg) + if isa(arg, Frame) + arg = arg.framecode + end + if isa(arg, FrameCode) + arg = arg.src + end + return (arg::CodeInfo).linetable::Union{Vector{Core.LineInfoNode},Vector{Any}} # issue #264 +end +_linetable(list::Vector, i::Integer) = list[i]::Union{Expr,LineTypes} +function linetable(arg, i::Integer; macro_caller::Bool=false)::Union{Expr,LineTypes} + lt = linetable(arg) + lineinfo = _linetable(lt, i) + if macro_caller + while lineinfo isa Core.LineInfoNode && lineinfo.method === Symbol("macro expansion") && lineinfo.inlined_at != 0 + lineinfo = _linetable(lt, lineinfo.inlined_at) + end + end + return lineinfo +end + +function codelocs(arg) + if isa(arg, Frame) + arg = arg.framecode + end + if isa(arg, FrameCode) + arg = arg.src + end + return (arg::CodeInfo).codelocs::Vector{Int32} +end +codelocs(arg, i::Integer) = codelocs(arg)[i] # for consistency with linetable (but no extra benefit here) + +function lineoffset(framecode::FrameCode) + offset = 0 + scope = framecode.scope + if isa(scope, Method) + _, line1 = whereis(scope) + offset = line1 - scope.line + end + return offset +end + +function getline(ln::Union{LineTypes,Expr}) + _getline(ln::LineTypes) = ln.line + _getline(ln::Expr) = ln.args[1] # assuming ln.head === :line + return Int(_getline(ln))::Int +end +function getfile(ln::Union{LineTypes,Expr}) + _getfile(ln::LineTypes) = ln.file::Symbol + _getfile(ln::Expr) = ln.args[2]::Symbol # assuming ln.head === :line + return CodeTracking.maybe_fixup_stdlib_path(String(_getfile(ln))) +end + +function firstline(ex::Expr) + for a in ex.args + isa(a, LineNumberNode) && return a + if isa(a, Expr) + line = firstline(a) + isa(line, LineNumberNode) && return line + end + end + return nothing +end + +""" + loc = whereis(frame, pc::Int=frame.pc; macro_caller=false) + +Return the file and line number for `frame` at `pc`. If this cannot be +determined, `loc == nothing`. Otherwise `loc == (filepath, line)`. + +By default, any statements expanded from a macro are attributed to the macro +definition, but with`macro_caller=true` you can obtain the location within the +method that issued the macro. +""" +function CodeTracking.whereis(framecode::FrameCode, pc::Int; kwargs...) + codeloc = codelocation(framecode.src, pc) + codeloc == 0 && return nothing + lineinfo = linetable(framecode, codeloc; kwargs...) + m = framecode.scope + return isa(m, Method) ? whereis(lineinfo, m) : (getfile(lineinfo), getline(lineinfo)) +end +CodeTracking.whereis(frame::Frame, pc::Int=frame.pc; kwargs...) = whereis(frame.framecode, pc; kwargs...) + +""" + line = linenumber(framecode, pc) + +Return the "static" line number at statement index `pc`. The static line number +is the location at the time the method was most recently defined. +See [`CodeTracking.whereis`](@ref) for dynamic line information. +""" +function linenumber(framecode::FrameCode, pc) + codeloc = codelocation(framecode.src, pc) + codeloc == 0 && return nothing + return getline(linetable(framecode, codeloc)) +end +linenumber(frame::Frame, pc=frame.pc) = linenumber(frame.framecode, pc) + +function getfile(framecode::FrameCode, pc) + codeloc = codelocation(framecode.src, pc) + codeloc == 0 && return nothing + return getfile(linetable(framecode, codeloc)) +end +getfile(frame::Frame, pc=frame.pc) = getfile(frame.framecode, pc) + +function codelocation(code::CodeInfo, idx::Int) + codeloc = codelocs(code)[idx] + while codeloc == 0 && (code.code[idx] === nothing || isexpr(code.code[idx], :meta)) && idx < length(code.code) + idx += 1 + codeloc = codelocs(code)[idx] + end + return codeloc +end + +function compute_corrected_linerange(method::Method) + _, line1 = whereis(method) + offset = line1 - method.line + @assert !is_generated(method) + src = JuliaInterpreter.get_source(method) + lastline = linetable(src)[end]::LineTypes + return line1:getline(lastline) + offset +end + +function compute_linerange(framecode) + getline(linetable(framecode, 1)):getline(last(linetable(framecode))) +end + +function statementnumbers(framecode::FrameCode, line::Integer, file::Symbol) + # Check to see if this framecode really contains that line. Methods that fill in a default positional argument, + # keyword arguments, and @generated sections may not contain the line. + scope = framecode.scope + offset = if scope isa Method + method = scope + _, line1 = whereis(method) + Int(line1 - method.line) + else + 0 + end + + lt = linetable(framecode) + + # Check if the exact line number exist + idxs = findall(entry::Union{LineInfoNode,LineNumberNode} -> entry.line + offset == line && entry.file == file, lt) + locs = codelocs(framecode) + if !isempty(idxs) + stmtidxs = Int[] + stmtidx = 1 + while stmtidx <= length(locs) + loc = locs[stmtidx] + if loc in idxs + push!(stmtidxs, stmtidx) + stmtidx += 1 + # Skip continous statements that are on the same line + while stmtidx <= length(locs) && loc == locs[stmtidx] + stmtidx += 1 + end + else + stmtidx += 1 + end + end + return stmtidxs + end + + + # If the exact line number does not exist in the line table, take the one that is closest after that line + # restricted to the line range of the current scope. + scope = framecode.scope + range = (scope isa Method && !is_generated(scope)) ? compute_corrected_linerange(scope) : compute_linerange(framecode) + if line in range + closest = nothing + closest_idx = nothing + for (i, entry) in enumerate(lt) + entry = entry::Union{LineInfoNode,LineNumberNode} + if entry.file == file && entry.line in range && entry.line >= line + if closest === nothing + closest = entry + closest_idx = i + else + if entry.line < closest.line + closest = entry + closest_idx = i + end + end + end + end + if closest_idx !== nothing + idx = let closest_idx=closest_idx # julia #15276 + findfirst(i-> i==closest_idx, locs) + end + return idx === nothing ? nothing : Int[idx] + end + end + + return nothing +end + +## Printing + +function framecode_lines(src::CodeInfo) + buf = IOBuffer() + if isdefined(Base.IRShow, :show_ir_stmt) + lines = String[] + src = replace_coretypes!(copy(src); rev=true) + reverse_lookup_globalref!(src.code) + io = IOContext(buf, :displaysize => displaysize(stdout), + :SOURCE_SLOTNAMES => Base.sourceinfo_slotnames(src)) + used = BitSet() + cfg = Core.Compiler.compute_basic_blocks(src.code) + for stmt in src.code + Core.Compiler.scan_ssa_use!(push!, used, stmt) + end + line_info_preprinter = Base.IRShow.lineinfo_disabled + line_info_postprinter = Base.IRShow.default_expr_type_printer + bb_idx = 1 + for idx = 1:length(src.code) + bb_idx = Base.IRShow.show_ir_stmt(io, src, idx, line_info_preprinter, line_info_postprinter, used, cfg, bb_idx) + push!(lines, chomp(String(take!(buf)))) + end + return lines + end + show(buf, src) + code = filter!(split(String(take!(buf)), '\n')) do line + !(line == "CodeInfo(" || line == ")" || isempty(line) || occursin("within `", line)) + end + code .= replace.(code, Ref(r"\$\(QuoteNode\((.+?)\)\)" => s"\1")) + return code +end +framecode_lines(framecode::FrameCode) = framecode_lines(framecode.src) + +breakpointchar(framecode, stmtidx) = + isassigned(framecode.breakpoints, stmtidx) ? breakpointchar(framecode.breakpoints[stmtidx]) : ' ' + +function print_framecode(io::IO, framecode::FrameCode; pc=0, range=1:nstatements(framecode), kwargs...) + iscolor = get(io, :color, false) + ndstmt = ndigits(nstatements(framecode)) + lt = linetable(framecode) + offset = lineoffset(framecode) + ndline = isempty(lt) ? 0 : ndigits(getline(lt[end]) + offset) + nullline = " "^ndline + src = copy(framecode.src) + replace_coretypes!(src; rev=true) + code = framecode_lines(src) + isfirst = true + for (stmtidx, stmtline) in enumerate(code) + stmtidx ∈ range || continue + bpc = breakpointchar(framecode, stmtidx) + isfirst || print(io, '\n') + isfirst = false + print(io, bpc, ' ') + if iscolor + color = stmtidx == pc ? Base.warn_color() : :normal + printstyled(io, lpad(stmtidx, ndstmt); color=color, kwargs...) + else + print(io, lpad(stmtidx, ndstmt), stmtidx == pc ? '*' : ' ') + end + line = linenumber(framecode, stmtidx) + print(io, ' ', line === nothing ? nullline : lpad(line, ndline), " ", stmtline) + end +end + +""" + local_variables = locals(frame::Frame)::Vector{Variable} + +Return the local variables as a vector of [`Variable`](@ref). +""" +function locals(frame::Frame) + vars, var_counter = Variable[], Int[] + varlookup = Dict{Symbol,Int}() + data, code = frame.framedata, frame.framecode + slotnames = code.src.slotnames + for (sym, counter, val) in zip(slotnames, data.last_reference, data.locals) + counter == 0 && continue + val = something(val) + if val isa Core.Box && !isdefined(val, :contents) + continue + end + var = Variable(val, sym) + idx = get(varlookup, sym, 0) + if idx > 0 + if counter > var_counter[idx] + vars[idx] = var + var_counter[idx] = counter + end + else + varlookup[sym] = length(vars)+1 + push!(vars, var) + push!(var_counter, counter) + end + end + scope = code.scope + if scope isa Method + syms = sparam_syms(scope) + for i in 1:length(syms) + if isassigned(data.sparams, i) + push!(vars, Variable(data.sparams[i], syms[i], true)) + end + end + end + for var in vars + if var.name === Symbol("#self#") + for field in fieldnames(typeof(var.value)) + field = field::Symbol + if isdefined(var.value, field) + push!(vars, Variable(getfield(var.value, field), field, false, true)) + end + end + end + end + return vars +end + +function print_vars(io::IO, vars::Vector{Variable}) + for v in vars + v.name === Symbol("#self#") && (isa(v.value, Type) || sizeof(v.value) == 0) && continue + print(io, '\n', v) + end +end + +""" + eval_code(frame::Frame, code::Union{String, Expr}) + +Evaluate `code` in the context of `frame`, updating any local variables +(including type parameters) that are reassigned in `code`, however, new local variables +cannot be introduced. + +```jldoctest +julia> foo(x, y) = x + y; + +julia> frame = JuliaInterpreter.enter_call(foo, 1, 3); + +julia> JuliaInterpreter.eval_code(frame, "x + y") +4 + +julia> JuliaInterpreter.eval_code(frame, "x = 5"); + +julia> JuliaInterpreter.finish_and_return!(frame) +8 +``` + +When variables are captured in closures (and thus gets wrapped in a `Core.Box`) +they will be automatically unwrapped and rewrapped upon evaluating them: + +```jldoctest +julia> function capture() + x = 1 + f = ()->(x = 2) # x captured in closure and is thus a Core.Box + f() + x + end; + +julia> frame = JuliaInterpreter.enter_call(capture); + +julia> JuliaInterpreter.step_expr!(frame); + +julia> JuliaInterpreter.step_expr!(frame); + +julia> JuliaInterpreter.locals(frame) +2-element Vector{JuliaInterpreter.Variable}: + #self# = capture + x = Core.Box(1) + +julia> JuliaInterpreter.eval_code(frame, "x") +1 + +julia> JuliaInterpreter.eval_code(frame, "x = 2") +2 + +julia> JuliaInterpreter.locals(frame) +2-element Vector{JuliaInterpreter.Variable}: + #self# = capture + x = Core.Box(2) +``` + +"Special" values like SSA values and slots (shown in lowered code as e.g. `%3` and `@_4` +respectively) can be evaluated using the syntax `var"%3"` and `var"@_4"` respectively. +""" +function eval_code end + +function extract_usage!(s::Set{Symbol}, expr) + if expr isa Expr + for arg in expr.args + if arg isa Symbol + push!(s, arg) + elseif arg isa Expr + extract_usage!(s, arg) + end + end + elseif expr isa Symbol + push!(s, expr) + end + return s +end + +function eval_code(frame::Frame, command::AbstractString) + expr = Base.parse_input_line(command) + expr === nothing && return nothing + return eval_code(frame, expr) +end +function eval_code(frame::Frame, expr::Expr) + code = frame.framecode + data = frame.framedata + isexpr(expr, :toplevel) && (expr = expr.args[end]) + + if isexpr(expr, :toplevel) + expr = Expr(:block, expr.args...) + end + + used_symbols = Set{Symbol}((Symbol("#self#"),)) + extract_usage!(used_symbols, expr) + # see https://github.com/JuliaLang/julia/issues/31255 for the Symbol("") check + vars = filter(v -> v.name != Symbol("") && v.name in used_symbols, locals(frame)) + defined_ssa = findall(i -> isassigned(data.ssavalues, i) && Symbol("%$i") in used_symbols, 1:length(data.ssavalues)) + defined_locals = findall(i-> data.locals[i] isa Some && Symbol("@_$i") in used_symbols, 1:length(data.locals)) + res = gensym() + eval_expr = Expr(:let, + Expr(:block, map(x->Expr(:(=), x...), [(v.name, QuoteNode(v.value isa Core.Box ? v.value.contents : v.value)) for v in vars])..., + map(x->Expr(:(=), x...), [(Symbol("%$i"), QuoteNode(data.ssavalues[i])) for i in defined_ssa])..., + map(x->Expr(:(=), x...), [(Symbol("@_$i"), QuoteNode(data.locals[i].value)) for i in defined_locals])..., + ), + Expr(:block, + Expr(:(=), res, expr), + Expr(:tuple, res, Expr(:tuple, [v.name for v in vars]...)) + )) + eval_res, res = Core.eval(moduleof(frame), eval_expr) + j = 1 + for (i, v) in enumerate(vars) + if v.isparam + data.sparams[j] = res[i] + j += 1 + elseif v.is_captured_closure + selfidx = findfirst(v -> v.name === Symbol("#self#"), vars) + @assert selfidx !== nothing + self = vars[selfidx].value + closed_over_var = getfield(self, v.name) + if closed_over_var isa Core.Box + setfield!(closed_over_var, :contents, res[i]) + end + # We cannot rebind closed over variables that the frontend identified as constant + else + slot_indices = code.slotnamelists[v.name] + idx = argmax(data.last_reference[slot_indices]) + slot_idx = slot_indices[idx] + data.last_reference[slot_idx] = (frame.assignment_counter += 1) + data.locals[slot_idx] = Some{Any}(v.value isa Core.Box ? Core.Box(res[i]) : res[i]) + end + end + eval_res +end + +function show_stackloc(io::IO, frame) + indent = "" + fr = root(frame) + shown = false + while fr !== nothing + print(io, indent, scopeof(fr)) + if fr === frame + println(io, ", pc = ", frame.pc) + shown = true + else + print(io, '\n') + end + indent *= " " + fr = fr.callee + end + if !shown + println(io, indent, scopeof(frame), ", pc = ", frame.pc) + end +end +show_stackloc(frame) = show_stackloc(stdout, frame) + +# Printing of stacktraces and errors with Frame +function Base.StackTraces.StackFrame(frame::Frame) + scope = scopeof(frame) + if scope isa Method + method = scope + method_args = [something(frame.framedata.locals[i]) for i in 1:method.nargs] + atypes = Tuple{mapany(_Typeof, method_args)...} + sig = method.sig + sparams = Core.svec(frame.framedata.sparams...) + mi = Core.Compiler.specialize_method(method, atypes, sparams) + fname = method.name + else + mi = frame.framecode.src + fname = gensym() + end + Base.StackFrame( + fname, + Symbol(getfile(frame)), + @something(linenumber(frame), getline(linetable(frame, 1))), + mi, + false, + false, + C_NULL + ) +end + +function Base.show_backtrace(io::IO, frame::Frame) + stackframes = Tuple{Base.StackTraces.StackFrame, Int}[] + while frame !== nothing + push!(stackframes, (Base.StackTraces.StackFrame(frame), 1)) + frame = JuliaInterpreter.caller(frame) + end + print(io, "\nStacktrace:") + try invokelatest(Base.update_stackframes_callback[], stackframes) catch end + frame_counter = 0 + nd = ndigits(length(stackframes)) + for (i, (last_frame, n)) in enumerate(stackframes) + frame_counter += 1 + if isdefined(Base, :print_stackframe) + println(io) + Base.print_stackframe(io, i, last_frame, n, nd, Base.info_color()) + else + Base.show_trace_entry(IOContext(io, :backtrace => true), last_frame, n, prefix = string(" [", frame_counter, "] ")) + end + end +end + +function Base.display_error(io::IO, er, frame::Frame) + printstyled(io, "ERROR: "; bold=true, color=Base.error_color()) + showerror(IOContext(io, :limit => true), er, frame) + println(io) +end + +function static_eval(ex) + try + eval(ex) + catch + nothing + end +end diff --git a/packages/JuliaInterpreter/test/breakpoints.jl b/packages/JuliaInterpreter/test/breakpoints.jl new file mode 100644 index 0000000..1070ec4 --- /dev/null +++ b/packages/JuliaInterpreter/test/breakpoints.jl @@ -0,0 +1,592 @@ +radius2(x, y) = x^2 + y^2 +function loop_radius2(n) + s = 0 + for i = 1:n + s += radius2(1, i) + end + s +end + +tmppath = "" +global tmppath +tmppath, io = mktemp() +print(io, """ +function jikwfunc(x, y=0; z="hello") + a = x + y + b = z^a + return length(b) +end +""") +close(io) +include(tmppath) + +using JuliaInterpreter, CodeTracking, Test + +function stacklength(frame) + n = 1 + frame = frame.callee + while frame !== nothing + n += 1 + frame = frame.callee + end + return n +end + +struct Squarer end + +@testset "Breakpoints" begin + Δ = CodeTracking.line_is_decl + + breakpoint(radius2) + frame = JuliaInterpreter.enter_call(loop_radius2, 2) + bp = JuliaInterpreter.finish_and_return!(frame) + @test isa(bp, JuliaInterpreter.BreakpointRef) + @test stacklength(frame) == 2 + @test leaf(frame).framecode.scope == @which radius2(0, 0) + bp = JuliaInterpreter.finish_stack!(frame) + @test isa(bp, JuliaInterpreter.BreakpointRef) + @test stacklength(frame) == 2 + @test JuliaInterpreter.finish_stack!(frame) == loop_radius2(2) + + # Conditional breakpoints + function runsimple() + frame = JuliaInterpreter.enter_call(loop_radius2, 2) + bp = JuliaInterpreter.finish_and_return!(frame) + @test isa(bp, JuliaInterpreter.BreakpointRef) + @test stacklength(frame) == 2 + @test leaf(frame).framecode.scope == @which radius2(0, 0) + @test JuliaInterpreter.finish_stack!(frame) == loop_radius2(2) + end + remove() + breakpoint(radius2, :(y > x)) + runsimple() + remove() + @breakpoint radius2(0,0) y>x + runsimple() + # Demonstrate the problem that we have with scope + local_identity(x) = identity(x) + remove() + @breakpoint radius2(0,0) y>local_identity(x) + @test_broken @interpret loop_radius2(2) + + # Conditional breakpoints on local variables + remove() + halfthresh = loop_radius2(5) + bp = @breakpoint loop_radius2(10) 5 s>$halfthresh + frame, bpref = @interpret loop_radius2(10) + @test isa(bpref, JuliaInterpreter.BreakpointRef) + lframe = leaf(frame) + s_extractor = eval(JuliaInterpreter.prepare_slotfunction(lframe.framecode, :s)) + @test s_extractor(lframe) == loop_radius2(6) + JuliaInterpreter.finish_stack!(frame) + @test s_extractor(lframe) == loop_radius2(7) + disable(bp) + @test JuliaInterpreter.finish_stack!(frame) == loop_radius2(10) + + # Return value with breakpoints + @breakpoint sum([1,2]) any(x->x>4, a) + val = @interpret sum([1,2,3]) + @test val == 6 + frame, bp = @interpret sum([1,2,5]) + @test isa(frame, Frame) && isa(bp, JuliaInterpreter.BreakpointRef) + + # Next line with breakpoints + function outer(x) + inner(x) + end + function inner(x) + return 2 + end + breakpoint(inner) + frame = JuliaInterpreter.enter_call(outer, 0) + bp = JuliaInterpreter.next_line!(frame) + @test isa(bp, JuliaInterpreter.BreakpointRef) + @test JuliaInterpreter.finish_stack!(frame) == 2 + + # Breakpoints by file/line + remove() + method = which(JuliaInterpreter.locals, Tuple{Frame}) + breakpoint(String(method.file), method.line+1) + frame = JuliaInterpreter.enter_call(loop_radius2, 2) + ret = @interpret JuliaInterpreter.locals(frame) + @test isa(ret, Tuple{Frame,JuliaInterpreter.BreakpointRef}) + # Test kwarg method + remove() + bp = breakpoint(tmppath, 3) + frame, bp2 = @interpret jikwfunc(2) + var = JuliaInterpreter.locals(leaf(frame)) + @test !any(v->v.name === :b, var) + @test filter(v->v.name === :a, var)[1].value == 2 + + # Method with local scope (two slots with same name) + ln = @__LINE__ + function ftwoslots() + y = 1 + z = let y = y + y = y + 2 + rand() + end + y = y + 1 + return z + end + bp = breakpoint(@__FILE__, ln+5, :(y > 2)) + frame, bp2 = @interpret ftwoslots() + var = JuliaInterpreter.locals(leaf(frame)) + @test filter(v->v.name === :y, var)[1].value == 3 + remove(bp) + bp = breakpoint(@__FILE__, ln+8, :(y > 2)) + @test isa(@interpret(ftwoslots()), Float64) + + # Direct return + @breakpoint gcd(1,1) a==5 + @test @interpret(gcd(10,20)) == 10 + # FIXME: even though they pass, these tests break Test! + # frame, bp = @interpret gcd(5, 20) + # @test stacklength(frame) == 1 + # @test isa(bp, JuliaInterpreter.BreakpointRef) + remove() + + # break on error + try + @test_throws ArgumentError("unsupported state :missing") break_on(:missing) + break_on(:error) + + inner(x) = error("oops") + outer() = inner(1) + frame = JuliaInterpreter.enter_call(outer) + bp = JuliaInterpreter.finish_and_return!(frame) + @test bp.err == ErrorException("oops") + @test stacklength(frame) >= 2 + @test frame.framecode.scope.name === :outer + cframe = frame.callee + @test cframe.framecode.scope.name === :inner + + # Don't break on caught exceptions + function f_exc_outer() + try + f_exc_inner() + catch err + return err + end + end + function f_exc_inner() + error() + end + frame = JuliaInterpreter.enter_call(f_exc_outer); + v = JuliaInterpreter.finish_and_return!(frame) + @test v isa ErrorException + @test stacklength(frame) == 1 + + # Break on caught exception when enabled + break_on(:throw) + try + frame = JuliaInterpreter.enter_call(f_exc_outer); + v = JuliaInterpreter.finish_and_return!(frame) + @test v isa BreakpointRef + @test v.err isa ErrorException + @test v.framecode.scope == @which error() + finally + break_off(:throw) + end + finally + break_off(:error) + end + + # Breakpoint display + io = IOBuffer() + frame = JuliaInterpreter.enter_call(loop_radius2, 2) + if VERSION < v"1.9.0-DEV.846" # https://github.com/JuliaLang/julia/pull/45069 + LOC = " in $(@__MODULE__) at $(@__FILE__)" + else + LOC = "\n @ $(@__MODULE__) $(contractuser(@__FILE__))" + end + bp = JuliaInterpreter.BreakpointRef(frame.framecode, 1) + @test repr(bp) == "breakpoint(loop_radius2(n)$LOC:$(3-Δ), line 3)" + bp = JuliaInterpreter.BreakpointRef(frame.framecode, 0) # fictive breakpoint + @test repr(bp) == "breakpoint(loop_radius2(n)$LOC:$(3-Δ), %0)" + bp = JuliaInterpreter.BreakpointRef(frame.framecode, 1, ArgumentError("whoops")) + @test repr(bp) == "breakpoint(loop_radius2(n)$LOC:$(3-Δ), line 3, ArgumentError(\"whoops\"))" + + # In source breakpointing + f_outer_bp(x) = g_inner_bp(x) + function g_inner_bp(x) + sin(x) + @bp + @bp + @bp + x = 3 + return 2 + end + fr, bp = @interpret f_outer_bp(3) + @test leaf(fr).framecode.scope.name === :g_inner_bp + @test bp.stmtidx == 3 + + # Breakpoints on types + remove() + g() = Int(5.0) + @breakpoint Int(5.0) + frame, bp = @interpret g() + @test bp isa BreakpointRef + @test leaf(frame).framecode.scope === @which Int(5.0) + + # Breakpoint on call overloads + (::Squarer)(x) = x^2 + squarer = Squarer() + @breakpoint squarer(2) + frame, bp = @interpret squarer(3.0) + @test bp isa BreakpointRef + @test leaf(frame).framecode.scope === @which squarer(3.0) +end + +mktemp() do path, io + print(io, """ + function somefunc(x, y=0) + a = x + y + b = z^a + return a + b + end + """) + close(io) + breakpoint(path, 3) + include(path) + frame, bp = @interpret somefunc(2, 3) + @test bp isa BreakpointRef + @test JuliaInterpreter.whereis(frame) == (path, 3) + breakpoint(path, 2) + frame, bp = @interpret somefunc(2, 3) + @test bp isa BreakpointRef + @test JuliaInterpreter.whereis(frame) == (path, 2) + remove() + # Test relative paths + mktempdir(dirname(path)) do tmp + cd(tmp) do + breakpoint(joinpath("..", basename(path)), 3) + frame, bp = @interpret somefunc(2, 3) + @test bp isa BreakpointRef + @test JuliaInterpreter.whereis(frame) == (path, 3) + remove() + breakpoint(joinpath("..", basename(path)), 3) + cd(homedir()) do + frame, bp = @interpret somefunc(2, 3) + @test bp isa BreakpointRef + @test JuliaInterpreter.whereis(frame) == (path, 3) + end + end + end +end + +if tmppath != "" + rm(tmppath) +end + +@testset "toggling" begin + remove() + f_break(x::Int) = x + bp = breakpoint(f_break) + frame, bpref = @interpret f_break(5) + @test bpref isa BreakpointRef + toggle(bp) + @test (@interpret f_break(5)) == 5 + f_break(x::Float64) = 2x + @test (@interpret f_break(2.0)) == 4.0 + toggle(bp) + frame, bpref = @interpret f_break(5) + @test bpref isa BreakpointRef + frame, bpref = @interpret f_break(2.0) + @test bpref isa BreakpointRef +end + +using Dates +@testset "breakpoint in stdlibs by path" begin + m = @which now() - Month(2) + f = String(m.file) + l = m.line + 1 + for f in (f, basename(f)) + remove() + breakpoint(f, l) + frame, bp = @interpret now() - Month(2) + @test bp isa BreakpointRef + @test JuliaInterpreter.whereis(frame)[2] == l + end +end + +@testset "breakpoint in Base by path" begin + m = @which sin(2.0) + f = String(m.file) + l = m.line + 1 + for f in (f, basename(f)) + remove() + breakpoint(f, l) + frame, bp = @interpret sin(2.0) + @test bp isa BreakpointRef + @test JuliaInterpreter.whereis(frame)[2] == l + end +end + +@testset "breakpoint by type" begin + remove() + breakpoint(sin, Tuple{Float64}) + frame, bp = @interpret sin(2.0) + @test bp isa BreakpointRef +end + +const breakpoint_update_hooks = JuliaInterpreter.breakpoint_update_hooks +const on_breakpoints_updated = JuliaInterpreter.on_breakpoints_updated +@testset "hooks" begin + remove() + f_break(x) = x + + # Check creating hits hook + empty!(breakpoint_update_hooks) + hook_hit = false + on_breakpoints_updated((f,_)->hook_hit = f == breakpoint) + orig_bp = breakpoint(f_break) + @test hook_hit + + # Check re-creating hits remove *and* breakpoint (create) + empty!(breakpoint_update_hooks) + hit_remove_old = false + hit_create_new = false + hit_other = false # don't want this + on_breakpoints_updated() do f, hbp + if f==remove + hit_remove_old = hbp === orig_bp + elseif f==breakpoint + hit_create_new = hbp !== orig_bp + else + hit_other = true + end + end + push!(breakpoint_update_hooks, (f,_)->hook_hit = f == breakpoint) + bp = breakpoint(f_break) + @test hit_remove_old + @test hit_create_new + @test !hit_other + + @testset "update_states! $op hits hook" for op in (disable, enable, toggle) + empty!(breakpoint_update_hooks) + hook_hit = false + on_breakpoints_updated((f, _) -> hook_hit = f == JuliaInterpreter.update_states!) + op(bp) + @test hook_hit + end + + # Test removing hits hooks + empty!(breakpoint_update_hooks) + hook_hit = false + on_breakpoints_updated((f, _) -> hook_hit = f === remove) + remove(bp) + @test hook_hit + + @testset "make sure error in hook function doesn't throw" begin + empty!(breakpoint_update_hooks) + on_breakpoints_updated((_, _) -> error("bad hook")) + @test_logs (:warn, r"hook"i) breakpoint(f_break) + end +end +# Run outside testset so that if it fails, the hooks get removed. So other tests can pass +empty!(breakpoint_update_hooks) + +@testset "toplevel breakpoints" begin + mktemp() do path, io + print(io, """ + 1+1 # bp + begin + 2+2 + 3+3 # bp + end + function foo(x) + x + x # bp + x + end + """) + close(io) + + expr = Base.parse_input_line(String(read(path)), filename = path) + exprs = collect(ExprSplitter(Main, expr)) + + breakpoint(path, 1) + breakpoint(path, 4) + breakpoint(path, 8) + + # breakpoint in top-level line + mod, ex = exprs[1] + frame = Frame(mod, ex) + @test JuliaInterpreter.shouldbreak(frame, frame.pc) + ret = JuliaInterpreter.finish_and_return!(frame, true) + @test ret === 2 + + # breakpoint in top-level block + mod, ex = exprs[3] + frame = Frame(mod, ex) + @test JuliaInterpreter.shouldbreak(frame, frame.pc) + ret = JuliaInterpreter.finish_and_return!(frame, true) + @test ret === 6 + + # don't break for bp in function definition + mod, ex = exprs[4] + frame = Frame(mod, ex) + @test JuliaInterpreter.shouldbreak(frame, frame.pc) == false + ret = JuliaInterpreter.finish_and_return!(frame, true) + @test ret isa Function + + remove() + end +end + +@testset "duplicate slotnames" begin + tmp_dupl() = (1,2,3,4) + ln = @__LINE__ + function duplnames(x) + for iter in CartesianIndices(x) + i = iter[1] + c = i + a, b, c, d = tmp_dupl() + end + return x + end + bp = breakpoint(@__FILE__, ln+5, :(i == 1)) + c = @code_lowered(duplnames((1,2))) + if length(unique(c.slotnames)) < length(c.slotnames) + f = JuliaInterpreter.enter_call(duplnames, (1,2)) + ex = JuliaInterpreter.prepare_slotfunction(f.framecode, :(i==1)) + @test ex isa Expr + found = false + for arg in ex.args[end].args + if arg.args[1] === :i + found = true + end + end + @test found + @test last(JuliaInterpreter.debug_command(f, :c)) isa BreakpointRef + end +end + +@testset "recursive builtins" begin + g(args...) = args + args = (iterate, g, (1,3)) + + h() = Core._apply_iterate(iterate, Core._apply_iterate, args) + breakpoint(g) + frame, bp = @interpret h() + var = JuliaInterpreter.locals(leaf(frame)) + @test filter(v->v.name === :args, var)[1].value == (1,3) +end + +struct Constructor + x::Int +end +Constructor(x::AbstractString, y::Int) = Constructor(x) + +@testset "constructors" begin + breakpoint(Constructor, Tuple{String, Int}) + frame, bp = @interpret Constructor("foo", 3) + @test bp isa BreakpointRef + @test @interpret Constructor(3) isa Constructor + + breakpoint(Constructor) + frame, bp = @interpret Constructor(2) + @test bp isa BreakpointRef +end + +@testset "test breaking on a line with no statement" begin + ln = @__LINE__ + function f_emptylines() + sin(2.0) + + + + return sin(2.0) + end + + bp = breakpoint(@__FILE__, ln + 4) + frame, bpref = @interpret f_emptylines() + @test bpref isa BreakpointRef + @test JuliaInterpreter.whereis(frame) == (@__FILE__, ln + 6) + remove(bp) + + # Don't break if the line is outside the function + breakpoint(@__FILE__, ln) + @test (@interpret f_emptylines()) == sin(2.0) +end + +@testset "macro expansion breakpoint tests" begin + function f_macro() + sin(2.0) + @info "foo" + sin(2.0) + @info "foo" + return 2 + end + frame = JuliaInterpreter.enter_call(f_macro) + file_logging = "logging.jl" + line_logging = 0 + for entry in frame.framecode.src.linetable + if entry.file === Symbol(file_logging) + line_logging = entry.line + break + end + end + bp_log = breakpoint(file_logging, line_logging) + with_logger(NullLogger()) do + frame, bp = @interpret f_macro() + @test bp isa BreakpointRef + file, ln = JuliaInterpreter.whereis(frame) + @test ln == line_logging + @test basename(file) == file_logging + bp = JuliaInterpreter.finish_stack!(frame) + @test bp isa BreakpointRef + frame = leaf(frame) + ret = JuliaInterpreter.finish_stack!(frame) + @test ret == 2 + end + + JuliaInterpreter.remove(bp_log) + + # Check that stopping on a line only stops in the correct file + mktemp() do path, io + for _ in 1:line_logging-5 + print(io, "\n") + end + print(io, + """ + function f_check(x) + sin(x) + @info "foo" + sin(x) + sin(x) + sin(x) + sin(x) + sin(x) + + return x + end + """) + bp_f = breakpoint(path, line_logging) + flush(io) + include(path) + + with_logger(NullLogger()) do + frame, bp = @interpret f_check(1) + file, ln = JuliaInterpreter.whereis(frame) + @test file == path # Should not have stopped in logging.jl at line `line_logging` + @test ln == line_logging + remove(bp_f) + @test (@interpret f_check(1)) == 1 + breakpoint(f_check, line_logging) + frame, bp = @interpret f_check(1) + file, ln = JuliaInterpreter.whereis(frame) + @test file == path # Should not have stopped in logging.jl at line `line_logging` + @test ln == line_logging + end + end +end + +@testset "breakpoint in kwfuncs" begin + fkw(;x=1) = x + breakpoint(fkw) + g() = fkw(; x=1) + frame, bp = @interpret g() + @test isa(frame, Frame) && isa(bp, JuliaInterpreter.BreakpointRef) +end diff --git a/packages/JuliaInterpreter/test/check_builtins.jl b/packages/JuliaInterpreter/test/check_builtins.jl new file mode 100644 index 0000000..ab1b2c0 --- /dev/null +++ b/packages/JuliaInterpreter/test/check_builtins.jl @@ -0,0 +1,19 @@ +using Test, DeepDiffs + +@static if !Base.GIT_VERSION_INFO.tagged_commit && # only run on nightly + !Sys.iswindows() # TODO: Understand why this fails, probably some line endings + @testset "Check builtin.jl consistency" begin + builtins_path = joinpath(@__DIR__, "..", "src", "builtins.jl") + old_builtins = read(builtins_path, String) + new_builtins_dir = mktempdir() + withenv("JULIAINTERPRETER_BUILTINS_DIR" => new_builtins_dir) do + include("../bin/generate_builtins.jl") + end + new_builtins = read(joinpath(new_builtins_dir, "builtins.jl"), String) + consistent = old_builtins == new_builtins + if !consistent + println(deepdiff(old_builtins, new_builtins)) + end + @test consistent + end +end diff --git a/packages/JuliaInterpreter/test/code_coverage/code_coverage.jl b/packages/JuliaInterpreter/test/code_coverage/code_coverage.jl new file mode 100644 index 0000000..67dd319 --- /dev/null +++ b/packages/JuliaInterpreter/test/code_coverage/code_coverage.jl @@ -0,0 +1,59 @@ +function cleanup_coverage_files(pid) + # clean up coverage files for source code + dir, _, files = first(walkdir(normpath(@__DIR__, "..", "..", "src"))) + for file in files + reg = Regex(string(".+\\.jl\\.$pid\\.cov")) + if occursin(reg, file) + rm(joinpath(dir, file)) + end + end + + # clean up coverage files for this file + dir, _, files = first(walkdir(@__DIR__)) + for file in files + reg = Regex(string("coverage_example\\.jl\\.$pid\\.cov")) + if occursin(reg, file) + rm(joinpath(dir, file)) + end + end +end + +# using DiffUtils + +let + local pid + try + @testset "code coverage" begin + io = IOBuffer() + filepath = normpath(@__DIR__, "coverage_example.jl") + cmd = `$(Base.julia_cmd()) --startup=no --project=$(dirname(dirname(@__DIR__))) + --code-coverage=user $filepath` + p = run(pipeline(cmd; stdout=io); wait=false) + pid = Libc.getpid(p) + wait(p) + out = String(take!(io)) + @test out == "1 2 fizz 4 " + + dir, _, files = first(walkdir(@__DIR__)) + i = findfirst(contains(r"coverage_example\.jl\.\d+\.cov"), files) + i === nothing && error("no coverage files found in $dir: $files") + cov_file = joinpath(dir, files[i]) + cov_data = read(cov_file, String) + expected = read(joinpath(dir, "coverage_example.jl.cov"), String) + if Sys.iswindows() + cov_data = replace(cov_data, "\r\n" => "\n") + expected = replace(cov_data, "\r\n" => "\n") + end + + # if cov_data != expected + # DiffUtils.diff(cov_data, expected) + # end + @test cov_data == expected + end + finally + if @isdefined(pid) + # clean up generated files + cleanup_coverage_files(pid) + end + end +end diff --git a/packages/JuliaInterpreter/test/code_coverage/coverage_example.jl b/packages/JuliaInterpreter/test/code_coverage/coverage_example.jl new file mode 100644 index 0000000..7e36607 --- /dev/null +++ b/packages/JuliaInterpreter/test/code_coverage/coverage_example.jl @@ -0,0 +1,17 @@ +fizz() = print("fizz ") +buzz() = print("buzz ") + +function fizzbuzz(n) + for i in 1:n + if i % 3 == 0 || i % 5 == 0 + i % 3 == 0 && fizz() + i % 5 == 0 && buzz() + else + print(i, " ") + end + end + return n +end + +using JuliaInterpreter +@interpret fizzbuzz(4) diff --git a/packages/JuliaInterpreter/test/code_coverage/coverage_example.jl.cov b/packages/JuliaInterpreter/test/code_coverage/coverage_example.jl.cov new file mode 100644 index 0000000..fb8f303 --- /dev/null +++ b/packages/JuliaInterpreter/test/code_coverage/coverage_example.jl.cov @@ -0,0 +1,17 @@ + 1 fizz() = print("fizz ") + - buzz() = print("buzz ") + - + - function fizzbuzz(n) + 4 for i in 1:n + 4 if i % 3 == 0 || i % 5 == 0 + 1 i % 3 == 0 && fizz() + 1 i % 5 == 0 && buzz() + - else + 3 print(i, " ") + - end + 4 end + 1 return n + - end + - + - using JuliaInterpreter + - @interpret fizzbuzz(4) diff --git a/packages/JuliaInterpreter/test/core.jl b/packages/JuliaInterpreter/test/core.jl new file mode 100644 index 0000000..42b1cbd --- /dev/null +++ b/packages/JuliaInterpreter/test/core.jl @@ -0,0 +1,33 @@ +using JuliaInterpreter +using Test + +@testset "core" begin + @test JuliaInterpreter.is_quoted_type(QuoteNode(Int32), :Int32) + @test !JuliaInterpreter.is_quoted_type(QuoteNode(Int32), :Int64) + @test !JuliaInterpreter.is_quoted_type(QuoteNode(Int32(0)), :Int32) + @test !JuliaInterpreter.is_quoted_type(Int32, :Int32) + + function buildexpr() + items = [7, 3] + ex = quote + X = $items + for x in X + println(x) + end + end + return ex + end + frame = JuliaInterpreter.enter_call(buildexpr) + lines = JuliaInterpreter.framecode_lines(frame.framecode.src) + # Test that the :copyast ends up on the same line as the println + if isdefined(Base.IRShow, :show_ir_stmt) # only works on Julia 1.6 and higher + @test any(str->occursin(":copyast", str) && occursin("println", str), lines) + end + + thunk = Meta.lower(Main, :(return 1+2)) + stmt = thunk.args[1].code[end] # the return + @test JuliaInterpreter.get_return_node(stmt) isa Core.SSAValue + + @test string(JuliaInterpreter.parametric_type_to_expr(Base.Iterators.Stateful{String})) ∈ + ("Base.Iterators.Stateful{String, VS}", "(Base.Iterators).Stateful{String, VS}", "Base.Iterators.Stateful{String, VS, N}") +end diff --git a/packages/JuliaInterpreter/test/debug.jl b/packages/JuliaInterpreter/test/debug.jl new file mode 100644 index 0000000..68c78c2 --- /dev/null +++ b/packages/JuliaInterpreter/test/debug.jl @@ -0,0 +1,582 @@ +using CodeTracking, JuliaInterpreter, Test +using JuliaInterpreter: enter_call, enter_call_expr, get_return, @lookup +using Base.Meta: isexpr +include("utils.jl") + +const ALL_COMMANDS = (:n, :s, :c, :finish, :nc, :se, :si, :until) + +function step_through_command(fr::Frame, cmd::Symbol) + while true + ret = JuliaInterpreter.debug_command(JuliaInterpreter.finish_and_return!, fr, cmd) + ret == nothing && break + fr, pc = ret + end + @test fr.callee === nothing + @test fr.caller === nothing + return get_return(fr) +end + +function step_through_frame(frame_creator) + rets = [] + for cmd in ALL_COMMANDS + frame = frame_creator() + ret = step_through_command(frame, cmd) + push!(rets, ret) + end + @test all(ret -> ret == rets[1], rets) + return rets[1] +end +step_through(f, args...; kwargs...) = step_through_frame(() -> enter_call(f, args...; kwargs...)) +step_through(expr::Expr) = step_through_frame(() -> enter_call_expr(expr)) + +@generated function generatedfoo(T) + :(return $T) +end +callgenerated() = generatedfoo(1) +@generated function generatedparams(a::Array{T,N}) where {T,N} + :(return ($T,$N)) +end +callgeneratedparams() = generatedparams([1 2; 3 4]) + +macro insert_some_calls() + esc(quote + x = sin(b) + y = asin(x) + z = sin(y) + end) +end + +trivial(x) = x + +struct B{T} end + +# Putting this into a @testset introduces a closure that breaks the kwprep detection +function complicated_keyword_stuff(a, b=-1; x=1, y=2) + a == a + (a, b, :x=>x, :y=>y) +end +function complicated_keyword_stuff_splatargs(args...; x=1, y=2) + args[1] == args[1] + (args..., :x=>x, :y=>y) +end +function complicated_keyword_stuff_splatkws(a, b=-1; kw...) + a == a + (a, b, kw...) +end +function complicated_keyword_stuff_splat2(args...; kw...) + args[1] == args[1] + (args..., kw...) +end + +# @testset "Debug" begin + @testset "Basics" begin + frame = enter_call(map, x->2x, 1:10) + @test debug_command(frame, :finish) === nothing + @test frame.caller === frame.callee === nothing + @test get_return(frame) == map(x->2x, 1:10) + + for func in (complicated_keyword_stuff, complicated_keyword_stuff_splatargs, + complicated_keyword_stuff_splatkws, complicated_keyword_stuff_splat2) + for (args, kwargs) in (((1,), ()), ((1, 2), (x=7, y=33))) + oframe = frame = enter_call(func, args...; kwargs...) + frame = JuliaInterpreter.maybe_step_through_kwprep!(frame, false) + frame = JuliaInterpreter.maybe_step_through_wrapper!(frame) + @test any(stmt->isa(stmt, Expr) && JuliaInterpreter.hasarg(isequal(QuoteNode(==)), stmt.args), frame.framecode.src.code) + f, pc = debug_command(frame, :n) + @test f === frame + @test isa(pc, Int) + @test oframe.callee !== nothing + @test debug_command(frame, :finish) === nothing + @test oframe.caller === oframe.callee === nothing + @test get_return(oframe) == func(args...; kwargs...) + + @test @interpret(complicated_keyword_stuff(args...; kwargs...)) == complicated_keyword_stuff(args...; kwargs...) + end + end + + let f22() = string(:(a+b)) + @test step_through(f22) == "a + b" + end + let f22() = string(QuoteNode(:a)) + @test step_through(f22) == ":a" + end + + frame = enter_call(trivial, 2) + @test debug_command(frame, :s) === nothing + @test get_return(frame) == 2 + + @test step_through(trivial, 2) == 2 + @test step_through(:($(+)(1,2.5))) == 3.5 + @test step_through(:($(sin)(1))) == sin(1) + @test step_through(:($(gcd)(10,20))) == gcd(10, 20) + end + + @testset "until" begin + function f_with_lines(s) + sin(2.0) + cos(2.0) + for i in 1:100 + s += i + end + sin(2.0) + end + meth_def = @__LINE__() - 8 + + frame = enter_call(f_with_lines, 0) + @test whereis(frame)[2] == meth_def + 1 + debug_command(frame, :until) + @test whereis(frame)[2] == meth_def + 2 + debug_command(frame, :until; line=(meth_def + 4)) + @test whereis(frame)[2] == meth_def + 4 + debug_command(frame, :until; line=(meth_def + 6)) + @test whereis(frame)[2] == meth_def + 6 + end + + @testset "generated" begin + frame = enter_call_expr(:($(callgenerated)())) + f, pc = debug_command(frame, :s) + @test isa(pc, BreakpointRef) + @test JuliaInterpreter.scopeof(f).name === :generatedfoo + stmt = JuliaInterpreter.pc_expr(f) + @test JuliaInterpreter.is_return(stmt) && JuliaInterpreter.lookup_return(frame, stmt) === Int + @test debug_command(frame, :c) === nothing + @test frame.callee === nothing + @test get_return(frame) === Int + # This time, step into the generated function itself + frame = enter_call_expr(:($(callgenerated)())) + f, pc = debug_command(frame, :sg) + # Aside: generators can have `Expr(:line, ...)` in their line tables, test that this is OK + lt = JuliaInterpreter.linetable(f, 2) + @test isexpr(lt, :line) || isa(lt, Core.LineInfoNode) + @test isa(pc, BreakpointRef) + @test JuliaInterpreter.scopeof(f).name === :generatedfoo + stmt = JuliaInterpreter.pc_expr(f) + @test JuliaInterpreter.is_return(stmt) && JuliaInterpreter.lookup_return(f, stmt) === 1 + f2, pc = debug_command(f, :finish) + @test JuliaInterpreter.scopeof(f2).name === :callgenerated + # Now finish the regular function + @test debug_command(frame, :finish) === nothing + @test frame.callee === nothing + @test get_return(frame) === 1 + + # Parametric generated function (see #157) + frame = fr = JuliaInterpreter.enter_call(callgeneratedparams) + while fr.pc < JuliaInterpreter.nstatements(fr.framecode) - 1 + fr, pc = debug_command(fr, :se) + end + fr, pc = debug_command(fr, :sg) + @test JuliaInterpreter.scopeof(fr).name === :generatedparams + fr, pc = debug_command(fr, :finish) + @test debug_command(fr, :finish) === nothing + @test JuliaInterpreter.get_return(fr) == (Int, 2) + end + + @testset "Optional arguments" begin + function optional(n = sin(1)) + x = asin(n) + cos(x) + end + frame = JuliaInterpreter.enter_call_expr(:($(optional)())) + # Step through the wrapper + f = JuliaInterpreter.maybe_step_through_wrapper!(frame) + @test frame !== f + # asin(n) + f, pc = debug_command(f, :n) + # cos(1.0) + f, pc = debug_command(f, :n) + # return + @test debug_command(f, :n) === nothing + end + + @testset "Keyword arguments" begin + f(x; b = 1) = x+b + g() = f(1; b = 2) + frame = JuliaInterpreter.enter_call_expr(:($(g)())); + fr, pc = debug_command(frame, :nc) + fr, pc = debug_command(fr, :nc) + fr, pc = debug_command(fr, :nc) + fr, pc = debug_command(fr, :s) + fr, pc = debug_command(fr, :finish) + @test debug_command(fr, :finish) === nothing + @test frame.callee === nothing + @test get_return(frame) == 3 + + frame = JuliaInterpreter.enter_call(f, 2; b = 4) + fr = JuliaInterpreter.maybe_step_through_wrapper!(frame) + fr, pc = debug_command(fr, :nc) + debug_command(fr, :nc) + @test get_return(frame) == 6 + end + + @testset "Optional + keyword wrappers" begin + opkw(a, b=1; c=2, d=3) = 1 + callopkw1() = opkw(0) + callopkw2() = opkw(0, -1) + callopkw3() = opkw(0; c=-2) + callopkw4() = opkw(0, -1; c=-2) + callopkw5() = opkw(0; c=-2, d=-3) + callopkw6() = opkw(0, -1; c=-2, d=-3) + scopes = Method[] + for f in (callopkw1, callopkw2, callopkw3, callopkw4, callopkw5, callopkw6) + frame = fr = JuliaInterpreter.enter_call(f) + pc = fr.pc + while pc <= JuliaInterpreter.nstatements(fr.framecode) - 2 + fr, pc = debug_command(fr, :se) + end + fr, pc = debug_command(frame, :si) + @test stacklength(frame) == 2 + frame = fr = JuliaInterpreter.enter_call(f) + pc = fr.pc + while pc <= JuliaInterpreter.nstatements(fr.framecode) - 2 + fr, pc = debug_command(fr, :se) + end + fr, pc = debug_command(frame, :s) + @test stacklength(frame) > 2 + push!(scopes, JuliaInterpreter.scopeof(fr)) + end + @test length(unique(scopes)) == 1 # all get to the body method + end + + @testset "Macros" begin + # Work around the fact that we can't detect macro expansions if the macro + # is defined in the same file + include_string(Main, """ + function test_macro() + a = sin(5) + b = asin(a) + @insert_some_calls + z + end + ""","file.jl") + frame = JuliaInterpreter.enter_call_expr(:($(test_macro)())) + f, pc = debug_command(frame, :n) # a is set + f, pc = debug_command(f, :n) # b is set + f, pc = debug_command(f, :n) # x is set + f, pc = debug_command(f, :n) # y is set + f, pc = debug_command(f, :n) # z is set + @test debug_command(f, :n) === nothing # return + end + + @testset "Quoting" begin + # Test that symbols don't get an extra QuoteNode + f_symbol() = :limit => true + frame = JuliaInterpreter.enter_call(f_symbol) + fr, pc = debug_command(frame, :s) + fr, pc = debug_command(fr, :finish) + @test debug_command(fr, :finish) === nothing + @test get_return(frame) == f_symbol() + end + + @testset "Varargs" begin + f_va_inner(x) = x + 1 + f_va_outer(args...) = f_va_inner(args...) + frame = fr = JuliaInterpreter.enter_call(f_va_outer, 1) + # depending on whether this is in or out of a @testset, the first statement may differ + stmt1 = fr.framecode.src.code[1] + if isexpr(stmt1, :call) && @lookup(frame, stmt1.args[1]) === getfield + fr, pc = debug_command(fr, :se) + end + fr, pc = debug_command(fr, :s) + fr, pc = debug_command(fr, :n) + @test root(fr) !== fr + fr, pc = debug_command(fr, :finish) + @test debug_command(fr, :finish) === nothing + @test get_return(frame) === 2 + end + + @testset "ASTI#17" begin + function (::B)(y) + x = 42*y + return x + y + end + B_inst = B{Int}() + step_through(B_inst, 10) == B_inst(10) + end + + @testset "Exceptions" begin + # Don't break on caught exceptions + err_caught = Any[nothing] + function f_exc_outer() + try + f_exc_inner() + catch err; + err_caught[1] = err + end + x = 1 + 1 + return x + end + f_exc_inner() = error() + fr = JuliaInterpreter.enter_call(f_exc_outer) + fr, pc = debug_command(fr, :s) + fr, pc = debug_command(fr, :n) + fr, pc = debug_command(fr, :n) + debug_command(fr, :finish) + @test get_return(fr) == 2 + @test first(err_caught) isa ErrorException + @test stacklength(fr) == 1 + + err_caught = Any[nothing] + fr = JuliaInterpreter.enter_call(f_exc_outer) + fr, pc = debug_command(fr, :s) + debug_command(fr, :c) + @test get_return(root(fr)) == 2 + @test first(err_caught) isa ErrorException + @test stacklength(root(fr)) == 1 + + # Rethrow on uncaught exceptions + f_outer() = g_inner() + g_inner() = error() + fr = JuliaInterpreter.enter_call(f_outer) + @test_throws ErrorException debug_command(fr, :finish) + @test stacklength(fr) == 3 + + # Break on error + try + break_on(:error) + fr = JuliaInterpreter.enter_call(f_outer) + fr, pc = debug_command(JuliaInterpreter.finish_and_return!, fr, :finish) + @test fr.framecode.scope.name === :error + + fundef() = undef_func() + frame = JuliaInterpreter.enter_call(fundef) + fr, pc = debug_command(frame, :s) + @test isa(pc, BreakpointRef) + @test pc.err isa UndefVarError + finally + break_off(:error) + end + end + + @testset "breakpoints" begin + # In source breakpoints + function f_bp(x) + #=1=# i = 1 + #=2=# @label foo + #=3=# @bp + #=4=# repr("foo") + #=5=# i += 1 + #=6=# i > 3 && return x + #=7=# @goto foo + end + ln = @__LINE__ + method_start = ln - 9 + fr = enter_call(f_bp, 2) + @test JuliaInterpreter.linenumber(fr) == method_start + 1 + fr, pc = JuliaInterpreter.debug_command(fr, :c) + # Hit the breakpoint x1 + @test JuliaInterpreter.linenumber(fr) == method_start + 3 + @test pc isa BreakpointRef + fr, pc = JuliaInterpreter.debug_command(fr, :n) + @test JuliaInterpreter.linenumber(fr) == method_start + 4 + fr, pc = JuliaInterpreter.debug_command(fr, :c) + # Hit the breakpoint again x2 + @test pc isa BreakpointRef + @test JuliaInterpreter.linenumber(fr) == method_start + 3 + fr, pc = JuliaInterpreter.debug_command(fr, :c) + # Hit the breakpoint for the last time x3 + @test pc isa BreakpointRef + @test JuliaInterpreter.linenumber(fr) == method_start + 3 + JuliaInterpreter.debug_command(fr, :c) + @test get_return(fr) == 2 + end + + f_inv(x::Real) = x^2; + f_inv(x::Integer) = 1 + invoke(f_inv, Tuple{Real}, x) + @testset "invoke" begin + fr = JuliaInterpreter.enter_call(f_inv, 2) + fr, pc = JuliaInterpreter.debug_command(fr, :s) # apply_type + frame, pc = JuliaInterpreter.debug_command(fr, :s) # step into invoke + @test frame.framecode.scope.sig == Tuple{typeof(f_inv),Real} + JuliaInterpreter.debug_command(frame, :c) + frame = root(frame) + @test get_return(frame) == f_inv(2) + end + + f_inv_latest(x::Real) = 1 + (@static isdefined(Core, :_call_latest) ? Core._call_latest(f_inv, x) : Core._apply_latest(f_inv, x)) + @testset "invokelatest" begin + fr = JuliaInterpreter.enter_call(f_inv_latest, 2.0) + fr, pc = JuliaInterpreter.debug_command(fr, :nc) + frame, pc = JuliaInterpreter.debug_command(fr, :s) # step into invokelatest + @test frame.framecode.scope.sig == Tuple{typeof(f_inv),Real} + JuliaInterpreter.debug_command(frame, :c) + frame = root(frame) + @test get_return(frame) == f_inv_latest(2.0) + end + + @testset "Issue #178" begin + remove() + a = [1, 2, 3, 4] + @breakpoint length(LinearIndices(a)) + frame, bp = @interpret sum(a) + @test debug_command(frame, :c) === nothing + @test get_return(frame) == sum(a) + end + + @testset "Stepping over kwfunc preparation" begin + stepkw! = JuliaInterpreter.maybe_step_through_kwprep! # for brevity + a = [4, 1, 3, 2] + reversesort(x) = sort(x; rev=true) + frame = JuliaInterpreter.enter_call(reversesort, a) + frame = stepkw!(frame) + @test frame.pc == JuliaInterpreter.nstatements(frame.framecode) - 1 + + scopedreversesort(x) = Base.sort(x, rev=true) # https://github.com/JuliaDebug/Debugger.jl/issues/141 + frame = JuliaInterpreter.enter_call(scopedreversesort, a) + frame = stepkw!(frame) + @test frame.pc == JuliaInterpreter.nstatements(frame.framecode) - 1 + + frame = JuliaInterpreter.enter_call(sort, a) + frame = stepkw!(frame) + @test frame.pc == JuliaInterpreter.nstatements(frame.framecode) - 1 + + frame, pc = debug_command(frame, :s) + frame, pc = debug_command(frame, :se) # get past copymutable + frame = stepkw!(frame) + @test frame.pc > 4 + + frame = JuliaInterpreter.enter_call(sort, a; rev=true) + frame, pc = debug_command(frame, :se) + frame, pc = debug_command(frame, :s) + frame, pc = debug_command(frame, :se) # get past copymutable + frame = stepkw!(frame) + @test frame.pc == JuliaInterpreter.nstatements(frame.framecode) - 1 + end + + function f(x, y) + sin(2.0) + g(x; y = 3) + end + g(x; y) = x + y + @testset "interaction of :n with kw functions" begin + frame = JuliaInterpreter.enter_call(f, 2, 3) # at sin + frame, pc = debug_command(frame, :n) + # Check that we are at the kw call to g + @test Core.kwfunc(g) == JuliaInterpreter.@lookup frame JuliaInterpreter.pc_expr(frame).args[1] + # Step into the inner g + frame, pc = debug_command(frame, :s) + # Finish the frame and make sure we step out of the wrapper + frame, pc = debug_command(frame, :finish) + @test frame.framecode.scope == @which f(2, 3) + end + + h_1(x, y) = h_2(x, y) + h_2(x, y) = h_3(x; y=y) + h_3(x; y = 2) = x + y + @testset "stepping through kwprep after stepping through wrapper" begin + frame = JuliaInterpreter.enter_call(h_1, 2, 1) + frame, pc = debug_command(frame, :s) + # Should have skipped the kwprep in h_2 and be at call to kwfunc h_3 + @test Core.kwfunc(h_3) == JuliaInterpreter.@lookup frame JuliaInterpreter.pc_expr(frame).args[1] + end + + @testset "si should not step through wrappers or kwprep" begin + frame = JuliaInterpreter.enter_call(h_1, 2, 1) + frame, pc = debug_command(frame, :si) + @test frame.pc == 1 + end + + @testset "breakpoints hit during wrapper step through" begin + f(x = g()) = x + g() = 5 + @breakpoint g() + frame = JuliaInterpreter.enter_call(f) + JuliaInterpreter.maybe_step_through_wrapper!(frame) + @test leaf(frame).framecode.scope == @which g() + end + + @testset "preservation of stack when throwing to toplevel" begin + f() = "αβ"[2] + frame1 = JuliaInterpreter.enter_call(f); + err = try debug_command(frame1, :c) + catch err + err + end + try + break_on(:error) + frame2, pc = @interpret f() + @test leaf(frame2).framecode.scope === leaf(frame1).framecode.scope + finally + break_off(:error) + end + end + + @testset "breakpoint in next line" begin + function f(a, b) + a == 0 && return abs(b) + @bp + return b + end + + frame = JuliaInterpreter.enter_call(f, 5, 10) + frame, pc = JuliaInterpreter.debug_command(frame, :n) + @test pc isa BreakpointRef + end + + @testset "kw wrapper heuristic #435" begin + foo() = UInt8('\t') + frame = JuliaInterpreter.enter_call(foo) + frame, pc = debug_command(frame, :s) + @test pc isa BreakpointRef + end +# end + +module Foo + using ..JuliaInterpreter + function f(x) + x + @bp + x + end +end +@testset "interpreted methods" begin + g(x) = Foo.f(x) + + push!(JuliaInterpreter.compiled_modules, Foo) + frame = JuliaInterpreter.enter_call(g, 5) + frame, pc = JuliaInterpreter.debug_command(frame, :n) + @test !(pc isa BreakpointRef) + + push!(JuliaInterpreter.interpreted_methods, first(methods(Foo.f))) + frame = JuliaInterpreter.enter_call(g, 5) + frame, pc = JuliaInterpreter.debug_command(frame, :n) + @test pc isa BreakpointRef +end + +@testset "step last call on line" begin + g(x) = x + f(x) = x + h(x) = x + function z(x) + a = h(f(x) + g(x) - 3) + x = 3 + b = h(g(x)) + end + frame = JuliaInterpreter.enter_call(z, 5) + frame, pc = JuliaInterpreter.debug_command(frame, :sl) + @test JuliaInterpreter.scopeof(frame).name === :h + frame, pc = JuliaInterpreter.debug_command(frame, :finish) + frame, pc = JuliaInterpreter.debug_command(frame, :sl) + @test JuliaInterpreter.scopeof(frame).name === :h +end + +@testset "step until return" begin + function f(x) + if x == 1 + return 2 + end + return 3 + end + frame = JuliaInterpreter.enter_call(f, 1) + frame, _ = JuliaInterpreter.debug_command(frame, :sr) + @test JuliaInterpreter.get_return(frame) == f(1) + frame = JuliaInterpreter.enter_call(f, 4) + frame, _ = JuliaInterpreter.debug_command(frame, :sr) + @test JuliaInterpreter.get_return(frame) == f(4) + function g() + y = f(1) + f(2) + return y + end + frame = JuliaInterpreter.enter_call(g) + frame, _ = JuliaInterpreter.debug_command(frame, :sr) + @test JuliaInterpreter.get_return(frame) == g() +end diff --git a/packages/JuliaInterpreter/test/dummy_file.jl b/packages/JuliaInterpreter/test/dummy_file.jl new file mode 100644 index 0000000..7c77efe --- /dev/null +++ b/packages/JuliaInterpreter/test/dummy_file.jl @@ -0,0 +1,2 @@ +# Don't change the value below, it's used in an `include` test +55 diff --git a/packages/JuliaInterpreter/test/eval_code.jl b/packages/JuliaInterpreter/test/eval_code.jl new file mode 100644 index 0000000..d0aee34 --- /dev/null +++ b/packages/JuliaInterpreter/test/eval_code.jl @@ -0,0 +1,111 @@ +import JuliaInterpreter.eval_code + +# Simple evaling of function argument +function evalfoo1(x,y) + x+y +end +frame = JuliaInterpreter.enter_call(evalfoo1, 1, 2) +@test eval_code(frame, "x") == 1 +@test eval_code(frame, "y") == 2 + +# Evaling with sparams +evalsparams(x::T) where T = x +frame = JuliaInterpreter.enter_call(evalsparams, 1) +@test eval_code(frame, "x") == 1 +eval_code(frame, "x = 3") +@test eval_code(frame, "x") == 3 +@test eval_code(frame, "T") == Int +eval_code(frame, "T = Float32") +@test eval_code(frame, "T") == Float32 + +# Evaling with keywords +evalkw(x; bar=true) = x +frame = JuliaInterpreter.enter_call(evalkw, 2) +frame = JuliaInterpreter.maybe_step_through_wrapper!(frame) +@test eval_code(frame, "x") == 2 +@test eval_code(frame, "bar") == true +eval_code(frame, "bar = false") +@test eval_code(frame, "bar") == false + +# Evaling with symbols +evalsym() = (x = :foo) +frame = JuliaInterpreter.enter_call(evalsym) +# Step until the local actually end up getting defined +JuliaInterpreter.step_expr!(frame) +JuliaInterpreter.step_expr!(frame) +@test eval_code(frame, "x") === :foo + +# Evaling multiple statements (https://github.com/JuliaDebug/Debugger.jl/issues/188) +frame = JuliaInterpreter.enter_call(evalfoo1, 1, 2) +@test eval_code(frame, "x = 1; y = 2") == 2 +@test eval_code(frame, "x") == 1 +@test eval_code(frame, "y") == 2 + +# https://github.com/JuliaDebug/Debugger.jl/issues/177 +function f() + x = 1 + f = ()->(x = 2) + f() + x +end +frame = JuliaInterpreter.enter_call(f) +JuliaInterpreter.step_expr!(frame) +JuliaInterpreter.step_expr!(frame) +@test eval_code(frame, "x") == 1 +eval_code(frame, "x = 3") +@test eval_code(frame, "x") == 3 +JuliaInterpreter.finish!(frame) +@test JuliaInterpreter.get_return(frame) == 2 + +function debugfun(non_accessible_variable) + garbage = ones(10) + map(1:10) do i + 1+1 + a = 5 + @bp + garbage[i] + non_accessible_variable[i] + non_accessible_variable = 2 + end +end +fr = JuliaInterpreter.enter_call(debugfun, [1,2]) +fr, bp = debug_command(fr, :c) +@test eval_code(fr, "non_accessible_variable") == [1,2] +@test eval_code(fr, "garbage") == ones(10) +eval_code(fr, "non_accessible_variable = 5.0") +@test eval_code(fr, "non_accessible_variable") == 5.0 + +# Evaluating SSAValues +f(x) = x^2 +frame = JuliaInterpreter.enter_call(f, 5) +JuliaInterpreter.step_expr!(frame) +JuliaInterpreter.step_expr!(frame) +# This could change with changes to Julia lowering +@test eval_code(frame, "var\"%2\"") == Val(2) +@test eval_code(frame, "var\"@_1\"") == f + +function fun(;output=:sym) + x = 5 + y = 3 +end +fr = JuliaInterpreter.enter_call(fun) +fr = JuliaInterpreter.maybe_step_through_wrapper!(fr) +JuliaInterpreter.step_expr!(fr) +@test eval_code(fr, "x") == 5 +@test eval_code(fr, "output") === :sym +eval_code(fr, "output = :foo") +@test eval_code(fr, "output") === :foo + +let f() = GlobalRef(Main, :doesnotexist) + fr = JuliaInterpreter.enter_call(f) + JuliaInterpreter.step_expr!(fr) + @test eval_code(fr, "var\"%1\"") == GlobalRef(Main, :doesnotexist) +end + +# Don't error on empty input string +function empty_code(x) + x+x +end +frame = JuliaInterpreter.enter_call(empty_code, 1) +@test eval_code(frame, "") === nothing +@test eval_code(frame, " ") === nothing +@test eval_code(frame, "\n") === nothing diff --git a/packages/JuliaInterpreter/test/interpret.jl b/packages/JuliaInterpreter/test/interpret.jl new file mode 100644 index 0000000..d2a1763 --- /dev/null +++ b/packages/JuliaInterpreter/test/interpret.jl @@ -0,0 +1,981 @@ +using JuliaInterpreter +using JuliaInterpreter: enter_call_expr +using Test, InteractiveUtils, CodeTracking +using Mmap +using LinearAlgebra + +if !isdefined(@__MODULE__, :runframe) + include("utils.jl") +end + +module Isolated end + +function summer(A) + s = zero(eltype(A)) + for a in A + s += a + end + return s +end + +A = [0.12, -.99] +frame = JuliaInterpreter.enter_call(summer, A) +frame2 = JuliaInterpreter.enter_call(summer, A) +@test summer(A) == something(runframe(frame)) == something(runstack(frame2)) + +A = rand(1000) +@test @interpret(sum(A)) ≈ sum(A) # note: the compiler can leave things in registers to increase accuracy, doesn't happen with interpreted +fapply() = (Core.apply_type)(Base.NamedTuple, (), Tuple{}) +@test @interpret(fapply()) == fapply() +function fbc() + bc = Broadcast.broadcasted(CartesianIndex, 6, [1, 2, 3]) + copy(bc) +end +@test @interpret(fbc()) == fbc() +@test @interpret(repr("hi")) == repr("hi") # this tests kwargs and @generated functions + +fkw(x::Int8; y=0, z="hello") = y +@test @interpret(fkw(Int8(1); y=22, z="world")) == fkw(Int8(1); y=22, z="world") + +# generators that throw before returning the body expression +@test_throws ArgumentError("input tuple of length 3, requested 2") @interpret Base.fill_to_length((1,2,3), -1, Val(2)) + +# Throwing exceptions across frames +function f_exc_inner() + error("inner") +end + +f_exc_inner2() = f_exc_inner() + +const caught = Ref(false) +function f_exc_outer1() + try + f_exc_inner() + catch err # with an explicit err capture + caught[] = true + rethrow(err) + end +end + +function f_exc_outer2() + try + f_exc_inner() + catch # implicit err capture + caught[] = true + rethrow() + end +end + +function f_exc_outer3(f) + try + f() + catch err + return err + end +end + +@test !caught[] +ret = @interpret f_exc_outer3(f_exc_outer1) +@test ret == ErrorException("inner") +@test caught[] + +caught[] = false +ret = @interpret f_exc_outer3(f_exc_outer2) +@test ret == ErrorException("inner") +@test caught[] + +caught[] = false +ret = @interpret f_exc_outer3(f_exc_inner2) +@test ret == ErrorException("inner") +@test !caught[] + + +stc = try f_exc_outer1() catch + stacktrace(catch_backtrace()) +end +sti = try @interpret(f_exc_outer1()) catch + stacktrace(catch_backtrace()) +end +@test_broken stc == sti + +# issue #3 +@test @interpret(joinpath("/home/julia/base", "sysimg.jl")) == joinpath("/home/julia/base", "sysimg.jl") +@test @interpret(10.0^4) == 10.0^4 +# issue #6 +@test @interpret(Array.body.body.name) === Array.body.body.name +if Vararg isa UnionAll + @test @interpret(Vararg.body.body.name) === Vararg.body.body.name +else + @test @interpret(Vararg{Int}.T) === Vararg{Int}.T + @test @interpret(Vararg{Any,3}.N) === Vararg{Any,3}.N +end +@test !JuliaInterpreter.is_vararg_type(Union{}) +if Vararg isa UnionAll + frame = Frame(Main, :(Vararg.body.body.name)) + @test JuliaInterpreter.finish_and_return!(frame, true) === Vararg.body.body.name +else + frame = Frame(Main, :(Vararg{Int}.T)) + @test JuliaInterpreter.finish_and_return!(frame, true) === Vararg{Int}.T + frame = Frame(Main, :(Vararg{Any,3}.N)) + @test JuliaInterpreter.finish_and_return!(frame, true) === Vararg{Any,3}.N +end +frame = Frame(Base, :(Union{AbstractChar,Tuple{Vararg{AbstractChar}},AbstractVector{<:AbstractChar},Set{<:AbstractChar}})) +@test JuliaInterpreter.finish_and_return!(frame, true) isa Union + +# issue #8 +ex = quote + if sizeof(JLOptions) === ccall(:jl_sizeof_jl_options, Int, ()) + else + ccall(:jl_throw, Cvoid, (Any,), "Option structure mismatch") + end +end +frame = Frame(Base, ex) +JuliaInterpreter.finish_and_return!(frame, true) + +# ccall with two Symbols +ex = quote + @testset "Some tests" begin + @test 2 > 1 + end +end +frame = Frame(Main, ex) +JuliaInterpreter.finish_and_return!(frame, true) + +@test @interpret Base.Math.DoubleFloat64(-0.5707963267948967, 4.9789962508669555e-17).hi ≈ -0.5707963267948967 + +# ccall with cfunction +fcfun(x::Int, y::Int) = 1 +ex = quote # in lowered code, cf is a Symbol + cf = @eval @cfunction(fcfun, Int, (Int, Int)) + ccall(cf, Int, (Int, Int), 1, 2) +end +frame = Frame(Main, ex) +@test JuliaInterpreter.finish_and_return!(frame, true) == 1 +ex = quote + let # in lowered code, cf is a SlotNumber + cf = @eval @cfunction(fcfun, Int, (Int, Int)) + ccall(cf, Int, (Int, Int), 1, 2) + end +end +frame = Frame(Main, ex) +@test JuliaInterpreter.finish_and_return!(frame, true) == 1 +function cfcfun() + cf = @cfunction(fcfun, Int, (Int, Int)) + ccall(cf, Int, (Int, Int), 1, 2) +end +@test @interpret(cfcfun()) == 1 + +# From Julia's test/ambiguous.jl. This tests whether we renumber :enter statements correctly. +ambig(x, y) = 1 +ambig(x::Integer, y) = 2 +ambig(x, y::Integer) = 3 +ambig(x::Int, y::Int) = 4 +ambig(x::Number, y) = 5 +ex = quote + let + cf = @eval @cfunction(ambig, Int, (UInt8, Int)) + @test_throws(MethodError, ccall(cf, Int, (UInt8, Int), 1, 2)) + end +end +frame = Frame(Main, ex) +JuliaInterpreter.finish_and_return!(frame, true) + +# Core.Compiler +ex = quote + length(code_typed(fcfun, (Int, Int))) +end +frame = Frame(Main, ex) +@test JuliaInterpreter.finish_and_return!(frame, true) == 1 + +# copyast +ex = quote + struct CodegenParams + cached::Cint + + track_allocations::Cint + code_coverage::Cint + static_alloc::Cint + prefer_specsig::Cint + + module_setup::Any + module_activation::Any + raise_exception::Any + emit_function::Any + emitted_function::Any + + CodegenParams(;cached::Bool=true, + track_allocations::Bool=true, code_coverage::Bool=true, + static_alloc::Bool=true, prefer_specsig::Bool=false, + module_setup=nothing, module_activation=nothing, raise_exception=nothing, + emit_function=nothing, emitted_function=nothing) = + new(Cint(cached), + Cint(track_allocations), Cint(code_coverage), + Cint(static_alloc), Cint(prefer_specsig), + module_setup, module_activation, raise_exception, + emit_function, emitted_function) + end +end +frame = Frame(Isolated, ex) +JuliaInterpreter.finish_and_return!(frame, true) +@test Isolated.CodegenParams(cached=false).cached === Cint(false) + +# cglobal +val = @interpret(BigInt()) +@test isa(val, BigInt) && val == 0 +@test isa(@interpret(Base.GMP.version()), VersionNumber) + +# Issue #455 +using PyCall +let np = pyimport("numpy") + @test @interpret(PyCall.pystring_query(np.zeros)) === Union{} +end +# Issue #354 +using HTTP +headers = Dict("User-Agent" => "Debugger.jl") +@test @interpret(HTTP.request("GET", "https://httpbingo.julialang.org", headers)) isa HTTP.Messages.Response + +# "correct" line numbers +defline = @__LINE__() + 1 +function f(x) + x = 2x + # comment + # comment + x = 2x + # comment + return x*x +end +frame = JuliaInterpreter.enter_call(f, 3) +@test whereis(frame, 1)[2] == defline + 1 +@test whereis(frame, 3)[2] == defline + 4 +@test whereis(frame, 5)[2] == defline + 6 +m = which(iterate, Tuple{Dict}) # this method has `nothing` as its first statement and codeloc == 0 +framecode = JuliaInterpreter.get_framecode(m) +@test JuliaInterpreter.linenumber(framecode, 1) == m.line + CodeTracking.line_is_decl + +# issue #28 +let a = ['0'], b = ['a'] + @test @interpret(vcat(a, b)) == vcat(a, b) +end + +# issue #51 +if isdefined(Core.Compiler, :SNCA) + ci = @code_lowered gcd(10, 20) + cfg = Core.Compiler.compute_basic_blocks(ci.code) + @test isa(@interpret(Core.Compiler.SNCA(cfg)), Vector{Int}) +end + +# llvmcall +function add1234(x::Tuple{Int32,Int32,Int32,Int32}) + Base.llvmcall("""%3 = extractvalue [4 x i32] %0, 0 + %4 = extractvalue [4 x i32] %0, 1 + %5 = extractvalue [4 x i32] %0, 2 + %6 = extractvalue [4 x i32] %0, 3 + %7 = extractvalue [4 x i32] %1, 0 + %8 = extractvalue [4 x i32] %1, 1 + %9 = extractvalue [4 x i32] %1, 2 + %10 = extractvalue [4 x i32] %1, 3 + %11 = add i32 %3, %7 + %12 = add i32 %4, %8 + %13 = add i32 %5, %9 + %14 = add i32 %6, %10 + %15 = insertvalue [4 x i32] undef, i32 %11, 0 + %16 = insertvalue [4 x i32] %15, i32 %12, 1 + %17 = insertvalue [4 x i32] %16, i32 %13, 2 + %18 = insertvalue [4 x i32] %17, i32 %14, 3 + ret [4 x i32] %18""",Tuple{Int32,Int32,Int32,Int32}, + Tuple{Tuple{Int32,Int32,Int32,Int32},Tuple{Int32,Int32,Int32,Int32}}, + (Int32(1),Int32(2),Int32(3),Int32(4)), + x) +end +@test @interpret(add1234(map(Int32,(2,3,4,5)))) === map(Int32,(3,5,7,9)) + +# issue #74 +let A = [1] + wkd = WeakKeyDict() + @interpret setindex!(wkd, 2, A) + @test wkd[A] == 2 +end + +# issue #76 +let TT = Union{UInt8, Int8} + a = TT[0x0, 0x1] + pa = Ptr{UInt8}(pointer(a)) + GC.@preserve a begin + @interpret unsafe_store!(pa, 0x2, 2) + end + @test a == TT[0x0, 0x2] +end + +# issue #92 +let x = Core.TypedSlot(1, Any) + f(x) = objectid(x) + @test isa(@interpret(f(x)), UInt) +end + +# issue #98 +x98 = 5 +function f98() + global x98 + x98 = 7 + return nothing +end +@interpret f98() +@test x98 == 7 + +# issue #106 +function f106() + n = tempname() + w = open(n, "a") + write(w, "A") + flush(w) + return true +end +@test @interpret(f106()) == 1 +f106b() = rand() +f106c() = disable_sigint(f106b) +function f106d() + disable_sigint() do + reenable_sigint(f106b) + end +end +@interpret f106c() +@interpret f106d() + +# issue #113 +f113(;x) = x +@test @interpret(f113(;x=[1,2,3])) == f113(;x=[1,2,3]) + +# Some expressions can appear nontrivial but lower to nothing +# @test isa(Frame(Main, :(@static if ccall(:jl_get_UNAME, Any, ()) === :NoOS 1+1 end)), Nothing) +# @test isa(Frame(Main, :(Base.BaseDocs.@kw_str "using")), Nothing) + +@testset "locals" begin + f_locals(x::Int64, y::T, z::Vararg{Symbol}) where {T} = x + frame = JuliaInterpreter.enter_call(f_locals, Int64(1), 2.0, :a, :b) + locals = JuliaInterpreter.locals(frame) + @test JuliaInterpreter.Variable(Int64(1), :x, false) in locals + @test JuliaInterpreter.Variable(2.0, :y, false) in locals + @test JuliaInterpreter.Variable((:a, :b), :z, false) in locals + @test JuliaInterpreter.Variable(Float64, :T, true) in locals + + function f_multi(x) + c = x + x = 2 + x = 3 + x = 4 + return x + end + frame = JuliaInterpreter.enter_call(f_multi, 1) + nlocals = length(frame.framedata.locals) + @test_throws UndefVarError JuliaInterpreter.lookup_var(frame, JuliaInterpreter.SlotNumber(nlocals)) + stack = [frame] + locals = JuliaInterpreter.locals(frame) + @test length(locals) == 2 + @test JuliaInterpreter.Variable(1, :x, false) in locals + JuliaInterpreter.step_expr!(stack, frame) + JuliaInterpreter.step_expr!(stack, frame) + locals = JuliaInterpreter.locals(frame) + @test length(locals) == 3 + @test JuliaInterpreter.Variable(1, :c, false) in locals + JuliaInterpreter.step_expr!(stack, frame) + locals = JuliaInterpreter.locals(frame) + @test length(locals) == 3 + @test JuliaInterpreter.Variable(2, :x, false) in locals + JuliaInterpreter.step_expr!(stack, frame) + locals = JuliaInterpreter.locals(frame) + @test length(locals) == 3 + @test JuliaInterpreter.Variable(3, :x, false) in locals + + # Issue #404 + function aaa(F::Array{T,1}, Z::Array{T,1}) where {T} + M = length(Z) + + J = [1:M;] + z = T[] + f = T[] + w = T[] + + A = rand(10, 10) + G = svd(A[J, :]) + w = G.V[:, m] + + r = zz -> rhandle(zz, z, f, w) + end + + function rhandle(zz, z, f, w) + nothing + end + + fr = JuliaInterpreter.enter_call(aaa, rand(5), rand(5)) + fr, bp = JuliaInterpreter.debug_command(fr, :n) + locs = JuliaInterpreter.locals(fr) + @test !any(x -> x.name === :w, locs) +end + +@testset "getfield replacements" begin + f_gf(x) = false ? some_undef_var_zzzzzzz : x + @test @interpret f_gf(2) == 2 + + function g_gf() + eval(:(z = 2)) + return z + end + @test @interpret g_gf() == 2 + + global q_gf = 0 + function h_gf() + eval(:(q_gf = 2)) + return q_gf + end + @test @interpret h_gf() == 2 + + # https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/267 + function test_never_different(x) + if x < 5 + for g in never_defined + print(g) + end + end + end + @test @interpret(test_never_different(10)) === nothing + +end + +# https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/130 +@testset "vararg handling" begin + method_c1(x::Float64, s::AbstractString...) = true + buf = IOBuffer() + me = Base.MethodError(method_c1,(1, 1, "")) + @test (@interpret Base.show_method_candidates(buf, me)) == nothing + + varargidentity(x) = x + x = Union{Array{UInt8,N},Array{Int8,N}} where N + @test isa(JuliaInterpreter.prepare_call(varargidentity, [varargidentity, x])[1], JuliaInterpreter.FrameCode) +end + +# https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/141 +@test @interpret get(ENV, "THIS_IS_NOT_DEFINED_1234", "24") == "24" + +# Test return value of whereis +f() = nothing +fr = JuliaInterpreter.enter_call(f) +file, line = JuliaInterpreter.whereis(fr) +@test file == @__FILE__ +@test line == (@__LINE__() - 4) + +# Test path to files in stdlib +fr = JuliaInterpreter.enter_call(Test.eval, 1) +file, line = JuliaInterpreter.whereis(fr) +@test isfile(file) +@test isfile(JuliaInterpreter.getfile(fr.framecode.src.linetable[1])) +if VERSION < v"1.9.0-DEV.846" # https://github.com/JuliaLang/julia/pull/45069 + @test occursin(Sys.STDLIB, repr(fr)) +else + @test occursin(contractuser(Sys.STDLIB), repr(fr)) +end + +# Test undef sparam (https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/165) +function foo(x::T) where {T <: AbstractString, S <: AbstractString} + return S +end +e = try + @interpret foo("") + catch err + err + end +@test e isa UndefVarError +@test e.var === :S +# https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/200 +locs = JuliaInterpreter.locals(JuliaInterpreter.enter_call(foo, "")) +@test length(locs) == 3 # #self# + 2 variables +@test JuliaInterpreter.Variable("", :x, false) in locs +@test JuliaInterpreter.Variable(String, :T, true) in locs + +# Test interpreting subtypes finishes in a reasonable time +@test @interpret subtypes(Integer) == subtypes(Integer) +@test @interpret subtypes(Main, Integer) == subtypes(Main, Integer) +@test (@elapsed @interpret subtypes(Integer)) < 30 +@test (@elapsed @interpret subtypes(Main, Integer)) < 30 + +# Test showing stacktraces from frames +g_1(x) = g_2(x) +g_2(x) = g_3(x) +g_3(x) = error("foo") +line_g = @__LINE__ + +if isdefined(Base, :replaceuserpath) + _contractuser = Base.replaceuserpath +else + _contractuser = Base.contractuser +end + +try + break_on(:error) + local frame, bp = @interpret g_1(2.0) + stacktrace_lines = split(sprint(Base.display_error, bp.err, leaf(frame)), '\n') + @test occursin(string("ERROR: ", sprint(showerror, ErrorException("foo"))), stacktrace_lines[1]) + if isdefined(Base, :print_stackframe) + @test occursin("[1] error(s::String)", stacktrace_lines[3]) + @test occursin("[2] g_3(x::Float64)", stacktrace_lines[5]) + thefile = _contractuser(@__FILE__) + @test occursin("$thefile:$(line_g - 1)", stacktrace_lines[6]) + @test occursin("[3] g_2(x::Float64)", stacktrace_lines[7]) + @test occursin("$thefile:$(line_g - 2)", stacktrace_lines[8]) + @test occursin("[4] g_1(x::Float64)", stacktrace_lines[9]) + @test occursin("$thefile:$(line_g - 3)", stacktrace_lines[10]) + else + @test occursin("[1] error(::String) at error.jl:", stacktrace_lines[3]) + @test occursin("[2] g_3(::Float64) at $(@__FILE__):$(line_g - 1)", stacktrace_lines[4]) + @test occursin("[3] g_2(::Float64) at $(@__FILE__):$(line_g - 2)", stacktrace_lines[5]) + @test occursin("[4] g_1(::Float64) at $(@__FILE__):$(line_g - 3)", stacktrace_lines[6]) + end +finally + break_off(:error) +end + +try + break_on(:error) + exs = collect(ExprSplitter(Main, quote + g_1(2.0) + end)) + line2_g = @__LINE__ + local frame = Frame(exs[1]...) + frame, bp = JuliaInterpreter.debug_command(frame, :c, true) + stacktrace_lines = split(sprint(Base.display_error, bp.err, leaf(frame)), '\n') + @test occursin(string("ERROR: ", sprint(showerror, ErrorException("foo"))), stacktrace_lines[1]) + if isdefined(Base, :print_stackframe) + @test occursin("[1] error(s::String)", stacktrace_lines[3]) + thefile = _contractuser(@__FILE__) + @test occursin("[2] g_3(x::Float64)", stacktrace_lines[5]) + @test occursin("$thefile:$(line_g - 1)", stacktrace_lines[6]) + @test occursin("[3] g_2(x::Float64)", stacktrace_lines[7]) + @test occursin("$thefile:$(line_g - 2)", stacktrace_lines[8]) + @test occursin("[4] g_1(x::Float64)", stacktrace_lines[9]) + @test occursin("$thefile:$(line_g - 3)", stacktrace_lines[10]) + @test occursin("[5] top-level scope", stacktrace_lines[11]) + @test occursin("$thefile:$(line2_g - 2)", stacktrace_lines[12]) + else + @test occursin("[1] error(::String) at error.jl:", stacktrace_lines[3]) + @test occursin("[2] g_3(::Float64) at $(@__FILE__):$(line_g - 1)", stacktrace_lines[4]) + @test occursin("[3] g_2(::Float64) at $(@__FILE__):$(line_g - 2)", stacktrace_lines[5]) + @test occursin("[4] g_1(::Float64) at $(@__FILE__):$(line_g - 3)", stacktrace_lines[6]) + @test occursin("[5] top-level scope at $(@__FILE__):$(line2_g - 2)", stacktrace_lines[7]) + end +finally + break_off(:error) +end + +# https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/154 +q = QuoteNode([1]) +@test @interpret deepcopy(q) == q + +# Check #args for builtins (#217) +f217() = <:(Float64, Float32, Float16) +@test_throws ArgumentError @interpret(f217()) + +# issue #220 +function hash220(x::Tuple{Ptr{UInt8},Int}, h::UInt) + h += Base.memhash_seed + ccall(Base.memhash, UInt, (Ptr{UInt8}, Csize_t, UInt32), x[1], x[2], h % UInt32) + h +end +@test @interpret(hash220((Ptr{UInt8}(0),0), UInt(1))) == hash220((Ptr{UInt8}(0),0), UInt(1)) + +# ccall with type parameters +@test (@interpret Base.unsafe_convert(Ptr{Int}, [1,2])) isa Ptr{Int} + +# ccall with call to get the pointer +cf = [@cfunction(fcfun, Int, (Int, Int))] +function call_cf() + ccall(cf[1], Int, (Int, Int), 1, 2) +end +@test (@interpret call_cf()) == call_cf() +let mt = JuliaInterpreter.enter_call(call_cf).framecode.methodtables + @test any(1:length(mt)) do i + isassigned(mt, i) && mt[i] === Compiled() + end +end + +# ccall with integer static parameter +f_N() = Array{Float64, 4}(undef, 1, 3, 2, 1) +@test (@interpret f_N()) isa Array{Float64, 4} + +f() = ccall((:clock, "libc"), Int32, ()) +# See that the method gets compiled +try @interpret f() +catch +end +let mt = JuliaInterpreter.enter_call(f).framecode.methodtables + @test any(1:length(mt)) do i + isassigned(mt, i) && mt[i] === Compiled() + end +end + +# https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/194 +f() = Meta.lower(Main, Meta.parse("(a=1,0)")) +@test @interpret f() == f() + +# Test for vararg ccalls (used by mmap) +function f_mmap() + tmp = tempname() + local b_mmap + try + x = rand(10) + write(tmp, x) + b_mmap = Mmap.mmap(tmp, Vector{Float64}) + @test b_mmap == x + finally + finalize(b_mmap) + rm(tmp) + end +end +@interpret f_mmap() + +# parametric llvmcall (issues #112 and #288) +module VecTest + using Tensors + Vec{N,T} = NTuple{N,VecElement{T}} + # The following test mimic SIMD.jl + const _llvmtypes = Dict{DataType, String}( + Float64 => "double", + Float32 => "float", + Int32 => "i32", + Int64 => "i64" + ) + @generated function vecadd(x::Vec{N, T}, y::Vec{N, T}) where {N, T} + llvmT = _llvmtypes[T] + func = T <: AbstractFloat ? "fadd" : "add" + exp = """ + %3 = $(func) <$(N) x $(llvmT)> %0, %1 + ret <$(N) x $(llvmT)> %3 + """ + return quote + Base.@_inline_meta + Core.getfield(Base, :llvmcall)($exp, Vec{$N, $T}, Tuple{Vec{$N, $T}, Vec{$N, $T}}, x, y) + end + end + f() = 1.0 * one(Tensor{2,3}) +end +let + # NOTE we need to make sure this code block is compiled, since vecadd is generated function, + # but currently `@interpret` doesn't handle a call to generated functions very well + @static if isdefined(Base.Experimental, Symbol("@force_compile")) + Base.Experimental.@force_compile + end + a = (VecElement{Float64}(1.0), VecElement{Float64}(2.0)) + @test @interpret(VecTest.vecadd(a, a)) == VecTest.vecadd(a, a) +end +@test @interpret(VecTest.f()) == [1 0 0; 0 1 0; 0 0 1] + +# Test exception type for undefined variables +f() = s = s + 1 +@test_throws UndefVarError @interpret f() + +# Handling of SSAValues +function f() + z = [Core.SSAValue(5),] + repr(z[1]) +end +@test @interpret f() == f() + +# Test JuliaInterpreter version of #265 +f(x) = x +g(x) = f(x) +@test (@interpret g(5)) == g(5) +f(x) = x*x +@test (@interpret g(5)) == g(5) + +# Regression test https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/328 +module DataFramesTest + using Test + using JuliaInterpreter + using DataFrames + function df_debug1() + df = DataFrame(A=1:3, B=4:6) + df1 = hcat(df[!,[:A]], df[!,[:B]]) + end + @test @interpret(df_debug1()) == df_debug1() +end + +# issue #330 +@test @interpret(Base.PipeEndpoint()) isa Base.PipeEndpoint + +# issue #345 +@noinline f_345() = 1 +frame = JuliaInterpreter.enter_call(f_345) +@test JuliaInterpreter.whereis(frame) == (@__FILE__(), @__LINE__() - 2) + +# issue #285 +using LinearAlgebra, SparseArrays, Random +@testset "issue 285" begin + function solveit(A,b) + return A\b .+ det(A) + end + + Random.seed!(123456) + n = 5 + A = sprand(n,n,0.5) + A = A'*A + b = rand(n) + @test @interpret(solveit(A, b)) == solveit(A, b) +end + +@testset "issue 351" begin + f() = map(x -> 2x, 1:10) + @test @interpret(f()) == f() +end + +@testset "invoke" begin + # Example provided by jmert in #352 + f(d::Diagonal{T}) where {T} = invoke(f, Tuple{AbstractMatrix}, d) + f(m::AbstractMatrix{T}) where {T} = T + D = Diagonal([1.0, 2.0]) + @test @interpret(f(D)) === f(D) + + # issue #441 & #535 + flog() = @info "logging macros" + @test_logs (:info, "logging macros") @test @interpret flog() === nothing + flog2() = @error "this error is ok" + frame = JuliaInterpreter.enter_call(flog2) + @test_logs (:error, "this error is ok") @test debug_command(frame, :c) === nothing +end + +struct A396 + a::Int +end +@testset "constructor locals" begin + frame = JuliaInterpreter.enter_call(A396, 3) + @test length(JuliaInterpreter.locals(frame)) > 0 +end + +@static if Sys.islinux() + @testset "@ccall" begin + f(s) = @ccall strlen(s::Cstring)::Csize_t + @test @interpret(f("asd")) == 3 + end +end + +@testset "#466 parametric_type_to_expr" begin + @test JuliaInterpreter.parametric_type_to_expr(Array) == :(Core.Array{T, N}) +end + +@testset "#476 isdefined QuoteNode" begin + f() = !true + + @generated function g() + ci = @code_lowered f() + ci.code[1] = Expr(:isdefined, QuoteNode(Float64)) + return ci + end + + @test @interpret(g()) === true +end + +const override_world = typemax(Csize_t) - 1 +macro unreachable(ex) + quote + world_counter = cglobal(:jl_world_counter, Csize_t) + regular_world = unsafe_load(world_counter) + + $(Expr(:tryfinally, # don't introduce scope + quote + unsafe_store!(world_counter, $(override_world-1)) + $(esc(ex)) + end, + quote + unsafe_store!(world_counter, regular_world) + end + )) + end +end + +@testset "unreachable worlds" begin + foobar() = 42 + @unreachable foobar() = "nope" + + @test @interpret(foobar()) == foobar() +end + +@testset "issue #479" begin + function f() + ptr = @cfunction(+, Int, (Int, Int)) + ccall(ptr::Ptr{Cvoid}, Int, (Int, Int), 1, 2) + end + @test @interpret(f()) === 3 +end + +@testset "https://github.com/JuliaLang/julia/pull/41018" begin + m = Module() + @eval m begin + struct Foo + foo::Int + bar + end + end + # this shouldn't throw "type DataType has no field hasfreetypevars" + # even after https://github.com/JuliaLang/julia/pull/41018 + @test Int === @interpret Core.Compiler.getfield_tfunc(m.Foo, Core.Compiler.Const(:foo)) +end + +@testset "https://github.com/JuliaDebug/JuliaInterpreter.jl/issues/488" begin + m = Module() + ex = :(foo() = return) + JuliaInterpreter.finish_and_return!(Frame(m, ex), true) + @test isdefined(m, :foo) +end + +# Related to fixing https://github.com/timholy/Revise.jl/issues/625 +module ForInclude end +@testset "include" begin + ex = :(include("dummy_file.jl")) + @test JuliaInterpreter.finish_and_return!(Frame(ForInclude, ex), true) == 55 +end + +@static if VERSION >= v"1.7.0" + @testset "issue #432" begin + function f() + t = @ccall time()::Cint + end + @test @interpret(f()) !== 0 + @test @interpret(f()) !== 0 + end +end + +@testset "issue #385" begin + using FunctionWrappers:FunctionWrapper + @interpret FunctionWrapper{Int,Tuple{}}(()->42) +end + +@testset "issue #550" begin + using FunctionWrappers:FunctionWrapper + f = (obs) -> (obs[1] = obs[3] * obs[4]; obs) + Tout = Vector{Int} + Tin = Tuple{Vector{Int}} + fw = FunctionWrapper{Tout, Tin}(f) + + obs = [0,2,3,4] + @test @interpret(fw(obs)) == fw(obs) +end + +@testset "TypedSlots" begin + function foo(x, y) + z = x + y + if z < 4 + z += 1 + end + u = (x -> x + z)(x) + v = Ref{Union{Int, Missing}}(x)[] + y + return u + v + end + + ci = code_typed(foo, NTuple{2, Int}; optimize=false)[][1] + mi = Core.Compiler.method_instances(foo, NTuple{2, Int})[] + + frameargs = Any[foo, 1, 2] + framecode = JuliaInterpreter.FrameCode(mi.def, ci) + frame = JuliaInterpreter.prepare_frame(framecode, frameargs, mi.sparam_vals) + @test JuliaInterpreter.finish_and_return!(frame) === 8 +end + +@testset "interpretation of unoptimized frame" begin + let # should be able to interprete nested calls within `:foreigncall` expressions + # even if `JuliaInterpreter.optimize!` doesn't flatten them + M = Module() + lwr = Meta.@lower M begin + global foo = @ccall strlen("foo"::Cstring)::Csize_t + foo == 3 + end + src = lwr.args[1]::Core.CodeInfo + frame = Frame(M, src; optimize=false) + @test length(frame.framecode.src.code) == length(src.code) + @test JuliaInterpreter.finish_and_return!(frame, true) + + M = Module() + lwr = Meta.@lower M begin + strp = Ref{Ptr{Cchar}}(0) + fmt = "hi+%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%hhd-%.1f-%.1f-%.1f-%.1f-%.1f-%.1f-%.1f-%.1f-%.1f\n" + len = @ccall asprintf( + strp::Ptr{Ptr{Cchar}}, + fmt::Cstring, + ; # begin varargs + 0x1::UInt8, 0x2::UInt8, 0x3::UInt8, 0x4::UInt8, 0x5::UInt8, 0x6::UInt8, 0x7::UInt8, 0x8::UInt8, 0x9::UInt8, 0xa::UInt8, 0xb::UInt8, 0xc::UInt8, 0xd::UInt8, 0xe::UInt8, 0xf::UInt8, + 1.1::Cfloat, 2.2::Cfloat, 3.3::Cfloat, 4.4::Cfloat, 5.5::Cfloat, 6.6::Cfloat, 7.7::Cfloat, 8.8::Cfloat, 9.9::Cfloat, + )::Cint + str = unsafe_string(strp[], len) + @ccall free(strp[]::Cstring)::Cvoid + str == "hi+1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-1.1-2.2-3.3-4.4-5.5-6.6-7.7-8.8-9.9\n" + end + src = lwr.args[1]::Core.CodeInfo + frame = Frame(M, src; optimize=false) + @test length(frame.framecode.src.code) == length(src.code) + @test JuliaInterpreter.finish_and_return!(frame, true) + end + + iscallexpr(ex::Expr) = ex.head === :call + @test (@interpret iscallexpr(:(sin(3.14)))) +end + +if isdefined(Base, :have_fma) +f_fma() = Base.have_fma(Float64) +@testset "fma" begin + @test (@interpret f_fma()) == f_fma() + a, b, c = (1.0585073227945125, -0.00040303348596386557, 1.5051263504758005e-16) + @test (@interpret muladd(a, b, c)) === muladd(a,b,c) + a = 1.0883740903666346; b = 2/3 + @test (@interpret a^b) === a^b +end +end + +# issue 536 +function foo_536(y::T) where {T} + x = "A" + return ccall(:memcmp, Cint, (Ptr{UInt8}, Ref{T}, Csize_t), + pointer(x), Ref(y), 1) == 0 +end +@test !@interpret foo_536(0x00) +@test @interpret foo_536(UInt8('A')) + +@static if isdefined(Base.Experimental, Symbol("@opaque")) + @test @interpret (Base.Experimental.@opaque x->3*x)(4) == 12 +end + +# CassetteOverlay, issue #552 +@static if VERSION >= v"1.8" +using CassetteOverlay +end + +@static if VERSION >= v"1.8" +function foo() + x = IdDict() + x[:foo] = 1 +end +@MethodTable SinTable; +@testset "CassetteOverlay" begin + pass = @overlaypass SinTable; + @test (@interpret pass(foo)) == 1 +end +end + +using LoopVectorization + +@testset "interpolated llvmcall" begin + function f_lv!(A) + m, n = size(A) + k = 1 + @turbo for j in (k + 1):n + for i in (k + 1):m + A[i, j] -= A[i, k] * A[k, j] + end + end + return A + end + A = rand(5,5) + B = copy(A) + @interpret f_lv!(A) + f_lv!(B) + @test A ≈ B +end + +@testset "nargs foreigncall #560" begin + @test (@interpret string("", "pcre_h.jl")) == string("", "pcre_h.jl") + @test (@interpret Base.strcat("", "build_h.jl")) == Base.strcat("", "build_h.jl") +end diff --git a/packages/JuliaInterpreter/test/juliatests.jl b/packages/JuliaInterpreter/test/juliatests.jl new file mode 100644 index 0000000..6b4bd7e --- /dev/null +++ b/packages/JuliaInterpreter/test/juliatests.jl @@ -0,0 +1,166 @@ +using JuliaInterpreter +using Test, Random, InteractiveUtils, Distributed, Dates + +# Much of this file is taken from Julia's test/runtests.jl file. + +if !isdefined(Main, :read_and_parse) + include("utils.jl") +end + +const juliadir = dirname(dirname(Sys.BINDIR)) +const testdir = joinpath(juliadir, "test") +if isdir(testdir) + include(joinpath(testdir, "choosetests.jl")) +else + @error "Julia's test/ directory not found, can't run Julia tests" +end + +function test_path(test) + t = split(test, '/') + if t[1] in STDLIBS + if length(t) == 2 + return joinpath(STDLIB_DIR, t[1], "test", t[2]) + else + return joinpath(STDLIB_DIR, t[1], "test", "runtests") + end + else + return joinpath(testdir, test) + end +end + +nstmts = 10^4 # very quick, aborts a lot +outputfile = "results.md" +i = 1 +while i <= length(ARGS) + global i + a = ARGS[i] + if a == "--nstmts" + global nstmts = parse(Int, ARGS[i+1]) + deleteat!(ARGS, i:i+1) + elseif a == "--output" + global outputfile = ARGS[i+1] + deleteat!(ARGS, i:i+1) + else + i += 1 + end +end + +tests, _, exit_on_error, seed = choosetests(ARGS) + +function spin_up_worker() + p = addprocs(1)[1] + remotecall_wait(include, p, "utils.jl") + remotecall_wait(configure_test, p) + return p +end + +function spin_up_workers(n) + procs = addprocs(n) + @sync begin + @async for p in procs + remotecall_wait(include, p, "utils.jl") + remotecall_wait(configure_test, p) + end + end + return procs +end + +# Really, we're just going to skip all the tests that run on node1 +const node1_tests = String[] +function move_to_node1(t) + if t in tests + splice!(tests, findfirst(isequal(t), tests)) + push!(node1_tests, t) + end + nothing +end +move_to_node1("precompile") +move_to_node1("SharedArrays") +move_to_node1("stress") +move_to_node1("Distributed") + +@testset "Julia tests" begin + nworkers = min(Sys.CPU_THREADS, length(tests)) + println("Using $nworkers workers") + results = Dict{String,Any}() + tests0 = copy(tests) + all_tasks = Union{Task,Nothing}[] + try + @sync begin + for i = 1:nworkers + @async begin + push!(all_tasks, current_task()) + while length(tests) > 0 + nleft = length(tests) + test = popfirst!(tests) + println(nleft, " remaining, starting ", test, " on task ", i) + local resp + fullpath = test_path(test) * ".jl" + try + resp = disable_sigint() do + p = spin_up_worker() + result = remotecall_fetch(run_test_by_eval, p, test, fullpath, nstmts) + rmprocs(p; waitfor=5) + result + end + catch e + if isa(e, InterruptException) + println("interrupting ", test) + break # rethrow(e) + end + resp = e + if isa(e, ProcessExitedException) + println("exited on ", test) + end + end + results[test] = resp + if resp isa Exception && exit_on_error + skipped = length(tests) + empty!(tests) + end + end + println("Task ", i, " complete") + all_tasks[i] = nothing + end + end + end + catch err + isa(err, InterruptException) || rethrow(err) + # If the test suite was merely interrupted, still print the + # summary, which can be useful to diagnose what's going on + foreach(all_tasks) do task + try + if isa(task, Task) + println("trying to interrupt ", task) + schedule(task, InterruptException(); error=true) + end + catch + end + end + foreach(wait, all_tasks) + end + + open(outputfile, "w") do io + versioninfo(io) + println(io, "Test run at: ", now()) + println(io) + println(io, "Maximum number of statements per lowered expression: ", nstmts) + println(io) + println(io, "| Test file | Passes | Fails | Errors | Broken | Aborted blocks |") + println(io, "| --------- | ------:| -----:| ------:| ------:| --------------:|") + for test in tests0 + result = get(results, test, "") + if isa(result, Tuple{Test.AbstractTestSet, Vector}) + ts, aborts = result + passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = Test.get_test_counts(ts) + naborts = length(aborts) + println(io, "| ", test, " | ", passes+c_passes, " | ", fails+c_fails, " | ", errors+c_errors, " | ", broken+c_broken, " | ", naborts, " |") + elseif isa(result, ProcessExitedException) + println(io, "| ", test, " | ☠️ | ☠️ | ☠️ | ☠️ | ☠️ |") + else + println(test, " => ", result) + println(io, "| ", test, " | X | X | X | X | X |") + end + end + end +end diff --git a/packages/JuliaInterpreter/test/limits.jl b/packages/JuliaInterpreter/test/limits.jl new file mode 100644 index 0000000..7ac0a6f --- /dev/null +++ b/packages/JuliaInterpreter/test/limits.jl @@ -0,0 +1,128 @@ +using JuliaInterpreter +using CodeTracking +using Test + +# This is a test-for-tests, verifying the code in utils.jl. +if !isdefined(@__MODULE__, :read_and_parse) + include("utils.jl") +end + +@testset "Abort" begin + ex = Base.parse_input_line(""" + x = 1 + for i = 1:10 + x += 1 + end + let y = 0 + z = 5 + end + if 2 > 1 + println("Hello, world!") + end + @elapsed sum(rand(5)) + """; filename="fake.jl") + modexs = collect(ExprSplitter(Main, ex)) + # find the 3rd assignment statement in the 2nd frame (corresponding to the x += 1 line) + frame = Frame(modexs[2]...) + i = 0 + for k = 1:3 + i = findnext(stmt->isexpr(stmt, :(=)), frame.framecode.src.code, i+1) + end + @test Aborted(frame, i).at.line == 3 + # Check interior of let block + frame = Frame(modexs[3]...) + i = 0 + for k = 1:2 + i = findnext(stmt->isexpr(stmt, :(=)), frame.framecode.src.code, i+1) + end + @test Aborted(frame, i).at.line == 6 + # Check conditional + frame = Frame(modexs[4]...) + i = findfirst(stmt->JuliaInterpreter.is_gotoifnot(stmt), frame.framecode.src.code) + 1 + @test Aborted(frame, i).at.line == 9 + # Check macro + frame = Frame(modexs[5]...) + @test Aborted(frame, 1).at.file === Symbol("fake.jl") + @test whereis(frame, 1; macro_caller=true) == ("fake.jl", 11) +end + +module EvalLimited end + +@testset "evaluate_limited" begin + aborts = Aborted[] + ex = Base.parse_input_line(""" + s = 0 + for i = 1:5 + global s + s += 1 + end + """) + modexs = collect(ExprSplitter(EvalLimited, ex)) + nstmts = 1000 # enough to ensure it finishes + for (mod, ex) in modexs + frame = Frame(mod, ex) + @test isa(frame, Frame) + nstmtsleft = nstmts + while true + ret, nstmtsleft = evaluate_limited!(frame, nstmtsleft, true) + isa(ret, Some{Any}) && break + isa(ret, Aborted) && push!(aborts, ret) + end + end + @test EvalLimited.s == 5 + @test isempty(aborts) + + ex = Base.parse_input_line(""" + s = 0 + for i = 1:50 + global s + s += 1 + end + """; filename="fake.jl") + if length(ex.args) == 2 + # Sadly, on some Julia versions parse_input_line doesn't insert line info at toplevel, so do it manually + insert!(ex.args, 2, LineNumberNode(2, Symbol("fake.jl"))) + insert!(ex.args, 1, LineNumberNode(1, Symbol("fake.jl"))) + end + modexs = collect(ExprSplitter(EvalLimited, ex)) + @static if isdefined(Core, :get_binding_type) + nstmts = 10*12 + 20 # 10 * 12 statements per iteration + α + else + nstmts = 9*12 + 20 # 10 * 9 statements per iteration + α + end + for (mod, ex) in modexs + frame = Frame(mod, ex) + @test isa(frame, Frame) + nstmtsleft = nstmts + while true + ret, nstmtsleft = evaluate_limited!(Compiled(), frame, nstmtsleft, true) + isa(ret, Some{Any}) && break + isa(ret, Aborted) && (push!(aborts, ret); break) + end + end + @test 10 ≤ EvalLimited.s < 50 + @test length(aborts) == 1 + @test aborts[1].at.line ∈ (2, 3, 4, 5) # 2 corresponds to lowering of the for loop + + # Now try again with recursive stack + empty!(aborts) + modexs = collect(ExprSplitter(EvalLimited, ex)) + for (mod, ex) in modexs + frame = Frame(mod, ex) + @test isa(frame, Frame) + nstmtsleft = nstmts + while true + ret, nstmtsleft = evaluate_limited!(frame, nstmtsleft, true) + isa(ret, Some{Any}) && break + isa(ret, Aborted) && (push!(aborts, ret); break) + end + end + @test EvalLimited.s < 5 + @test length(aborts) == 1 + lin = aborts[1].at + if lin.file === Symbol("fake.jl") + @test lin.line ∈ (2, 3, 4, 5) + else + @test lin.method === :iterate || lin.method === :getproperty + end +end diff --git a/packages/JuliaInterpreter/test/runtests.jl b/packages/JuliaInterpreter/test/runtests.jl new file mode 100644 index 0000000..65073af --- /dev/null +++ b/packages/JuliaInterpreter/test/runtests.jl @@ -0,0 +1,25 @@ +using JuliaInterpreter +using Test +using Logging + + +@test isempty(detect_ambiguities(JuliaInterpreter, Base, Core)) + +if !isdefined(@__MODULE__, :read_and_parse) + include("utils.jl") +end + +Core.eval(JuliaInterpreter, :(debug_mode() = true)) + +@testset "Main tests" begin + include("check_builtins.jl") + include("core.jl") + include("interpret.jl") + include("toplevel.jl") + include("limits.jl") + include("eval_code.jl") + include("breakpoints.jl") + @static VERSION >= v"1.8.0-DEV.370" && include("code_coverage/code_coverage.jl") + remove() + include("debug.jl") +end diff --git a/packages/JuliaInterpreter/test/toplevel.jl b/packages/JuliaInterpreter/test/toplevel.jl new file mode 100644 index 0000000..044059d --- /dev/null +++ b/packages/JuliaInterpreter/test/toplevel.jl @@ -0,0 +1,574 @@ +if !isdefined(@__MODULE__, :read_and_parse) + include("utils.jl") +end + +module JIVisible +module JIInvisible +end +end + +@testset "Basics" begin + @test JuliaInterpreter.is_doc_expr(:(@doc "string" sum)) + @test JuliaInterpreter.is_doc_expr(:(Core.@doc "string" sum)) + ex = quote + """ + a docstring + """ + sum + end + @test JuliaInterpreter.is_doc_expr(ex.args[2]) + @test !JuliaInterpreter.is_doc_expr(:(1+1)) + # https://github.com/JunoLab/Juno.jl/issues/271 + ex = quote + """ + Special Docstring + """ + module DocStringTest + function foo() + x = 4 + 5 + end + end + end + modexs = collect(ExprSplitter(Main, ex)) + m, ex = first(modexs) + @test JuliaInterpreter.is_doc_expr(ex.args[2]) + Core.eval(m, ex) + io = IOBuffer() + show(io, @doc(Main.DocStringTest)) + @test occursin("Special", String(take!(io))) + # issue #538 + @test !JuliaInterpreter.is_doc_expr(:(Core.@doc "string")) + ex = quote + @doc("no docstring") + + sum + end + modexs = collect(ExprSplitter(Main, ex)) + m, ex = first(modexs) + @test !JuliaInterpreter.is_doc_expr(ex.args[2]) + + @test !isdefined(Main, :JIInvisible) + collect(ExprSplitter(JIVisible, :(module JIInvisible f() = 1 end))) + @test !isdefined(Main, :JIInvisible) + @test isdefined(JIVisible, :JIInvisible) + mktempdir() do path + push!(LOAD_PATH, path) + open(joinpath(path, "TmpPkg1.jl"), "w") do io + println(io, """ + module TmpPkg1 + using TmpPkg2 + end + """) + end + open(joinpath(path, "TmpPkg2.jl"), "w") do io + println(io, """ + module TmpPkg2 + f() = 1 + end + """) + end + @eval using TmpPkg1 + # Every package is technically parented in Main but the name may not be visible in Main + @test isdefined(@__MODULE__, :TmpPkg1) + @test !isdefined(@__MODULE__, :TmpPkg2) + collect(ExprSplitter(Main, quote + module TmpPkg2 + f() = 2 + end + end)) + @test isdefined(@__MODULE__, :TmpPkg1) + @test !isdefined(@__MODULE__, :TmpPkg2) + end +end + +module Toplevel end + +@testset "toplevel" begin + modexs = ExprSplitter(Toplevel, read_and_parse(joinpath(@__DIR__, "toplevel_script.jl"))) + for (mod, ex) in modexs + frame = Frame(mod, ex) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + end + + @test isconst(Toplevel, :StructParent) + @test isconst(Toplevel, :Struct) + @test isconst(Toplevel, :MyInt8) + + s = Toplevel.Struct([2.0]) + + @test Toplevel.f1(0) == 1 + @test Toplevel.f1(0.0) == 2 + @test Toplevel.f1(0.0f0) == 3 + @test Toplevel.f2("hi") == -1 + @test Toplevel.f2(UInt16(1)) == UInt16 + @test Toplevel.f2(3.2) == 0 + @test Toplevel.f2(view([1,2], 1:1)) == 2 + @test Toplevel.f2([1,2]) == 3 + @test Toplevel.f2(reshape(view([1,2], 1:2), 2, 1)) == 4 + @test Toplevel.f3(1, 1) == 1 + @test Toplevel.f3(1, :hi) == 2 + @test Toplevel.f3(UInt16(1), :hi) == Symbol + @test Toplevel.f3(rand(2, 2), :hi, :there) == 2 + @test_throws MethodError Toplevel.f3([1.0], :hi, :there) + @test Toplevel.f4(1, 1.0) == 1 + @test Toplevel.f4(1, 1) == Toplevel.f4(1) == 2 + @test Toplevel.f4(UInt(1), "hey", 2) == 3 + @test Toplevel.f4(rand(2,2)) == 2 + @test Toplevel.f5(Int8(1); y=22) == 22 + @test Toplevel.f5(Int16(1)) == 2 + @test Toplevel.f5(Int32(1)) == 3 + @test Toplevel.f5(Int64(1)) == 4 + @test Toplevel.f5(rand(2,2); y=7) == 2 + @test Toplevel.f6(1, "hi"; z=8) == 1 + @test Toplevel.f7(1, (1, :hi)) == 1 + @test Toplevel.f8(0) == 1 + @test Toplevel.f9(3) == 9 + @test Toplevel.f9(3.0) == 3.0 + @test s("hello") == [2.0] + @test Toplevel.Struct{Float32}(Dict(1=>"two")) == 4 + @test Toplevel.first_two_funcs == (Toplevel.f1, Toplevel.f2) + @test isconst(Toplevel, :first_two_funcs) + @test Toplevel.myint isa Toplevel.MyInt8 + @test_throws UndefVarError Toplevel.ffalse(1) + @test Toplevel.ftrue(1) == 3 + @test Toplevel.fctrue(0) == 1 + @test_throws UndefVarError Toplevel.fcfalse(0) + @test !Toplevel.Consts.b2 + @test Toplevel.fb1true(0) == 1 + @test_throws UndefVarError Toplevel.fb1false(0) + @test Toplevel.fb2false(0) == 1 + @test_throws UndefVarError Toplevel.fb2true(0) + @test Toplevel.fstrue(0) == 1 + @test Toplevel.fouter(1) === 2 + @test Toplevel.feval1(1.0) === 1 + @test Toplevel.feval1(1.0f0) === 1 + @test_throws MethodError Toplevel.feval1(1) + @test Toplevel.feval2(1.0, Int8(1)) == 2 + @test length(s) === nothing + @test size(s) === nothing + @test Toplevel.nbytes(Float32) == 4 + @test Toplevel.typestring(1.0) == "Float64" + @test Toplevel._feval3(0) == 3 + @test Toplevel.feval_add!(0) == 1 + @test Toplevel.feval_min!(0) == 1 + @test Toplevel.paramtype(Vector{Int8}) == Int8 + @test Toplevel.paramtype(Vector) == Toplevel.NoParam + @test Toplevel.Inner.g() == 5 + @test Toplevel.Inner.InnerInner.g() == 6 + @test isdefined(Toplevel, :Beat) + @test Toplevel.Beat <: Toplevel.DatesMod.Period + + @test @interpret(Toplevel.f1(0)) == 1 + @test @interpret(Toplevel.f1(0.0)) == 2 + @test @interpret(Toplevel.f1(0.0f0)) == 3 + @test @interpret(Toplevel.f2("hi")) == -1 + @test @interpret(Toplevel.f2(UInt16(1))) == UInt16 + @test @interpret(Toplevel.f2(3.2)) == 0 + @test @interpret(Toplevel.f2(view([1,2], 1:1))) == 2 + @test @interpret(Toplevel.f2([1,2])) == 3 + @test @interpret(Toplevel.f2(reshape(view([1,2], 1:2), 2, 1))) == 4 + @test @interpret(Toplevel.f3(1, 1)) == 1 + @test @interpret(Toplevel.f3(1, :hi)) == 2 + @test @interpret(Toplevel.f3(UInt16(1), :hi)) == Symbol + @test @interpret(Toplevel.f3(rand(2, 2), :hi, :there)) == 2 + @test_throws MethodError @interpret(Toplevel.f3([1.0], :hi, :there)) + @test @interpret(Toplevel.f4(1, 1.0)) == 1 + @test @interpret(Toplevel.f4(1, 1)) == @interpret(Toplevel.f4(1)) == 2 + @test @interpret(Toplevel.f4(UInt(1), "hey", 2)) == 3 + @test @interpret(Toplevel.f4(rand(2,2))) == 2 + @test @interpret(Toplevel.f5(Int8(1); y=22)) == 22 + @test @interpret(Toplevel.f5(Int16(1))) == 2 + @test @interpret(Toplevel.f5(Int32(1))) == 3 + @test @interpret(Toplevel.f5(Int64(1))) == 4 + @test @interpret(Toplevel.f5(rand(2,2); y=7)) == 2 + @test @interpret(Toplevel.f6(1, "hi"; z=8)) == 1 + @test @interpret(Toplevel.f7(1, (1, :hi))) == 1 + @test @interpret(Toplevel.f8(0)) == 1 + @test @interpret(Toplevel.f9(3)) == 9 + @test @interpret(Toplevel.f9(3.0)) == 3.0 + @test @interpret(s("hello")) == [2.0] + @test @interpret(Toplevel.Struct{Float32}(Dict(1=>"two"))) == 4 + @test_throws UndefVarError @interpret(Toplevel.ffalse(1)) + @test @interpret(Toplevel.ftrue(1)) == 3 + @test @interpret(Toplevel.fctrue(0)) == 1 + @test_throws UndefVarError @interpret(Toplevel.fcfalse(0)) + @test @interpret(Toplevel.fb1true(0)) == 1 + @test_throws UndefVarError @interpret(Toplevel.fb1false(0)) + @test @interpret(Toplevel.fb2false(0)) == 1 + @test_throws UndefVarError @interpret(Toplevel.fb2true(0)) + @test @interpret(Toplevel.fstrue(0)) == 1 + @test @interpret(Toplevel.fouter(1)) === 2 + @test @interpret(Toplevel.feval1(1.0)) === 1 + @test @interpret(Toplevel.feval1(1.0f0)) === 1 + @test_throws MethodError @interpret(Toplevel.feval1(1)) + @test @interpret(Toplevel.feval2(1.0, Int8(1))) == 2 + @test @interpret(length(s)) === nothing + @test @interpret(size(s)) === nothing + @test @interpret(Toplevel.nbytes(Float32)) == 4 + @test @interpret(Toplevel.typestring(1.0)) == "Float64" + @test @interpret(Toplevel._feval3(0)) == 3 + @test @interpret(Toplevel.feval_add!(0)) == 1 + @test @interpret(Toplevel.feval_min!(0)) == 1 + @test @interpret(Toplevel.paramtype(Vector{Int8})) == Int8 + @test @interpret(Toplevel.paramtype(Vector)) == Toplevel.NoParam + @test @interpret(Toplevel.Inner.g()) == 5 + @test @interpret(Toplevel.Inner.InnerInner.g()) == 6 + # FIXME: even though they pass, these tests break Test! + # @test @interpret(isdefined(Toplevel, :Beat)) + # @test @interpret(Toplevel.Beat <: Toplevel.DatesMod.Period) + + # Check that nested expressions are handled appropriately (module-in-block, internal `using`) + ex = quote + module Testing + if true + using JuliaInterpreter + end + end + end + modexs = ExprSplitter(Toplevel, ex) + for (mod, ex) in modexs + frame = Frame(mod, ex) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + end + @test Toplevel.Testing.Frame === Frame +end + +# Proper handling of namespaces +# https://github.com/timholy/Revise.jl/issues/579 +module Namespace end +@testset "Namespace" begin + frame = Frame(Namespace, :(sin(::Int) = 10)) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + @test Namespace.sin(0) == 10 + @test Base.sin(0) == 0 +end +# When retrospectively parsing through modules to analyze code, Julia's stdlibs pose a bit +# of a namespace challenge too: we never want to redefine new modules with the same name. +@testset "Namespace stdlibs" begin + # Get the "real" LibCURL_jll module (Julia 1.6 and higher) + modref = nothing + for (id, mod) in Base.loaded_modules + if id.name == "LibCURL_jll" + modref = mod + break + end + end + if modref !== nothing + # Now try to find it by splitting + exsplit = JuliaInterpreter.ExprSplitter(Base.__toplevel__, :( + baremodule LibCURL_jll + using Base + Base.Experimental.@compiler_options compile=min optimize=0 infer=false + end)) + (mod1, ex1), state1 = iterate(exsplit) + @test mod1 === modref + end +end + +# incremental interpretation solves world-age problems +# Taken straight from Julia's test/tuple.jl +module IncTest +using Test + +struct A_15703{N} + keys::NTuple{N, Int} +end + +struct B_15703 + x::A_15703 +end +end + +ex = quote + @testset "issue #15703" begin + function bug_15703(xs...) + [x for x in xs] + end + + function test_15703() + s = (1,) + a = A_15703(s) + ss = B_15703(a).x.keys + @test ss === s + bug_15703(ss...) + end + + test_15703() + end +end +modexs = collect(ExprSplitter(IncTest, ex)) +for (i, (mod, ex)) in enumerate(modexs) + local frame = Frame(mod, ex) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + if i == length(modexs) + @test isa(JuliaInterpreter.get_return(frame), Test.DefaultTestSet) + end +end + +@testset "Enum" begin + ex = Expr(:toplevel, + :(@enum EnumParent begin + EnumChild0 + EnumChild1 + end)) + modexs = ExprSplitter(Toplevel, ex) + for (mod, ex) in modexs + frame = Frame(mod, ex) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + end + @test isa(Toplevel.EnumChild1, Toplevel.EnumParent) +end + +module LowerAnon +ret = Ref{Any}(nothing) +end + +@testset "Anonymous functions" begin + ex1 = quote + f = x -> parse(Int16, x) + ret[] = map(f, AbstractString[]) + end + ex2 = quote + ret[] = map(x->parse(Int16, x), AbstractString[]) + end + modexs = ExprSplitter(LowerAnon, ex1) + for (mod, ex) in modexs + frame = Frame(mod, ex) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + end + @test isa(LowerAnon.ret[], Vector{Int16}) + LowerAnon.ret[] = nothing + modexs = ExprSplitter(LowerAnon, ex2) + for (mod, ex) in modexs + frame = Frame(mod, ex) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + end + @test isa(LowerAnon.ret[], Vector{Int16}) + LowerAnon.ret[] = nothing + + ex3 = quote + const BitIntegerType = Union{map(T->Type{T}, Base.BitInteger_types)...} + end + modexs = ExprSplitter(LowerAnon, ex3) + for (mod, ex) in modexs + frame = Frame(mod, ex) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + end + @test isa(LowerAnon.BitIntegerType, Union) + + ex4 = quote + y = 3 + z = map(x->x^2+y, [1,2,3]) + y = 4 + end + modexs = ExprSplitter(LowerAnon, ex4) + for (mod, ex) in modexs + frame = Frame(mod, ex) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + end + @test LowerAnon.z == [4,7,12] +end + +@testset "Docstrings" begin + ex = quote + """ + A docstring + """ + f(x) = 1 + + g(T::Type) = 1 + g(x) = 2 + + """ + Docstring 2 + """ + g(T::Type) + + module Sub + """ + Docstring 3 + """ + f(x) = 2 + end + end + Core.eval(Toplevel, Expr(:toplevel, ex.args...)) + modexs = ExprSplitter(Toplevel, ex) + nt = nsub = 0 + for (mod, ex) in modexs + if JuliaInterpreter.is_doc_expr(ex.args[2]) + mod == Toplevel && (nt += 1) + mod == Toplevel.Sub && (nsub += 1) + ex = ex.args[2].args[4] + ex isa Expr || continue + ex.head === :call && continue + end + frame = Frame(mod, ex) + while true + JuliaInterpreter.through_methoddef_or_done!(frame) === nothing && break + end + end + @test nt == 2 + @test nsub == 1 + @test Toplevel.f("check") == 1 + @test Toplevel.Sub.f("check") == 2 +end + +@testset "Self referential" begin + # Revise issue #304 + ex = :(mutable struct Node t :: Node end) + frame = Frame(Toplevel, ex) + JuliaInterpreter.finish!(frame, true) + @test Toplevel.Node isa Type +end + + +@testset "Non-frames" begin + ex = Base.parse_input_line(""" + \"\"\" + An expr that produces an `export nffoo` that doesn't produce a Frame + \"\"\" + module NonFrame + nfbar(x) = 1 + @deprecate nffoo nfbar + global CoolStuff + const thresh = 1.0 + export nfbar + end + """) + modexs = ExprSplitter(Toplevel, ex) + for (mod, ex) in modexs + if ex.head === :global + Core.eval(mod, ex) + continue + end + frame = Frame(mod, ex) + frame === nothing && continue + JuliaInterpreter.finish!(frame, true) + end + Core.eval(Toplevel, :(using .NonFrame)) + @test isdefined(Toplevel, :nffoo) +end + +@testset "LOAD_PATH and modules" begin + tmpdir = joinpath(tempdir(), randstring()) + mkpath(tmpdir) + push!(LOAD_PATH, tmpdir) + filename = joinpath(tmpdir, "NewModule.jl") + open(filename, "w") do io + print(io, """ + module NewModule + f() = 1 + end""") + end + str = read(filename, String) + ex = Base.parse_input_line(str) + modexs = ExprSplitter(Main, ex) + @test !isempty(modexs) + pop!(LOAD_PATH) + rm(tmpdir, recursive=true) +end + +@testset "`used` for abstract types" begin + ex = :(abstract type AbstractType <: AbstractArray{Union{Int,Missing},2} end) + frame = Frame(Toplevel, ex) + JuliaInterpreter.finish!(frame, true) + @test isabstracttype(Toplevel.AbstractType) +end + +@testset "Recursive type definitions" begin + # See https://github.com/timholy/Revise.jl/issues/417 + ex = :(struct RecursiveType x::Vector{RecursiveType} end) + frame = Frame(Toplevel, ex) + JuliaInterpreter.finish!(frame, true) + @test Toplevel.RecursiveType(Vector{Toplevel.RecursiveType}()) isa Toplevel.RecursiveType +end + +# https://github.com/timholy/Revise.jl/issues/420 +module ToplevelParameters +Base.@kwdef struct MyStruct + x::Array{<:Real, 1} = [.05] +end +end +@testset "Nested references in type definitions" begin + ex = quote + Base.@kwdef struct MyStruct + x::Array{<:Real, 1} = [.05] + end + end + frame = Frame(ToplevelParameters, ex) + @test JuliaInterpreter.finish!(frame, true) === nothing +end + +@testset "Issue #427" begin + ex = :(begin + local foo = 10 + sin(foo) + end) + for (mod, ex) in ExprSplitter(@__MODULE__, ex) + @test JuliaInterpreter.finish!(Frame(mod, ex), true) === nothing + end + @test length(collect(ExprSplitter(@__MODULE__, ex))) == 1 + ex = :(begin + 3 + 7 + module Local + local foo = 10 + sin(foo) + end + end) + modexs = collect(ExprSplitter(@__MODULE__, ex)) + @test length(modexs) == 2 + @test modexs[2][1] == getfield(@__MODULE__, :Local) + for (mod, ex) in modexs + @test JuliaInterpreter.finish!(Frame(mod, ex), true) === nothing + end + ex = :(begin + 3 + 7 + module Local + local foo = 10 + sin(foo) + end + 3 + 7 + end) + modexs = collect(ExprSplitter(@__MODULE__, ex)) + @test length(modexs) == 3 +end + +@testset "toplevel scope annotation" begin + ex = Base.parse_input_line(""" + global foo_g = 10 + sin(foo_g) + """) + modexs = collect(ExprSplitter(@__MODULE__, ex)) + for (mod, ex) in modexs + @test JuliaInterpreter.finish!(Frame(mod, ex), true) === nothing + end + @test length(modexs) == 2 + + ex = Base.parse_input_line(""" + local foo = 10 + sin(42) + """) + modexs = collect(ExprSplitter(@__MODULE__, ex)) + for (mod, ex) in modexs + @test JuliaInterpreter.finish!(Frame(mod, ex), true) === nothing + end + @test length(modexs) == 2 +end diff --git a/packages/JuliaInterpreter/test/toplevel_script.jl b/packages/JuliaInterpreter/test/toplevel_script.jl new file mode 100644 index 0000000..de22a1c --- /dev/null +++ b/packages/JuliaInterpreter/test/toplevel_script.jl @@ -0,0 +1,148 @@ +abstract type StructParent{T,N} <: AbstractArray{T,N} end +struct Struct{T} <: StructParent{T,1} + x::Vector{T} +end +primitive type MyInt8 <: Integer 8 end +MyInt8(x::Integer) = Base.bitcast(MyInt8, convert(Int8, x)) + +myint = MyInt8(2) + +const TypeAlias = Float32 + +# Methods and signatures +f1(x::Int) = 1 +f1(x) = 2 +f1(x::TypeAlias) = 3 +# where signatures +f2(x::T) where T = -1 +f2(x::T) where T<:Integer = T +f2(x::T) where Unsigned<:T<:Real = 0 +f2(x::V) where V<:SubArray{T} where T = 2 +f2(x::V) where V<:Array{T,N} where {T,N} = 3 +f2(x::V) where V<:Base.ReshapedArray{T,N} where T where N = 4 +# Varargs +f3(x::Int, y...) = 1 +f3(x::Int, y::Symbol...) = 2 +f3(x::T, y::U...) where {T<:Integer,U} = U +f3(x::Array{Float64,K}, y::Vararg{Symbol,K}) where K = K +# Default args +f4(x, y=0) = 1 +f4(x, y::Int=0) = 2 +f4(x::UInt, y="hello", z::Int=0) = 3 +f4(x::Array{Float64,K}, y::Int=0) where K = K +# Keyword args +f5(x::Int8; y=0) = y +f5(x::Int16; y::Int=0) = 2 +f5(x::Int32; y="hello", z::Int=0) = 3 +f5(x::Int64;) = 4 +f5(x::Array{Float64,K}; y::Int=0) where K = K +# Default and keyword args +f6(x, y="hello"; z::Int=0) = 1 +# Destructured args +f7(x, (count, name)) = 1 +# Return-type annotations +f8(x)::Int = 1 +# generated functions +@generated function f9(x) + if x <: Integer + return :(x ^ 2) + else + return :(x) + end +end +# Call overloading +(i::Struct)(::String) = i.x +(::Type{Struct{T}})(::Dict) where T = sizeof(T) + +const first_two_funcs = (f1, f2) + +# Conditional methods +if false + ffalse(x) = 2 +end +if true + ftrue(x) = 3 +end + +if 0.8 > 0.2 + fctrue(x) = 1 +else + fcfalse(x) = 1 +end + +module Consts +export b1 +b1 = true +b2 = false +g() = 2 +end +using .Consts +if b1 + fb1true(x) = 1 +else + fb1false(x) = 1 +end +if Consts.b2 + fb2true(x) = 1 +else + fb2false(x) = 1 +end + +if @isdefined(sum) + fstrue(x) = 1 +end + +# Inner methods +function fouter(x) + finner(::Float16) = 2x + return finner(Float16(1)) +end + +## Evaled methods +for T in (Float32, Float64) + @eval feval1(::$T) = 1 +end +for T1 in (Float32, Float64), T2 in (Int8,) + @eval feval2(::$T1, ::$T2) = 2 +end +for f in (:length, :size) + @eval Base.$f(i::Struct, args...) = nothing +end +for (T, v) in Dict(Float32=>4, Float64=>8) + @eval nbytes(::Type{$T}) = $v +end +for x in (1, 1.1) + @eval typestring(::$(typeof(x))) = $(string(typeof(x))) +end +for name in (:feval3,) + _f = Symbol("_", name) + @eval ($_f)(arg) = 3 +end +const opnames = Dict{Symbol, Symbol}(:+ => :add, :- => :sub) +for op in [:+, :-, :max, :min] + opname = get(opnames, op, op) + @eval $(Symbol("feval_", opname, "!"))(var) = 1 +end + +# Methods with @isdefined +struct NoParam end +myeltype(::Type{Vector{T}}) where T = @isdefined(T) ? T : NoParam +paramtype(::Type{V}) where V<:Vector = isa(V, UnionAll) ? myeltype(Base.unwrap_unionall(V)) : myeltype(V) + +## Submodules +module Inner +g() = 5 +module InnerInner +g() = 6 +end +end + +module DatesMod + abstract type Period end +end + +struct Beat <: DatesMod.Period + value::Int64 +end + +module Empty end diff --git a/packages/JuliaInterpreter/test/utils.jl b/packages/JuliaInterpreter/test/utils.jl new file mode 100644 index 0000000..1a3f058 --- /dev/null +++ b/packages/JuliaInterpreter/test/utils.jl @@ -0,0 +1,200 @@ +using JuliaInterpreter +using JuliaInterpreter: Frame, @lookup +using JuliaInterpreter: finish_and_return!, evaluate_call!, step_expr!, shouldbreak, + do_assignment!, SSAValue, isassign, pc_expr, handle_err, get_return, + moduleof +using Base.Meta: isexpr +using Test, Random, SHA + +function stacklength(frame) + n = 1 + frame = frame.callee + while frame !== nothing + n += 1 + frame = frame.callee + end + return n +end + +# Execute a frame using Julia's regular compiled-code dispatch for any :call expressions +runframe(frame) = Some{Any}(finish_and_return!(Compiled(), frame)) + +# Execute a frame using the interpreter for all :call expressions (except builtins & intrinsics) +runstack(frame) = Some{Any}(finish_and_return!(frame)) + +## For juliatests.jl + +function read_and_parse(filename) + src = read(filename, String) + ex = Base.parse_input_line(src; filename=filename) +end + +## For running interpreter frames under resource limitations + +struct Aborted # for signaling that some statement or test blocks were interrupted + at::Core.LineInfoNode +end + +function Aborted(frame::Frame, pc) + src = frame.framecode.src + lineidx = src.codelocs[pc] + lineinfo = JuliaInterpreter.linetable(frame, lineidx; macro_caller=true) + return Aborted(lineinfo) +end + +""" + ret, nstmtsleft = evaluate_limited!(recurse, frame, nstmts, istoplevel::Bool=true) + +Run `frame` until one of: +- execution terminates normally (`ret = Some{Any}(val)`, where `val` is the returned value of `frame`) +- if `istoplevel` and a `thunk` or `method` expression is encountered (`ret = nothing`) +- more than `nstmts` have been executed (`ret = Aborted(lin)`, where `lnn` is the `LineInfoNode` of termination). +""" +function evaluate_limited!(@nospecialize(recurse), frame::Frame, nstmts::Int, istoplevel::Bool=false) + refnstmts = Ref(nstmts) + limexec!(s, f, istl) = limited_exec!(s, f, refnstmts, istl) + # The following is like finish!, except we intercept :call expressions so that we can run them + # with limexec! rather than the default finish_and_return! + pc = frame.pc + while nstmts > 0 + shouldbreak(frame, pc) && return BreakpointRef(frame.framecode, pc), refnstmts[] + stmt = pc_expr(frame, pc) + if isa(stmt, Expr) + if stmt.head === :call && !isa(recurse, Compiled) + refnstmts[] = nstmts + try + rhs = evaluate_call!(limexec!, frame, stmt) + isa(rhs, Aborted) && return rhs, refnstmts[] + lhs = SSAValue(pc) + do_assignment!(frame, lhs, rhs) + new_pc = pc + 1 + catch err + new_pc = handle_err(recurse, frame, err) + end + nstmts = refnstmts[] + elseif stmt.head === :(=) && isexpr(stmt.args[2], :call) && !isa(recurse, Compiled) + refnstmts[] = nstmts + try + rhs = evaluate_call!(limexec!, frame, stmt.args[2]) + isa(rhs, Aborted) && return rhs, refnstmts[] + do_assignment!(frame, stmt.args[1], rhs) + new_pc = pc + 1 + catch err + new_pc = handle_err(recurse, frame, err) + end + nstmts = refnstmts[] + elseif istoplevel && stmt.head === :thunk + code = stmt.args[1] + if length(code.code) == 1 && JuliaInterpreter.is_return(code.code[end]) && isexpr(code.code[end].args[1], :method) + # Julia 1.2+ puts a :thunk before the start of each method + new_pc = pc + 1 + else + refnstmts[] = nstmts + newframe = Frame(moduleof(frame), stmt) + if isa(recurse, Compiled) + finish!(recurse, newframe, true) + else + newframe.caller = frame + frame.callee = newframe + ret = limited_exec!(recurse, newframe, refnstmts, istoplevel) + isa(ret, Aborted) && return ret, refnstmts[] + frame.callee = nothing + end + JuliaInterpreter.recycle(newframe) + # Because thunks may define new methods, return to toplevel + frame.pc = pc + 1 + return nothing, refnstmts[] + end + elseif istoplevel && stmt.head === :method && length(stmt.args) == 3 + step_expr!(recurse, frame, stmt, istoplevel) + frame.pc = pc + 1 + return nothing, nstmts - 1 + else + new_pc = step_expr!(recurse, frame, stmt, istoplevel) + nstmts -= 1 + end + else + new_pc = step_expr!(recurse, frame, stmt, istoplevel) + nstmts -= 1 + end + (new_pc === nothing || isa(new_pc, BreakpointRef)) && break + pc = frame.pc = new_pc + end + # Handle the return + stmt = pc_expr(frame, pc) + if nstmts == 0 && !JuliaInterpreter.is_return(stmt) + ret = Aborted(frame, pc) + return ret, nstmts + end + ret = get_return(frame) + return Some{Any}(ret), nstmts +end + +evaluate_limited!(@nospecialize(recurse), modex::Tuple{Module,Expr,Frame}, nstmts::Int, istoplevel::Bool=true) = + evaluate_limited!(recurse, modex[end], nstmts, istoplevel) +evaluate_limited!(@nospecialize(recurse), modex::Tuple{Module,Expr,Expr}, nstmts::Int, istoplevel::Bool=true) = + Some{Any}(Core.eval(modex[1], modex[3])), nstmts + +evaluate_limited!(frame::Union{Frame, Tuple}, nstmts::Int, istoplevel::Bool=false) = + evaluate_limited!(finish_and_return!, frame, nstmts, istoplevel) + +function limited_exec!(@nospecialize(recurse), newframe, refnstmts, istoplevel) + ret, nleft = evaluate_limited!(recurse, newframe, refnstmts[], istoplevel) + refnstmts[] = nleft + return isa(ret, Aborted) ? ret : something(ret) +end + +### Functions needed on workers for running tests + +function configure_test() + # To run tests efficiently, certain methods must be run in Compiled mode, + # in particular those that are used by the Test infrastructure + cm = JuliaInterpreter.compiled_methods + empty!(cm) + JuliaInterpreter.set_compiled_methods() + push!(cm, which(Test.eval_test, Tuple{Expr, Expr, LineNumberNode})) + push!(cm, which(Test.get_testset, Tuple{})) + push!(cm, which(Test.push_testset, Tuple{Test.AbstractTestSet})) + push!(cm, which(Test.pop_testset, Tuple{})) + for f in (Test.record, Test.finish) + for m in methods(f) + push!(cm, m) + end + end + push!(cm, which(Random.seed!, Tuple{Union{Integer,Vector{UInt32}}})) + push!(cm, which(copy!, Tuple{Random.MersenneTwister, Random.MersenneTwister})) + push!(cm, which(copy, Tuple{Random.MersenneTwister})) + push!(cm, which(Base.include, Tuple{Module, String})) + push!(cm, which(Base.show_backtrace, Tuple{IO, Vector})) + push!(cm, which(Base.show_backtrace, Tuple{IO, Vector{Any}})) + # issue #101 + push!(cm, which(SHA.update!, Tuple{SHA.SHA1_CTX,Vector{UInt8}})) +end + +function run_test_by_eval(test, fullpath, nstmts) + Core.eval(Main, Expr(:toplevel, :(module JuliaTests using Test, Random end), quote + # These must be run at top level, so we can't put this in a function + println("Working on ", $test, "...") + ex = read_and_parse($fullpath) + isexpr(ex, :error) && @error "error parsing $($test): $ex" + aborts = Aborted[] + ts = Test.DefaultTestSet($test) + Test.push_testset(ts) + current_task().storage[:SOURCE_PATH] = $fullpath + modexs = collect(ExprSplitter(JuliaTests, ex)) + for (i, modex) in enumerate(modexs) # having the index can be useful for debugging + nstmtsleft = $nstmts + # mod, ex = modex + # @show mod ex + frame = Frame(modex) + yield() # allow communication between processes + ret, nstmtsleft = evaluate_limited!(frame, nstmtsleft, true) + if isa(ret, Aborted) + push!(aborts, ret) + JuliaInterpreter.finish_stack!(Compiled(), frame, true) + end + end + println("Finished ", $test) + return ts, aborts + end)) +end diff --git a/packages/LoweredCodeUtils/.github/workflows/CI.yml b/packages/LoweredCodeUtils/.github/workflows/CI.yml new file mode 100644 index 0000000..fabc020 --- /dev/null +++ b/packages/LoweredCodeUtils/.github/workflows/CI.yml @@ -0,0 +1,46 @@ +name: CI +on: + pull_request: + push: + branches: + - master + tags: '*' +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.6' # current LTS + - '1' # latest stable + - 'nightly' + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x64 + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v1 + with: + file: lcov.info diff --git a/packages/LoweredCodeUtils/.github/workflows/Documenter.yml b/packages/LoweredCodeUtils/.github/workflows/Documenter.yml new file mode 100644 index 0000000..8faa2e2 --- /dev/null +++ b/packages/LoweredCodeUtils/.github/workflows/Documenter.yml @@ -0,0 +1,18 @@ +name: Documenter +on: + push: + branches: [master] + tags: [v*] + pull_request: + +jobs: + Documenter: + name: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/julia-buildpkg@latest + - uses: julia-actions/julia-docdeploy@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/packages/LoweredCodeUtils/.github/workflows/TagBot.yml b/packages/LoweredCodeUtils/.github/workflows/TagBot.yml new file mode 100644 index 0000000..f49313b --- /dev/null +++ b/packages/LoweredCodeUtils/.github/workflows/TagBot.yml @@ -0,0 +1,15 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/packages/LoweredCodeUtils/.gitignore b/packages/LoweredCodeUtils/.gitignore new file mode 100644 index 0000000..e338037 --- /dev/null +++ b/packages/LoweredCodeUtils/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +/Manifest.toml diff --git a/packages/LoweredCodeUtils/LICENSE b/packages/LoweredCodeUtils/LICENSE new file mode 100644 index 0000000..2cd7369 --- /dev/null +++ b/packages/LoweredCodeUtils/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2019 Tim Holy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/LoweredCodeUtils/Project.toml b/packages/LoweredCodeUtils/Project.toml new file mode 100644 index 0000000..eda58ba --- /dev/null +++ b/packages/LoweredCodeUtils/Project.toml @@ -0,0 +1,20 @@ +name = "LoweredCodeUtils" +uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" +authors = ["Tim Holy "] +version = "2.3.0" + +[deps] +JuliaInterpreter = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" + +[compat] +JuliaInterpreter = "0.9" +julia = "1.6" + +[extras] +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["InteractiveUtils", "Parameters", "Pkg", "Test"] diff --git a/packages/LoweredCodeUtils/README.md b/packages/LoweredCodeUtils/README.md new file mode 100644 index 0000000..4ad1da8 --- /dev/null +++ b/packages/LoweredCodeUtils/README.md @@ -0,0 +1,5 @@ +# LoweredCodeUtils + +[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://JuliaDebug.github.io/LoweredCodeUtils.jl/stable) + +This package performs operations on Julia's [lowered AST](https://docs.julialang.org/en/latest/devdocs/ast/). See the documentation for details. diff --git a/packages/LoweredCodeUtils/docs/.gitignore b/packages/LoweredCodeUtils/docs/.gitignore new file mode 100644 index 0000000..a303fff --- /dev/null +++ b/packages/LoweredCodeUtils/docs/.gitignore @@ -0,0 +1,2 @@ +build/ +site/ diff --git a/packages/LoweredCodeUtils/docs/Project.toml b/packages/LoweredCodeUtils/docs/Project.toml new file mode 100644 index 0000000..1b9ab1f --- /dev/null +++ b/packages/LoweredCodeUtils/docs/Project.toml @@ -0,0 +1,5 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" + +[compat] +Documenter = "0.24" diff --git a/packages/LoweredCodeUtils/docs/make.jl b/packages/LoweredCodeUtils/docs/make.jl new file mode 100644 index 0000000..16c68ee --- /dev/null +++ b/packages/LoweredCodeUtils/docs/make.jl @@ -0,0 +1,14 @@ +using Documenter +using LoweredCodeUtils + +makedocs( + sitename = "LoweredCodeUtils", + format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"), + modules = [LoweredCodeUtils], + pages = ["Home" => "index.md", "signatures.md", "edges.md", "api.md"] +) + +deploydocs( + repo = "github.com/JuliaDebug/LoweredCodeUtils.jl.git", + push_preview = true +) diff --git a/packages/LoweredCodeUtils/docs/src/api.md b/packages/LoweredCodeUtils/docs/src/api.md new file mode 100644 index 0000000..1d434f3 --- /dev/null +++ b/packages/LoweredCodeUtils/docs/src/api.md @@ -0,0 +1,35 @@ +# API + +## Signatures + +```@docs +signature +methoddef! +rename_framemethods! +bodymethod +``` + +## Edges + +```@docs +CodeEdges +lines_required +lines_required! +selective_eval! +selective_eval_fromstart! +``` + +## Internal utilities + +```@docs +LoweredCodeUtils.print_with_code +LoweredCodeUtils.next_or_nothing +LoweredCodeUtils.skip_until +LoweredCodeUtils.MethodInfo +LoweredCodeUtils.identify_framemethod_calls +LoweredCodeUtils.iscallto +LoweredCodeUtils.getcallee +LoweredCodeUtils.find_name_caller_sig +LoweredCodeUtils.replacename! +LoweredCodeUtils.Variable +``` diff --git a/packages/LoweredCodeUtils/docs/src/edges.md b/packages/LoweredCodeUtils/docs/src/edges.md new file mode 100644 index 0000000..2bab98f --- /dev/null +++ b/packages/LoweredCodeUtils/docs/src/edges.md @@ -0,0 +1,285 @@ +# Edges + +Edges here are a graph-theoretic concept relating the connections between individual statements in the source code. +For example, consider + +```julia +julia> ex = quote + s = 0 + k = 5 + for i = 1:3 + global s, k + s += rand(1:5) + k += i + end + end +quote + #= REPL[2]:2 =# + s = 0 + #= REPL[2]:3 =# + k = 5 + #= REPL[2]:4 =# + for i = 1:3 + #= REPL[2]:5 =# + global s, k + #= REPL[2]:6 =# + s += rand(1:5) + #= REPL[2]:7 =# + k += i + end +end + +julia> eval(ex) + +julia> s +10 # random + +julia> k +11 # reproducible +``` + +We lower it, + +``` +julia> lwr = Meta.lower(Main, ex) +:($(Expr(:thunk, CodeInfo( + @ REPL[2]:2 within `top-level scope' +1 ─ s = 0 +│ @ REPL[2]:3 within `top-level scope' +│ k = 5 +│ @ REPL[2]:4 within `top-level scope' +│ %3 = 1:3 +│ #s1 = Base.iterate(%3) +│ %5 = #s1 === nothing +│ %6 = Base.not_int(%5) +└── goto #4 if not %6 +2 ┄ %8 = #s1 +│ i = Core.getfield(%8, 1) +│ %10 = Core.getfield(%8, 2) +│ @ REPL[2]:5 within `top-level scope' +│ global k +│ global s +│ @ REPL[2]:6 within `top-level scope' +│ %13 = 1:5 +│ %14 = rand(%13) +│ %15 = s + %14 +│ s = %15 +│ @ REPL[2]:7 within `top-level scope' +│ %17 = k + i +│ k = %17 +│ #s1 = Base.iterate(%3, %10) +│ %20 = #s1 === nothing +│ %21 = Base.not_int(%20) +└── goto #4 if not %21 +3 ─ goto #2 +4 ┄ return +)))) +``` + +and then extract the edges: + +```julia +julia> edges = CodeEdges(lwr.args[1]) +CodeEdges: + s: assigned on [1, 16], depends on [15], and used by [12, 15] + k: assigned on [2, 18], depends on [17], and used by [11, 17] + statement 1 depends on [15, 16] and is used by [12, 15, 16] + statement 2 depends on [17, 18] and is used by [11, 17, 18] + statement 3 depends on ∅ and is used by [4, 19] + statement 4 depends on [3, 10, 19] and is used by [5, 8, 19, 20] + statement 5 depends on [4, 19] and is used by [6] + statement 6 depends on [5] and is used by [7] + statement 7 depends on [6] and is used by ∅ + statement 8 depends on [4, 19] and is used by [9, 10] + statement 9 depends on [8] and is used by [17] + statement 10 depends on [8] and is used by [4, 19] + statement 11 depends on [2, 18] and is used by ∅ + statement 12 depends on [1, 16] and is used by ∅ + statement 13 depends on ∅ and is used by [14] + statement 14 depends on [13] and is used by [15] + statement 15 depends on [1, 14, 16] and is used by [1, 16] + statement 16 depends on [1, 15] and is used by [1, 12, 15] + statement 17 depends on [2, 9, 18] and is used by [2, 18] + statement 18 depends on [2, 17] and is used by [2, 11, 17] + statement 19 depends on [3, 4, 10] and is used by [4, 5, 8, 20] + statement 20 depends on [4, 19] and is used by [21] + statement 21 depends on [20] and is used by [22] + statement 22 depends on [21] and is used by ∅ + statement 23 depends on ∅ and is used by ∅ + statement 24 depends on ∅ and is used by ∅ +``` + +This shows the dependencies of each line as well as the "named +variables" `s` and `k`. It's worth looking specifically to see how the +slot-variable `#s1` gets handled, as you'll notice there is no mention +of this in the "variables" section at the top. You can see that +`#s1` first gets assigned on line 4 (the `iterate` statement), +which you'll notice depends on 3 (via the SSAValue printed as `%3`). +But that line 4 also is shown as depending on 10 and 19. +You can see that line 19 is the 2-argument call to `iterate`, +and that this line depends on SSAValue `%10` (the state variable). +Consequently all the line-dependencies of this slot variable have +been aggregated into a single list by determining the "global" +influences on that slot variable. + +An even more useful output can be obtained from the following: +``` +julia> LoweredCodeUtils.print_with_code(stdout, lwr.args[1], edges) +Names: +s: assigned on [1, 16], depends on [15], and used by [12, 15] +k: assigned on [2, 18], depends on [17], and used by [11, 17] +Code: +1 ─ s = 0 +│ # preds: [15, 16], succs: [12, 15, 16] +│ k = 5 +│ # preds: [17, 18], succs: [11, 17, 18] +│ %3 = 1:3 +│ # preds: ∅, succs: [4, 19] +│ _1 = Base.iterate(%3) +│ # preds: [3, 10, 19], succs: [5, 8, 19, 20] +│ %5 = _1 === nothing +│ # preds: [4, 19], succs: [6] +│ %6 = Base.not_int(%5) +│ # preds: [5], succs: [7] +└── goto #4 if not %6 + # preds: [6], succs: ∅ +2 ┄ %8 = _1 +│ # preds: [4, 19], succs: [9, 10] +│ _2 = Core.getfield(%8, 1) +│ # preds: [8], succs: [17] +│ %10 = Core.getfield(%8, 2) +│ # preds: [8], succs: [4, 19] +│ global k +│ # preds: [2, 18], succs: ∅ +│ global s +│ # preds: [1, 16], succs: ∅ +│ %13 = 1:5 +│ # preds: ∅, succs: [14] +│ %14 = rand(%13) +│ # preds: [13], succs: [15] +│ %15 = s + %14 +│ # preds: [1, 14, 16], succs: [1, 16] +│ s = %15 +│ # preds: [1, 15], succs: [1, 12, 15] +│ %17 = k + _2 +│ # preds: [2, 9, 18], succs: [2, 18] +│ k = %17 +│ # preds: [2, 17], succs: [2, 11, 17] +│ _1 = Base.iterate(%3, %10) +│ # preds: [3, 4, 10], succs: [4, 5, 8, 20] +│ %20 = _1 === nothing +│ # preds: [4, 19], succs: [21] +│ %21 = Base.not_int(%20) +│ # preds: [20], succs: [22] +└── goto #4 if not %21 + # preds: [21], succs: ∅ +3 ─ goto #2 + # preds: ∅, succs: ∅ +4 ┄ return + # preds: ∅, succs: ∅ +``` + +Here the edges are printed right after each line. + +!!! note + "Nice" output from `print_with_code` requires at least version 1.6.0-DEV.95 of Julia. + +Suppose we want to evaluate just the lines needed to compute `s`. +We can find out which lines these are with + +```julia +julia> isrequired = lines_required(:s, lwr.args[1], edges) +24-element BitArray{1}: + 1 + 0 + 1 + 1 + 1 + 1 + 1 + 1 + 0 + 1 + 0 + 0 + 1 + 1 + 1 + 1 + 0 + 0 + 1 + 1 + 1 + 1 + 1 + 0 +``` + +and display them with + +``` +julia> LoweredCodeUtils.print_with_code(stdout, lwr.args[1], isrequired) + 1 t 1 ─ s = 0 + 2 f │ k = 5 + 3 t │ %3 = 1:3 + 4 t │ _1 = Base.iterate(%3) + 5 t │ %5 = _1 === nothing + 6 t │ %6 = Base.not_int(%5) + 7 t └── goto #4 if not %6 + 8 t 2 ┄ %8 = _1 + 9 f │ _2 = Core.getfield(%8, 1) +10 t │ %10 = Core.getfield(%8, 2) +11 f │ global k +12 f │ global s +13 t │ %13 = 1:5 +14 t │ %14 = rand(%13) +15 t │ %15 = s + %14 +16 t │ s = %15 +17 f │ %17 = k + _2 +18 f │ k = %17 +19 t │ _1 = Base.iterate(%3, %10) +20 t │ %20 = _1 === nothing +21 t │ %21 = Base.not_int(%20) +22 t └── goto #4 if not %21 +23 t 3 ─ goto #2 +24 f 4 ┄ return +``` + +We can test this with the following: + +```julia +julia> using JuliaInterpreter + +julia> frame = Frame(Main, lwr.args[1]) +Frame for Main + 1 2 1 ─ s = 0 + 2 3 │ k = 5 + 3 4 │ %3 = 1:3 +⋮ + +julia> k +11 + +julia> k = 0 +0 + +julia> selective_eval_fromstart!(frame, isrequired, true) + +julia> k +0 + +julia> s +12 # random + +julia> selective_eval_fromstart!(frame, isrequired, true) + +julia> k +0 + +julia> s +9 # random +``` + +You can see that `k` was not reset to its value of 11 when we ran this code selectively, +but that `s` was updated (to a random value) each time. diff --git a/packages/LoweredCodeUtils/docs/src/index.md b/packages/LoweredCodeUtils/docs/src/index.md new file mode 100644 index 0000000..2f9f988 --- /dev/null +++ b/packages/LoweredCodeUtils/docs/src/index.md @@ -0,0 +1,31 @@ +# LoweredCodeUtils.jl + +This package performs operations on Julia's [lowered AST](https://docs.julialang.org/en/latest/devdocs/ast/). +An introduction to this representation can be found at [JuliaInterpreter](https://juliadebug.github.io/JuliaInterpreter.jl/stable/). + +Lowered AST (like other ASTs, type-inferred AST and SSA IR form) is generally more amenable to analysis than "surface" Julia expressions. +However, sophisticated analyses can nevertheless require a fair bit of infrastructure. +The purpose of this package is to standardize a few operations that are important in some applications. + +Currently there are two major domains of this package: the "signatures" domain and the "edges" domain. + +## Signatures + +A major role of this package is to support extraction of method signatures, in particular to provide strong support for relating keyword-method "bodies" to their parent methods. +The central challenge this addresses is the lowering of keyword-argument functions and the fact that the "gensymmed" names are different each time you lower the code, and therefore you don't recover the actual (running) keyword-body method. +The technical details are described in [this Julia issue](https://github.com/JuliaLang/julia/issues/30908) and on the next page. +This package provides a workaround to rename gensymmed variables in newly-lowered code to match the name of the running keyword-body method, and provides a convenience function, `bodymethod`, to +obtain that otherwise difficult-to-discover method. + + + +## Edges + +Sometimes you want to run only a selected subset of code. For instance, Revise tracks methods +by their signatures, and therefore needs to compute signatures from the lowered representation of code. +Doing this robustly (including for `@eval`ed methods, etc.) requires running module top-level +code through the interpreter. +For reasons of performance and safety, it is important to minimize the amount of code that gets executed when extracting the signature. + +This package provides a general framework for computing dependencies in code, through the `CodeEdges` constructor. It allows you to determine the lines on which any given statement depends, the lines which "consume" the result of the current line, and any "named" dependencies (`Symbol` and `GlobalRef` dependencies). +In particular, this resolves the line-dependencies of all `SlotNumber` variables so that their own dependencies will be handled via the code-line dependencies. diff --git a/packages/LoweredCodeUtils/docs/src/signatures.md b/packages/LoweredCodeUtils/docs/src/signatures.md new file mode 100644 index 0000000..d385f6d --- /dev/null +++ b/packages/LoweredCodeUtils/docs/src/signatures.md @@ -0,0 +1,225 @@ +# Signatures and renaming + +We can demonstrate some of this package's functionality with the following simple example: + +```julia +julia> ex = :(f(x; color::Symbol=:green) = 2x) +:(f(x; color::Symbol = :green) = begin + #= REPL[1]:1 =# + 2x + end) + +julia> eval(ex) +f (generic function with 1 method) + +julia> f(3) +6 +``` + +Things get more interesting (and complicated) when we examine the lowered code: + +```julia +julia> lwr = Meta.lower(Main, ex) +:($(Expr(:thunk, CodeInfo( + @ none within `top-level scope' +1 ─ $(Expr(:thunk, CodeInfo( + @ none within `top-level scope' +1 ─ return $(Expr(:method, :f)) +))) +│ $(Expr(:thunk, CodeInfo( + @ none within `top-level scope' +1 ─ return $(Expr(:method, Symbol("#f#2"))) +))) +│ $(Expr(:method, :f)) +│ $(Expr(:method, Symbol("#f#2"))) +│ %5 = Core.typeof(var"#f#2") +│ %6 = Core.Typeof(f) +│ %7 = Core.svec(%5, Symbol, %6, Core.Any) +│ %8 = Core.svec() +│ %9 = Core.svec(%7, %8, $(QuoteNode(:(#= REPL[1]:1 =#)))) +│ $(Expr(:method, Symbol("#f#2"), :(%9), CodeInfo(quote + $(Expr(:meta, :nkw, 1)) + 2 * x + return %2 +end))) +│ $(Expr(:method, :f)) +│ %12 = Core.Typeof(f) +│ %13 = Core.svec(%12, Core.Any) +│ %14 = Core.svec() +│ %15 = Core.svec(%13, %14, $(QuoteNode(:(#= REPL[1]:1 =#)))) +│ $(Expr(:method, :f, :(%15), CodeInfo(quote + var"#f#2"(:green, #self#, x) + return %1 +end))) +│ $(Expr(:method, :f)) +│ %18 = Core.Typeof(f) +│ %19 = Core.kwftype(%18) +│ %20 = Core.Typeof(f) +│ %21 = Core.svec(%19, Core.Any, %20, Core.Any) +│ %22 = Core.svec() +│ %23 = Core.svec(%21, %22, $(QuoteNode(:(#= REPL[1]:1 =#)))) +│ $(Expr(:method, :f, :(%23), CodeInfo(quote + Base.haskey(@_2, :color) + unless %1 goto %11 + Base.getindex(@_2, :color) + %3 isa Symbol + unless %4 goto %7 + goto %9 + %new(Core.TypeError, Symbol("keyword argument"), :color, Symbol, %3) + Core.throw(%7) + @_6 = %3 + goto %12 + @_6 = :green + color = @_6 + Core.tuple(:color) + Core.apply_type(Core.NamedTuple, %13) + Base.structdiff(@_2, %14) + Base.pairs(%15) + Base.isempty(%16) + unless %17 goto %20 + goto %21 + Base.kwerr(@_2, @_3, x) + var"#f#2"(color, @_3, x) + return %21 +end))) +│ %25 = f +│ %26 = Core.ifelse(false, false, %25) +└── return %26 +)))) +``` + +This reveals the *three* methods actually got defined: +- one method of `f` with a single positional argument (this is the second 3-argument `:method` expression) +- a keyword-handling method that checks the names of supplied keyword arguments and fills in defaults (this is the third 3-argument `:method` expression). This method can be obtained from `Core.kwfunc(f)`, which returns a function named `f##kw`. +- a "keyword-body" method that actually does the work specifies by our function definition. This method gets called by the other two. (This is the first 3-argument `:method` expression.) + +From examining the lowered code we might guess that this function is called `#f#2`. +What happens if we try to get it? + +```julia +julia> fbody = var"#f#2" +ERROR: UndefVarError: #f#2 not defined +Stacktrace: + [1] top-level scope at REPL[6]:1 +``` + +Curiously, however, there is a closely-related function, and looking at its body code we see it is the one we wanted: + +```julia +julia> fbody = var"#f#1" +#f#1 (generic function with 1 method) + +julia> mbody = first(methods(fbody)) +#f#1(color::Symbol, ::typeof(f), x) in Main at REPL[1]:1 + +julia> Base.uncompressed_ast(mbody) +CodeInfo( + @ REPL[1]:1 within `#f#1' +1 ─ nothing +│ %2 = 2 * x +└── return %2 +) +``` + +It's named `#f#1`, rather than `#f#2`, because it was actually defined by that `eval(ex)` command at the top of this page. That `eval` caused it to be lowered once, and calling `Meta.lower` causes it to be lowered a second time, with different generated names. + +We can obtain the running version more directly (without having to guess) via the following: + +```julia +julia> m = first(methods(f)) +f(x; color) in Main at REPL[1]:1 + +julia> using LoweredCodeUtils + +julia> bodymethod(m) +#f#1(color::Symbol, ::typeof(f), x) in Main at REPL[1]:1 +``` + +We can also rename these methods, if we first turn it into a `frame`: + +```julia +julia> using JuliaInterpreter + +julia> frame = Frame(Main, lwr.args[1]) +Frame for Main + 1 0 1 ─ $(Expr(:thunk, CodeInfo( + 2 0 1 ─ return $(Expr(:method, :f)) + 3 0 ))) +⋮ + +julia> rename_framemethods!(frame) +Dict{Symbol,LoweredCodeUtils.MethodInfo} with 3 entries: + :f => MethodInfo(11, 24, [1]) + Symbol("#f#2") => MethodInfo(4, 10, [2]) + Symbol("#f#1") => MethodInfo(4, 10, [2]) + +julia> frame.framecode.src +CodeInfo( + @ none within `top-level scope' +1 ─ $(Expr(:thunk, CodeInfo( + @ none within `top-level scope' +1 ─ return $(Expr(:method, :f)) +))) +│ $(Expr(:thunk, CodeInfo( + @ none within `top-level scope' +1 ─ return $(Expr(:method, Symbol("#f#1"))) +))) +│ $(Expr(:method, :f)) +│ $(Expr(:method, Symbol("#f#1"))) +│ ($(QuoteNode(typeof)))(var"#f#1") +│ ($(QuoteNode(Core.Typeof)))(f) +│ ($(QuoteNode(Core.svec)))(%J5, Symbol, %J6, $(QuoteNode(Any))) +│ ($(QuoteNode(Core.svec)))() +│ ($(QuoteNode(Core.svec)))(%J7, %J8, $(QuoteNode(:(#= REPL[1]:1 =#)))) +│ $(Expr(:method, Symbol("#f#1"), %J9, CodeInfo(quote + $(Expr(:meta, :nkw, 1)) + 2 * x + return %2 +end))) +│ $(Expr(:method, :f)) +│ ($(QuoteNode(Core.Typeof)))(f) +│ ($(QuoteNode(Core.svec)))(%J12, $(QuoteNode(Any))) +│ ($(QuoteNode(Core.svec)))() +│ ($(QuoteNode(Core.svec)))(%J13, %J14, $(QuoteNode(:(#= REPL[1]:1 =#)))) +│ $(Expr(:method, :f, %J15, CodeInfo(quote + var"#f#1"(:green, #self#, x) + return %1 +end))) +│ $(Expr(:method, :f)) +│ ($(QuoteNode(Core.Typeof)))(f) +│ ($(QuoteNode(Core.kwftype)))(%J18) +│ ($(QuoteNode(Core.Typeof)))(f) +│ ($(QuoteNode(Core.svec)))(%J19, $(QuoteNode(Any)), %J20, $(QuoteNode(Any))) +│ ($(QuoteNode(Core.svec)))() +│ ($(QuoteNode(Core.svec)))(%J21, %J22, $(QuoteNode(:(#= REPL[1]:1 =#)))) +│ $(Expr(:method, :f, %J23, CodeInfo(quote + Base.haskey(@_2, :color) + unless %1 goto %11 + Base.getindex(@_2, :color) + %3 isa Symbol + unless %4 goto %7 + goto %9 + %new(Core.TypeError, Symbol("keyword argument"), :color, Symbol, %3) + Core.throw(%7) + @_6 = %3 + goto %12 + @_6 = :green + color = @_6 + Core.tuple(:color) + Core.apply_type(Core.NamedTuple, %13) + Base.structdiff(@_2, %14) + Base.pairs(%15) + Base.isempty(%16) + unless %17 goto %20 + goto %21 + Base.kwerr(@_2, @_3, x) + var"#f#1"(color, @_3, x) + return %21 +end))) +│ f +│ ($(QuoteNode(ifelse)))(false, false, %J25) +└── return %J26 +) +``` + +While there are a few differences in representation stemming from converting it to a frame, you can see that the `#f#2`s have been changed to `#f#1`s to match the currently-running names. diff --git a/packages/LoweredCodeUtils/src/LoweredCodeUtils.jl b/packages/LoweredCodeUtils/src/LoweredCodeUtils.jl new file mode 100644 index 0000000..4ba2cfb --- /dev/null +++ b/packages/LoweredCodeUtils/src/LoweredCodeUtils.jl @@ -0,0 +1,20 @@ +module LoweredCodeUtils + +# We use a code structure where all `using` and `import` +# statements in the package that load anything other than +# a Julia base or stdlib package are located in this file here. +# Nothing else should appear in this file here, apart from +# the `include("packagedef.jl")` statement, which loads what +# we would normally consider the bulk of the package code. +# This somewhat unusual structure is in place to support +# the VS Code extension integration. + +using JuliaInterpreter +using JuliaInterpreter: SSAValue, SlotNumber, Frame +using JuliaInterpreter: @lookup, moduleof, pc_expr, step_expr!, is_global_ref, is_quotenode_egal, whichtt, + next_until!, finish_and_return!, get_return, nstatements, codelocation, linetable, + is_return, lookup_return, is_GotoIfNot, is_ReturnNode + +include("packagedef.jl") + +end # module diff --git a/packages/LoweredCodeUtils/src/codeedges.jl b/packages/LoweredCodeUtils/src/codeedges.jl new file mode 100644 index 0000000..439590f --- /dev/null +++ b/packages/LoweredCodeUtils/src/codeedges.jl @@ -0,0 +1,913 @@ +const NamedVar = Union{Symbol,GlobalRef} + +## Phase 1: direct links + +# There are 3 types of entities to track: ssavalues (line/statement numbers), slots, and named objects. +# Each entity can have a number of "predecessors" (forward edges), which can be any combination of these +# three entity types. Likewise, each entity can have a number of "successors" (backward edges), also any +# combination of these entity types. +struct Links + ssas::Vector{Int} + slots::Vector{Int} + names::Vector{NamedVar} +end +Links() = Links(Int[], Int[], NamedVar[]) + +function Base.show(io::IO, l::Links) + print(io, "ssas: ", showempty(l.ssas), + ", slots: ", showempty(l.slots), + ", names: ") + print(IOContext(io, :typeinfo=>Vector{NamedVar}), showempty(l.names)) + print(io, ';') +end + +struct CodeLinks + ssapreds::Vector{Links} + ssasuccs::Vector{Links} + slotpreds::Vector{Links} + slotsuccs::Vector{Links} + slotassigns::Vector{Vector{Int}} + namepreds::Dict{NamedVar,Links} + namesuccs::Dict{NamedVar,Links} + nameassigns::Dict{NamedVar,Vector{Int}} +end +function CodeLinks(nlines::Int, nslots::Int) + makelinks(n) = [Links() for _ = 1:n] + + return CodeLinks(makelinks(nlines), + makelinks(nlines), + makelinks(nslots), + makelinks(nslots), + [Int[] for _ = 1:nslots], + Dict{NamedVar,Links}(), + Dict{NamedVar,Links}(), + Dict{NamedVar,Vector{Int}}()) +end +function CodeLinks(src::CodeInfo) + cl = CodeLinks(length(src.code), length(src.slotnames)) + direct_links!(cl, src) +end + +function Base.show(io::IO, cl::CodeLinks) + print(io, "CodeLinks:") + print_slots(io, cl) + print_names(io, cl) + nstmts = length(cl.ssapreds) + nd = ndigits(nstmts) + print(io, "\nCode:") + for i = 1:nstmts + print(io, '\n', lpad(i, nd), " preds: ") + show(io, cl.ssapreds[i]) + print(io, '\n', lpad(i, nd), " succs: ") + show(io, cl.ssasuccs[i]) + end +end + +function print_slots(io::IO, cl::CodeLinks) + nslots = length(cl.slotpreds) + nd = ndigits(nslots) + for i = 1:nslots + print(io, "\nslot ", lpad(i, nd), ':') + print(io, "\n preds: ") + show(io, cl.slotpreds[i]) + print(io, "\n succs: ") + show(io, cl.slotsuccs[i]) + print(io, "\n assign @: ") + show(io, cl.slotassigns[i]) + end +end + +function print_names(io::IO, cl::CodeLinks) + ukeys = namedkeys(cl) + for key in ukeys + print(io, '\n', key, ':') + if haskey(cl.namepreds, key) + print(io, "\n preds: ") + show(io, cl.namepreds[key]) + end + if haskey(cl.namesuccs, key) + print(io, "\n succs: ") + show(io, cl.namesuccs[key]) + end + if haskey(cl.nameassigns, key) + print(io, "\n assign @: ") + show(io, cl.nameassigns[key]) + end + end +end + +const preprinter_sentinel = isdefined(Base.IRShow, :statementidx_lineinfo_printer) ? 0 : typemin(Int32) + +if isdefined(Base.IRShow, :show_ir_stmt) + function print_with_code(preprint, postprint, io::IO, src::CodeInfo) + src = copy(src) + JuliaInterpreter.replace_coretypes!(src; rev=true) + if isdefined(JuliaInterpreter, :reverse_lookup_globalref!) + JuliaInterpreter.reverse_lookup_globalref!(src.code) + end + io = IOContext(io, :displaysize=>displaysize(io)) + used = BitSet() + cfg = Core.Compiler.compute_basic_blocks(src.code) + for stmt in src.code + Core.Compiler.scan_ssa_use!(push!, used, stmt) + end + line_info_preprinter = Base.IRShow.lineinfo_disabled + line_info_postprinter = Base.IRShow.default_expr_type_printer + preprint(io) + bb_idx_prev = bb_idx = 1 + for idx = 1:length(src.code) + preprint(io, idx) + bb_idx = Base.IRShow.show_ir_stmt(io, src, idx, line_info_preprinter, line_info_postprinter, used, cfg, bb_idx) + postprint(io, idx, bb_idx != bb_idx_prev) + bb_idx_prev = bb_idx + end + max_bb_idx_size = ndigits(length(cfg.blocks)) + line_info_preprinter(io, " "^(max_bb_idx_size + 2), preprinter_sentinel) + postprint(io) + return nothing + end +else + function print_with_code(preprint, postprint, io::IO, src::CodeInfo) + println(io, "No IR statement printer available on this version of Julia, just aligning statements.") + preprint(io) + for idx = 1:length(src.code) + preprint(io, idx) + print(io, src.code[idx]) + println(io) + postprint(io, idx, false) + end + postprint(io) + end +end + +""" + print_with_code(io, src::CodeInfo, cl::CodeLinks) + +Interweave display of code and links. + +!!! compat Julia 1.6 + This function produces dummy output if suitable support is missing in your version of Julia. +""" +function print_with_code(io::IO, src::CodeInfo, cl::CodeLinks) + function preprint(io::IO) + print(io, "Slots:") + print_slots(io, cl) + print(io, "\nNames:") + print_names(io, cl) + println(io) + end + preprint(::IO, ::Int) = nothing + postprint(::IO) = nothing + postprint(io::IO, idx::Int, bbchanged::Bool) = postprint_linelinks(io, idx, src, cl, bbchanged) + + print_with_code(preprint, postprint, io, src) +end + +function postprint_linelinks(io::IO, idx::Int, src::CodeInfo, cl::CodeLinks, bbchanged::Bool) + printstyled(io, bbchanged ? " " : "│", color=:light_black) + printstyled(io, " # ", color=:yellow) + stmt = src.code[idx] + if isexpr(stmt, :(=)) + lhs = stmt.args[1] + if @issslotnum(lhs) + # id = lhs.id + # preds, succs = cl.slotpreds[id], cl.slotsuccs[id] + printstyled(io, "see slot ", lhs.id, '\n', color=:yellow) + else + # preds, succs = cl.namepreds[lhs], cl.namesuccs[lhs] + printstyled(io, "see name ", lhs, '\n', color=:yellow) + end + else + preds, succs = cl.ssapreds[idx], cl.ssasuccs[idx] + printstyled(io, "preds: ", preds, " succs: ", succs, '\n', color=:yellow) + end + return nothing +end + + +function namedkeys(cl::CodeLinks) + ukeys = Set{NamedVar}() + for c in (cl.namepreds, cl.namesuccs, cl.nameassigns) + for k in keys(c) + push!(ukeys, k) + end + end + return ukeys +end + +function direct_links!(cl::CodeLinks, src::CodeInfo) + # Utility for when a stmt itself contains a CodeInfo + function add_inner!(cl::CodeLinks, icl::CodeLinks, idx) + for (name, _) in icl.nameassigns + assigns = get(cl.nameassigns, name, nothing) + if assigns === nothing + cl.nameassigns[name] = assigns = Int[] + end + push!(assigns, idx) + end + for (name, _) in icl.namesuccs + succs = get(cl.namesuccs, name, nothing) + if succs === nothing + cl.namesuccs[name] = succs = Links() + end + push!(succs.ssas, idx) + end + end + + P = Pair{Union{SSAValue,SlotNumber,NamedVar},Links} + + for (i, stmt) in enumerate(src.code) + if isexpr(stmt, :thunk) && isa(stmt.args[1], CodeInfo) + icl = CodeLinks(stmt.args[1]) + add_inner!(cl, icl, i) + continue + elseif isa(stmt, Expr) && stmt.head ∈ trackedheads + if stmt.head === :method && length(stmt.args) === 3 && isa(stmt.args[3], CodeInfo) + icl = CodeLinks(stmt.args[3]) + add_inner!(cl, icl, i) + end + name = stmt.args[1] + if isa(name, Symbol) + assign = get(cl.nameassigns, name, nothing) + if assign === nothing + cl.nameassigns[name] = assign = Int[] + end + push!(assign, i) + targetstore = get(cl.namepreds, name, nothing) + if targetstore === nothing + cl.namepreds[name] = targetstore = Links() + end + target = P(name, targetstore) + add_links!(target, stmt, cl) + end + rhs = stmt + target = P(SSAValue(i), cl.ssapreds[i]) + elseif isexpr(stmt, :(=)) + # An assignment + stmt = stmt::Expr + lhs, rhs = stmt.args[1], stmt.args[2] + if @issslotnum(lhs) + lhs = lhs::AnySlotNumber + id = lhs.id + target = P(SlotNumber(id), cl.slotpreds[id]) + push!(cl.slotassigns[id], i) + elseif isa(lhs, NamedVar) + targetstore = get(cl.namepreds, lhs, nothing) + if targetstore === nothing + cl.namepreds[lhs] = targetstore = Links() + end + target = P(lhs, targetstore) + assign = get(cl.nameassigns, lhs, nothing) + if assign === nothing + cl.nameassigns[lhs] = assign = Int[] + end + push!(assign, i) + else + error("lhs ", lhs, " not recognized") + end + else + rhs = stmt + target = P(SSAValue(i), cl.ssapreds[i]) + end + add_links!(target, rhs, cl) + end + return cl +end + +function add_links!(target::Pair{Union{SSAValue,SlotNumber,NamedVar},Links}, @nospecialize(stmt), cl::CodeLinks) + _targetid, targetstore = target + targetid = _targetid::Union{SSAValue,SlotNumber,NamedVar} + # Adds bidirectional edges + if @isssa(stmt) + stmt = stmt::AnySSAValue + push!(targetstore, SSAValue(stmt.id)) # forward edge + push!(cl.ssasuccs[stmt.id], targetid) # backward edge + elseif @issslotnum(stmt) + stmt = stmt::AnySlotNumber + push!(targetstore, SlotNumber(stmt.id)) + push!(cl.slotsuccs[stmt.id], targetid) + elseif isa(stmt, Symbol) || isa(stmt, GlobalRef) # NamedVar + push!(targetstore, stmt) + namestore = get(cl.namesuccs, stmt, nothing) + if namestore === nothing + cl.namesuccs[stmt] = namestore = Links() + end + push!(namestore, targetid) + elseif isa(stmt, Expr) && stmt.head !== :copyast + stmt = stmt::Expr + arng = 1:length(stmt.args) + if stmt.head === :call + f = stmt.args[1] + if !@isssa(f) && !@issslotnum(f) + # Avoid putting named callees on the namestore + arng = 2:length(stmt.args) + end + end + for i in arng + add_links!(target, stmt.args[i], cl) + end + elseif is_GotoIfNot(stmt) + add_links!(target, (stmt::Core.GotoIfNot).cond, cl) + elseif is_ReturnNode(stmt) + add_links!(target, (stmt::Core.ReturnNode).val, cl) + end + return nothing +end + +function Base.push!(l::Links, id) + if isa(id, SSAValue) + k = id.id + k ∉ l.ssas && push!(l.ssas, k) + elseif isa(id, SlotNumber) + k = id.id + k ∉ l.slots && push!(l.slots, k) + else + id = id::NamedVar + id ∉ l.names && push!(l.names, id) + end + return id +end + +## Phase 2: replacing slot-links with statement-links (and adding name-links to statement-links) + +# Now that we know the full set of dependencies, we can safely replace references to names +# by references to the relevant line numbers. + +""" +`Variable` holds information about named variables. +Unlike SSAValues, a single Variable can be assigned from multiple code locations. + +If `v` is a `Variable`, then +- `v.assigned` is a list of statement numbers on which it is assigned +- `v.preds` is the set of statement numbers upon which this assignment depends +- `v.succs` is the set of statement numbers which make use of this variable + +`preds` and `succs` are short for "predecessors" and "successors," respectively. +These are meant in the sense of execution order, not statement number; depending on control-flow, +a variable may have entries in `preds` that are larger than the smallest entry in `assigned`. +""" +struct Variable + assigned::Vector{Int} + preds::Vector{Int} + succs::Vector{Int} +end +Variable() = Variable(Int[], Int[], Int[]) + +function Base.show(io::IO, v::Variable) + print(io, "assigned on ", showempty(v.assigned)) + print(io, ", depends on ", showempty(v.preds)) + print(io, ", and used by ", showempty(v.succs)) +end + +# This will be documented below at the "user-level" constructor. +# preds[i] is the list of predecessors for the `i`th statement in the CodeInfo +# succs[i] is the list of successors for the `i`th statement in the CodeInfo +# byname[name] summarizes this CodeInfo's dependence on Variable `name`. +struct CodeEdges + preds::Vector{Vector{Int}} + succs::Vector{Vector{Int}} + byname::Dict{NamedVar,Variable} +end +CodeEdges(n::Integer) = CodeEdges([Int[] for i = 1:n], [Int[] for i = 1:n], Dict{Union{GlobalRef,Symbol},Variable}()) + +function Base.show(io::IO, edges::CodeEdges) + println(io, "CodeEdges:") + for (name, v) in edges.byname + print(io, " ", name, ": ") + show(io, v) + println(io) + end + n = length(edges.preds) + nd = ndigits(n) + for i = 1:n + println(io, " statement ", lpad(i, nd), " depends on ", showempty(edges.preds[i]), " and is used by ", showempty(edges.succs[i])) + end + return nothing +end + + +""" + edges = CodeEdges(src::CodeInfo) + +Analyze `src` and determine the chain of dependencies. + +- `edges.preds[i]` lists the preceding statements that statement `i` depends on. +- `edges.succs[i]` lists the succeeding statements that depend on statement `i`. +- `edges.byname[v]` returns information about the predecessors, successors, and assignment statements + for an object `v::$NamedVar`. +""" +function CodeEdges(src::CodeInfo) + src.inferred && error("supply lowered but not inferred code") + cl = CodeLinks(src) + CodeEdges(src, cl) +end + +function CodeEdges(src::CodeInfo, cl::CodeLinks) + # The main task here is to elide the slot-dependencies and convert + # everything to just ssas & names. + + # Replace/add named intermediates (slot & named-variable references) with statement numbers + nstmts, nslots = length(src.code), length(src.slotnames) + marked, slothandled = BitSet(), fill(false, nslots) # working storage during resolution + edges = CodeEdges(nstmts) + emptylink = Links() + emptylist = Int[] + for (i, stmt) in enumerate(src.code) + # Identify line predecents for slots and named variables + if isexpr(stmt, :(=)) + stmt = stmt::Expr + lhs = stmt.args[1] + # Mark predecessors and successors of this line by following ssas & named assignments + if @issslotnum(lhs) + lhs = lhs::AnySlotNumber + # This line assigns a slot. Mark all predecessors. + id = lhs.id + linkpreds, linksuccs, listassigns = cl.slotpreds[id], cl.slotsuccs[id], cl.slotassigns[id] + else + lhs = lhs::NamedVar + linkpreds = get(cl.namepreds, lhs, emptylink) + linksuccs = get(cl.namesuccs, lhs, emptylink) + listassigns = get(cl.nameassigns, lhs, emptylist) + end + else + linkpreds, linksuccs, listassigns = cl.ssapreds[i], cl.ssasuccs[i], emptylist + end + # Assign the predecessors + # For "named" predecessors, we depend only on their assignments + empty!(marked) + fill!(slothandled, false) + follow_links!(marked, linkpreds, cl.slotpreds, cl.slotassigns, slothandled) + pushall!(marked, listassigns) + for key in linkpreds.names + pushall!(marked, get(cl.nameassigns, key, emptylist)) + end + delete!(marked, i) + append!(edges.preds[i], marked) + # Similarly for successors + empty!(marked) + fill!(slothandled, false) + follow_links!(marked, linksuccs, cl.slotsuccs, cl.slotassigns, slothandled) + pushall!(marked, listassigns) + for key in linksuccs.names + pushall!(marked, get(cl.nameassigns, key, emptylist)) + end + delete!(marked, i) + append!(edges.succs[i], marked) + end + # Add named variables + ukeys = namedkeys(cl) + for key in ukeys + assigned = get(cl.nameassigns, key, Int[]) + empty!(marked) + linkpreds = get(cl.namepreds, key, emptylink) + pushall!(marked, linkpreds.ssas) + for j in linkpreds.slots + pushall!(marked, cl.slotassigns[j]) + end + for key in linkpreds.names + pushall!(marked, get(cl.nameassigns, key, emptylist)) + end + preds = append!(Int[], marked) + empty!(marked) + linksuccs = get(cl.namesuccs, key, emptylink) + pushall!(marked, linksuccs.ssas) + for j in linksuccs.slots + pushall!(marked, cl.slotassigns[j]) + pushall!(marked, cl.slotsuccs[j].ssas) + end + for key in linksuccs.names + pushall!(marked, get(cl.nameassigns, key, emptylist)) + end + succs = append!(Int[], marked) + edges.byname[key] = Variable(assigned, preds, succs) + end + + return edges +end + +# Follow slot links to their non-slot leaves +function follow_links!(marked, l::Links, slotlinks, slotassigns, slothandled) + pushall!(marked, l.ssas) + for id in l.slots + slothandled[id] && continue + slothandled[id] = true + pushall!(marked, slotassigns[id]) + follow_links!(marked, slotlinks[id], slotlinks, slotassigns, slothandled) + end + return marked +end + +""" + print_with_code(io, src::CodeInfo, edges::CodeEdges) + +Interweave display of code and edges. + +!!! compat Julia 1.6 + This function produces dummy output if suitable support is missing in your version of Julia. +""" +function print_with_code(io::IO, src::CodeInfo, edges::CodeEdges) + function preprint(io::IO) + printstyled(io, "Names:", color=:yellow) + for (name, var) in edges.byname + print(io, '\n', name, ": ") + show(io, var) + end + printstyled(io, "\nCode:\n", color=:yellow) + end + @static if isdefined(Base.IRShow, :show_ir_stmt) + preprint(::IO, ::Int) = nothing + else + nd = ndigits(length(src.code)) + preprint(io::IO, i::Int) = print(io, lpad(i, nd), " ") + end + postprint(::IO) = nothing + postprint(io::IO, idx::Int, bbchanged::Bool) = postprint_lineedges(io, idx, edges, bbchanged) + + print_with_code(preprint, postprint, io, src) +end + +function postprint_lineedges(io::IO, idx::Int, edges::CodeEdges, bbchanged::Bool) + printstyled(io, bbchanged ? " " : "│", color=:light_black) + printstyled(io, " # ", color=:yellow) + preds, succs = edges.preds[idx], edges.succs[idx] + printstyled(io, "preds: ", showempty(preds), ", succs: ", showempty(succs), '\n', color=:yellow) + return nothing +end + +function terminal_preds(i::Int, edges::CodeEdges) + function terminal_preds!(s, j, edges, covered) + j ∈ covered && return s + push!(covered, j) + preds = edges.preds[j] + if isempty(preds) + push!(s, j) + else + for p in preds + terminal_preds!(s, p, edges, covered) + end + end + return s + end + s, covered = BitSet(), BitSet() + push!(covered, i) + for p in edges.preds[i] + terminal_preds!(s, p, edges, covered) + end + return s +end + +""" + isrequired = lines_required(obj::$NamedVar, src::CodeInfo, edges::CodeEdges) + isrequired = lines_required(idx::Int, src::CodeInfo, edges::CodeEdges) + +Determine which lines might need to be executed to evaluate `obj` or the statement indexed by `idx`. +If `isrequired[i]` is `false`, the `i`th statement is *not* required. +In some circumstances all statements marked `true` may be needed, in others control-flow +will end up skipping a subset of such statements, perhaps while repeating others multiple times. + +See also [`lines_required!`](@ref) and [`selective_eval!`](@ref). +""" +function lines_required(obj::NamedVar, src::CodeInfo, edges::CodeEdges; kwargs...) + isrequired = falses(length(edges.preds)) + objs = Set{NamedVar}([obj]) + return lines_required!(isrequired, objs, src, edges; kwargs...) +end + +function lines_required(idx::Int, src::CodeInfo, edges::CodeEdges; kwargs...) + isrequired = falses(length(edges.preds)) + isrequired[idx] = true + objs = Set{NamedVar}() + return lines_required!(isrequired, objs, src, edges; kwargs...) +end + +""" + lines_required!(isrequired::AbstractVector{Bool}, src::CodeInfo, edges::CodeEdges; + norequire = ()) + +Like `lines_required`, but where `isrequired[idx]` has already been set to `true` for all statements +that you know you need to evaluate. All other statements should be marked `false` at entry. +On return, the complete set of required statements will be marked `true`. + +`norequire` keyword argument specifies statements (represented as iterator of `Int`s) that +should _not_ be marked as a requirement. +For example, use `norequire = LoweredCodeUtils.exclude_named_typedefs(src, edges)` if you're +extracting method signatures and not evaluating new definitions. +""" +function lines_required!(isrequired::AbstractVector{Bool}, src::CodeInfo, edges::CodeEdges; kwargs...) + objs = Set{NamedVar}() + return lines_required!(isrequired, objs, src, edges; kwargs...) +end + +function exclude_named_typedefs(src::CodeInfo, edges::CodeEdges) + norequire = BitSet() + i = 1 + nstmts = length(src.code) + while i <= nstmts + stmt = rhs(src.code[i]) + if istypedef(stmt) && !isanonymous_typedef(stmt::Expr) + r = typedef_range(src, i) + pushall!(norequire, r) + i = last(r)+1 + else + i += 1 + end + end + return norequire +end + +function lines_required!(isrequired::AbstractVector{Bool}, objs, src::CodeInfo, edges::CodeEdges; norequire = ()) + # Mark any requested objects (their lines of assignment) + objs = add_requests!(isrequired, objs, edges, norequire) + + # Compute basic blocks, which we'll use to make sure we mark necessary control-flow + cfg = Core.Compiler.compute_basic_blocks(src.code) # needed for control-flow analysis + + # We'll mostly use generic graph traversal to discover all the lines we need, + # but structs are in a bit of a different category (especially on Julia 1.5+). + # It's easiest to discover these at the beginning. + typedefs = find_typedefs(src) + + changed = true + iter = 0 + while changed + changed = false + + # Handle ssa predecessors + changed |= add_ssa_preds!(isrequired, src, edges, norequire) + + # Handle named dependencies + changed |= add_named_dependencies!(isrequired, edges, objs, norequire) + + # Add control-flow + changed |= add_control_flow!(isrequired, cfg, norequire) + + # So far, everything is generic graph traversal. Now we add some domain-specific information + changed |= add_typedefs!(isrequired, src, edges, typedefs, norequire) + + iter += 1 # just for diagnostics + end + return isrequired +end + +function add_requests!(isrequired, objs, edges::CodeEdges, norequire) + objsnew = Set{NamedVar}() + for obj in objs + add_obj!(isrequired, objsnew, obj, edges, norequire) + end + return objsnew +end + +function add_ssa_preds!(isrequired, src::CodeInfo, edges::CodeEdges, norequire) + changed = false + for idx = 1:length(src.code) + if isrequired[idx] + changed |= add_preds!(isrequired, idx, edges, norequire) + end + end + return changed +end + +function add_named_dependencies!(isrequired, edges::CodeEdges, objs, norequire) + changed = false + for (obj, uses) in edges.byname + obj ∈ objs && continue + if any(view(isrequired, uses.succs)) + changed |= add_obj!(isrequired, objs, obj, edges, norequire) + end + end + return changed +end + +function add_preds!(isrequired, idx, edges::CodeEdges, norequire) + chngd = false + preds = edges.preds[idx] + for p in preds + isrequired[p] && continue + p ∈ norequire && continue + isrequired[p] = true + chngd = true + add_preds!(isrequired, p, edges, norequire) + end + return chngd +end +function add_succs!(isrequired, idx, edges::CodeEdges, succs, norequire) + chngd = false + for p in succs + isrequired[p] && continue + p ∈ norequire && continue + isrequired[p] = true + chngd = true + add_succs!(isrequired, p, edges, edges.succs[p], norequire) + end + return chngd +end +function add_obj!(isrequired, objs, obj, edges::CodeEdges, norequire) + chngd = false + for d in edges.byname[obj].assigned + d ∈ norequire && continue + isrequired[d] || add_preds!(isrequired, d, edges, norequire) + isrequired[d] = true + chngd = true + end + push!(objs, obj) + return chngd +end + +# Add control-flow. For any basic block with an evaluated statement inside it, +# check to see if the block has any successors, and if so mark that block's exit statement. +# Likewise, any preceding blocks should have *their* exit statement marked. +function add_control_flow!(isrequired, cfg, norequire) + changed = false + blocks = cfg.blocks + nblocks = length(blocks) + _changed = true + while _changed + _changed = false + for (ibb, bb) in enumerate(blocks) + r = rng(bb) + if any(view(isrequired, r)) + if ibb != nblocks + idxlast = r[end] + idxlast ∈ norequire && continue + _changed |= !isrequired[idxlast] + isrequired[idxlast] = true + end + for ibbp in bb.preds + ibbp > 0 || continue # see Core.Compiler.compute_basic_blocks, near comment re :enter + rpred = rng(blocks[ibbp]) + idxlast = rpred[end] + idxlast ∈ norequire && continue + _changed |= !isrequired[idxlast] + isrequired[idxlast] = true + end + for ibbs in bb.succs + ibbs == nblocks && continue + rpred = rng(blocks[ibbs]) + idxlast = rpred[end] + idxlast ∈ norequire && continue + _changed |= !isrequired[idxlast] + isrequired[idxlast] = true + end + end + end + changed |= _changed + end + return changed +end + +# Do a traveral of "numbered" predecessors and find statement ranges and names of type definitions +function find_typedefs(src::CodeInfo) + typedef_blocks, typedef_names = UnitRange{Int}[], Symbol[] + i = 1 + nstmts = length(src.code) + while i <= nstmts + stmt = rhs(src.code[i]) + if istypedef(stmt) && !isanonymous_typedef(stmt::Expr) + stmt = stmt::Expr + r = typedef_range(src, i) + push!(typedef_blocks, r) + name = stmt.head === :call ? stmt.args[3] : stmt.args[1] + if isa(name, QuoteNode) + name = name.value + end + isa(name, Symbol) || @show src i r stmt + push!(typedef_names, name::Symbol) + i = last(r)+1 + else + i += 1 + end + end + return typedef_blocks, typedef_names +end + +# New struct definitions, including their constructors, get spread out over many +# statements. If we're evaluating any of them, it's important to evaluate *all* of them. +function add_typedefs!(isrequired, src::CodeInfo, edges::CodeEdges, (typedef_blocks, typedef_names), norequire) + changed = false + stmts = src.code + idx = 1 + while idx < length(stmts) + stmt = stmts[idx] + isrequired[idx] || (idx += 1; continue) + for (typedefr, typedefn) in zip(typedef_blocks, typedef_names) + if idx ∈ typedefr + ireq = view(isrequired, typedefr) + if !all(ireq) + changed = true + ireq .= true + # Also mark any by-type constructor(s) associated with this typedef + var = get(edges.byname, typedefn, nothing) + if var !== nothing + for s in var.succs + s ∈ norequire && continue + stmt2 = stmts[s] + if isexpr(stmt2, :method) && (fname = (stmt2::Expr).args[1]; fname === false || fname === nothing) + isrequired[s] = true + end + end + end + end + idx = last(typedefr) + 1 + continue + end + end + # Anonymous functions may not yet include the method definition + if isanonymous_typedef(stmt) + i = idx + 1 + while i <= length(stmts) && !ismethod3(stmts[i]) + i += 1 + end + if i <= length(stmts) && (stmts[i]::Expr).args[1] == false + tpreds = terminal_preds(i, edges) + if minimum(tpreds) == idx && i ∉ norequire + changed |= !isrequired[i] + isrequired[i] = true + end + end + end + idx += 1 + end + return changed +end + +""" + selective_eval!([recurse], frame::Frame, isrequired::AbstractVector{Bool}, istoplevel=false) + +Execute the code in `frame` in the manner of `JuliaInterpreter.finish_and_return!`, +but skipping all statements that are marked `false` in `isrequired`. +See [`lines_required`](@ref). Upon entry, if needed the caller must ensure that `frame.pc` is +set to the correct statement, typically `findfirst(isrequired)`. +See [`selective_eval_fromstart!`](@ref) to have that performed automatically. + +The default value for `recurse` is `JuliaInterpreter.finish_and_return!`. +`isrequired` pertains only to `frame` itself, not any of its callees. + +This will return either a `BreakpointRef`, the value obtained from the last executed statement +(if stored to `frame.framedata.ssavlues`), or `nothing`. +Typically, assignment to a variable binding does not result in an ssa store by JuliaInterpreter. +""" +function selective_eval!(@nospecialize(recurse), frame::Frame, isrequired::AbstractVector{Bool}, istoplevel::Bool=false) + pc = pcexec = pclast = frame.pc + while isa(pc, Int) + frame.pc = pc + te = isrequired[pc] + pclast = pcexec::Int + if te + pcexec = pc = step_expr!(recurse, frame, istoplevel) + else + pc = next_or_nothing!(frame) + end + end + isa(pc, BreakpointRef) && return pc + pcexec = (pcexec === nothing ? pclast : pcexec)::Int + frame.pc = pcexec + node = pc_expr(frame) + is_return(node) && return lookup_return(frame, node) + isassigned(frame.framedata.ssavalues, pcexec) && return frame.framedata.ssavalues[pcexec] + return nothing +end +function selective_eval!(frame::Frame, isrequired::AbstractVector{Bool}, istoplevel::Bool=false) + selective_eval!(finish_and_return!, frame, isrequired, istoplevel) +end + +""" + selective_eval_fromstart!([recurse], frame, isrequired, istoplevel=false) + +Like [`selective_eval!`](@ref), except it sets `frame.pc` to the first `true` statement in `isrequired`. +""" +function selective_eval_fromstart!(@nospecialize(recurse), frame, isrequired, istoplevel::Bool=false) + pc = findfirst(isrequired) + pc === nothing && return nothing + frame.pc = pc + return selective_eval!(recurse, frame, isrequired, istoplevel) +end +function selective_eval_fromstart!(frame::Frame, isrequired::AbstractVector{Bool}, istoplevel::Bool=false) + selective_eval_fromstart!(finish_and_return!, frame, isrequired, istoplevel) +end + +""" + print_with_code(io, src::CodeInfo, isrequired::AbstractVector{Bool}) + +Mark each line of code with its requirement status. + +!!! compat Julia 1.6 + This function produces dummy output if suitable support is missing in your version of Julia. +""" +function print_with_code(io::IO, src::CodeInfo, isrequired::AbstractVector{Bool}) + nd = ndigits(length(isrequired)) + preprint(::IO) = nothing + preprint(io::IO, idx::Int) = (c = isrequired[idx]; printstyled(io, lpad(idx, nd), ' ', c ? "t " : "f "; color = c ? :cyan : :plain)) + postprint(::IO) = nothing + postprint(io::IO, idx::Int, bbchanged::Bool) = nothing + + print_with_code(preprint, postprint, io, src) +end + +function print_with_code(io::IO, frame::Frame, obj) + src = frame.framecode.src + if isdefined(JuliaInterpreter, :reverse_lookup_globalref!) + src = copy(src) + JuliaInterpreter.reverse_lookup_globalref!(src.code) + end + print_with_code(io, src, obj) +end diff --git a/packages/LoweredCodeUtils/src/packagedef.jl b/packages/LoweredCodeUtils/src/packagedef.jl new file mode 100644 index 0000000..867ab4d --- /dev/null +++ b/packages/LoweredCodeUtils/src/packagedef.jl @@ -0,0 +1,51 @@ +if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@optlevel")) + @eval Base.Experimental.@optlevel 1 +end + +using Core: SimpleVector, CodeInfo, NewvarNode, GotoNode +using Base.Meta: isexpr + +const SSAValues = Union{Core.Compiler.SSAValue, JuliaInterpreter.SSAValue} + +const trackedheads = (:method,) +const structdecls = (:_structtype, :_abstracttype, :_primitivetype) + +export signature, rename_framemethods!, methoddef!, methoddefs!, bodymethod +export CodeEdges, lines_required, lines_required!, selective_eval!, selective_eval_fromstart! + +include("utils.jl") +include("signatures.jl") +include("codeedges.jl") + +# precompilation + +if ccall(:jl_generating_output, Cint, ()) == 1 + ex = :(f(x; color::Symbol=:green) = 2x) + lwr = Meta.lower(@__MODULE__, ex) + frame = Frame(@__MODULE__, lwr.args[1]) + rename_framemethods!(frame) + ex = quote + s = 0 + k = 5 + for i = 1:3 + global s, k + s += rand(1:5) + k += i + end + end + lwr = Meta.lower(@__MODULE__, ex) + src = lwr.args[1] + edges = CodeEdges(src) + isrequired = lines_required(:s, src, edges) + lines_required(:s, src, edges; norequire=()) + lines_required(:s, src, edges; norequire=exclude_named_typedefs(src, edges)) + for isreq in (isrequired, convert(Vector{Bool}, isrequired)) + lines_required!(isreq, src, edges; norequire=()) + lines_required!(isreq, src, edges; norequire=exclude_named_typedefs(src, edges)) + end + frame = Frame(@__MODULE__, src) + # selective_eval_fromstart!(frame, isrequired, true) + precompile(selective_eval_fromstart!, (typeof(frame), typeof(isrequired), Bool)) # can't @eval during precompilation + print_with_code(Base.inferencebarrier(devnull)::IO, src, edges) + print_with_code(Base.inferencebarrier(devnull)::IO, src, isrequired) +end diff --git a/packages/LoweredCodeUtils/src/signatures.jl b/packages/LoweredCodeUtils/src/signatures.jl new file mode 100644 index 0000000..354dbf6 --- /dev/null +++ b/packages/LoweredCodeUtils/src/signatures.jl @@ -0,0 +1,625 @@ +""" + sig = signature(sigsv::SimpleVector) + +Compute a method signature from a suitable `SimpleVector`: `sigsv[1]` holds the signature +and `sigsv[2]` the `TypeVar`s. + +# Example: + +For `f(x::AbstractArray{T}) where T`, the corresponding `sigsv` is constructed as + + T = TypeVar(:T) + sig1 = Core.svec(typeof(f), AbstractArray{T}) + sig2 = Core.svec(T) + sigsv = Core.svec(sig1, sig2) + sig = signature(sigsv) +""" +function signature(sigsv::SimpleVector) + sigp::SimpleVector, sigtv::SimpleVector = sigsv + sig = Tuple{sigp...} + for i = length(sigtv):-1:1 + sig = UnionAll(sigtv[i], sig) + end + return sig::Union{DataType,UnionAll} +end + +""" + sigt, lastpc = signature(recurse, frame, pc) + sigt, lastpc = signature(frame, pc) + +Compute the signature-type `sigt` of a method whose definition in `frame` starts at `pc`. +Generally, `pc` should point to the `Expr(:method, methname)` statement, in which case +`lastpc` is the final statement number in `frame` that is part of the signature +(i.e, the line above the 3-argument `:method` expression). +Alternatively, `pc` can point to the 3-argument `:method` expression, +as long as all the relevant SSAValues have been assigned. +In this case, `lastpc == pc`. + +If no 3-argument `:method` expression is found, `sigt` will be `nothing`. +""" +function signature(@nospecialize(recurse), frame::Frame, @nospecialize(stmt), pc) + mod = moduleof(frame) + lastpc = frame.pc = pc + while !isexpr(stmt, :method, 3) # wait for the 3-arg version + if isanonymous_typedef(stmt) + lastpc = pc = step_through_methoddef(recurse, frame, stmt) # define an anonymous function + elseif isexpr(stmt, :call) && (q = (stmt::Expr).args[1]; isa(q, QuoteNode) && q.value === Core.Typeof) && + (sym = (stmt::Expr).args[2]; isa(sym, Symbol) && !isdefined(mod, sym)) + return nothing, pc + else + lastpc = pc + pc = step_expr!(recurse, frame, stmt, true) + pc === nothing && return nothing, lastpc + end + stmt = pc_expr(frame, pc) + end + isa(stmt, Expr) || return nothing, pc + sigsv = @lookup(frame, stmt.args[2])::SimpleVector + sigt = signature(sigsv) + return sigt, lastpc +end +signature(@nospecialize(recurse), frame::Frame, pc) = signature(recurse, frame, pc_expr(frame, pc), pc) +signature(frame::Frame, pc) = signature(finish_and_return!, frame, pc) + +function minid(@nospecialize(node), stmts, id) + if isa(node, SSAValue) + id = min(id, node.id) + stmt = stmts[node.id] + return minid(stmt, stmts, id) + elseif isa(node, Expr) + for a in node.args + id = minid(a, stmts, id) + end + end + return id +end + +function signature_top(frame, stmt::Expr, pc) + @assert ismethod3(stmt) + return minid(stmt.args[2], frame.framecode.src.code, pc) +end + +function step_through_methoddef(@nospecialize(recurse), frame, @nospecialize(stmt)) + while !isexpr(stmt, :method) + pc = step_expr!(recurse, frame, stmt, true) + stmt = pc_expr(frame, pc) + end + return step_expr!(recurse, frame, stmt, true) # also define the method +end + +""" + MethodInfo(start, stop, refs) + +Given a frame and its CodeInfo, `start` is the line of the first `Expr(:method, name)`, +whereas `stop` is the line of the last `Expr(:method, name, sig, src)` expression for `name`. +`refs` is a vector of line numbers of other references. +Some of these will be the location of the "declaration" of a method, +the `:thunk` expression containing a CodeInfo that just returns a 1-argument `:method` expression. +Others may be `:global` declarations. + +In some cases there may be more than one method with the same name in the `start:stop` range. +""" +mutable struct MethodInfo + start::Int + stop::Int + refs::Vector{Int} +end +MethodInfo(start) = MethodInfo(start, -1, Int[]) + +struct SelfCall + linetop::Int + linebody::Int + callee::Symbol + caller::Union{Symbol,Bool,Nothing} +end + +""" + methodinfos, selfcalls = identify_framemethod_calls(frame) + +Analyze the code in `frame` to locate method definitions and "self-calls," i.e., calls +to methods defined in `frame` that occur within `frame`. + +`methodinfos` is a Dict of `name=>info` pairs, where `info` is a [`MethodInfo`](@ref). + +`selfcalls` is a list of `SelfCall(linetop, linebody, callee, caller)` that holds the location of +calls the methods defined in `frame`. `linetop` is the line in `frame` (top meaning "top level"), +which will correspond to a 3-argument `:method` expression containing a `CodeInfo` body. +`linebody` is the line within the `CodeInfo` body from which the call is made. +`callee` is the Symbol of the called method. +""" +function identify_framemethod_calls(frame) + refs = Pair{Symbol,Int}[] + methodinfos = Dict{Symbol,MethodInfo}() + selfcalls = SelfCall[] + for (i, stmt) in enumerate(frame.framecode.src.code) + isa(stmt, Expr) || continue + if stmt.head === :global && length(stmt.args) == 1 + key = stmt.args[1]::Symbol + # We don't know for sure if this is a reference to a method, but let's + # tentatively cue it + push!(refs, key=>i) + elseif stmt.head === :thunk && stmt.args[1] isa CodeInfo + tsrc = stmt.args[1]::CodeInfo + if length(tsrc.code) == 1 + tstmt = tsrc.code[1] + if is_return(tstmt) + tex = JuliaInterpreter.get_return_node(tstmt) + if isa(tex, Expr) + if tex.head === :method && (methname = tex.args[1]; isa(methname, Symbol)) + push!(refs, methname=>i) + end + end + end + end + elseif ismethod1(stmt) + key = stmt.args[1] + key = normalize_defsig(key, frame) + key = key::Symbol + mi = get(methodinfos, key, nothing) + if mi === nothing + methodinfos[key] = MethodInfo(i) + else + mi.stop == -1 && (mi.start = i) # advance the statement # unless we've seen the method3 + end + elseif ismethod3(stmt) + key = stmt.args[1] + key = normalize_defsig(key, frame) + if key isa Symbol + mi = methodinfos[key] + mi.stop = i + elseif key isa Expr # this is a module-scoped call. We don't have to worry about these because they are named + continue + end + msrc = stmt.args[3] + if msrc isa CodeInfo + key = key::Union{Symbol,Bool,Nothing} + for (j, mstmt) in enumerate(msrc.code) + isa(mstmt, Expr) || continue + if mstmt.head === :call + mkey = mstmt.args[1] + if isa(mkey, Symbol) + # Could be a GlobalRef but then it's outside frame + haskey(methodinfos, mkey) && push!(selfcalls, SelfCall(i, j, mkey, key)) + elseif is_global_ref(mkey, Core, isdefined(Core, :_apply_iterate) ? :_apply_iterate : :_apply) + ssaref = mstmt.args[end-1] + if isa(ssaref, JuliaInterpreter.SSAValue) + id = ssaref.id + has_self_call(msrc, msrc.code[id]) || continue + end + mkey = mstmt.args[end-2] + if isa(mkey, Symbol) + haskey(methodinfos, mkey) && push!(selfcalls, SelfCall(i, j, mkey, key)) + end + end + elseif mstmt.head === :meta && mstmt.args[1] === :generated + newex = mstmt.args[2] + if isa(newex, Expr) + if newex.head === :new && length(newex.args) >= 2 && is_global_ref(newex.args[1], Core, :GeneratedFunctionStub) + mkey = newex.args[2]::Symbol + haskey(methodinfos, mkey) && push!(selfcalls, SelfCall(i, j, mkey, key)) + end + end + end + end + end + end + end + for r in refs + mi = get(methodinfos, r.first, nothing) + mi === nothing || push!(mi.refs, r.second) + end + return methodinfos, selfcalls +end + +# try to normalize `def` to `Symbol` representation +function normalize_defsig(@nospecialize(def), frame::Frame) + if def isa QuoteNode + def = nameof(def.value) + elseif def isa GlobalRef + def = def.name + end + return def +end + +function callchain(selfcalls) + calledby = Dict{Symbol,Union{Symbol,Bool,Nothing}}() + for sc in selfcalls + startswith(String(sc.callee), '#') || continue + caller = get(calledby, sc.callee, nothing) + if caller === nothing + calledby[sc.callee] = sc.caller + elseif caller == sc.caller + else + error("unexpected multiple callers, ", caller, " and ", sc.caller) + end + end + return calledby +end + +function set_to_running_name!(@nospecialize(recurse), replacements, frame, methodinfos, selfcall, calledby, callee, caller) + if isa(caller, Symbol) && startswith(String(caller), '#') + rep = get(replacements, caller, nothing) + if rep === nothing + parentcaller = get(calledby, caller, nothing) + if parentcaller !== nothing + set_to_running_name!(recurse, replacements, frame, methodinfos, selfcall, calledby, caller, parentcaller) + end + else + caller = rep + end + end + # Back up to the beginning of the signature + pc = selfcall.linetop + stmt = pc_expr(frame, pc) + while pc > 1 && !ismethod1(stmt) + pc -= 1 + stmt = pc_expr(frame, pc) + end + @assert ismethod1(stmt) + cname, _pc, _ = get_running_name(recurse, frame, pc+1, callee, get(replacements, caller, caller)) + replacements[callee] = cname + mi = methodinfos[cname] = methodinfos[callee] + src = frame.framecode.src + replacename!(src.code[mi.start:mi.stop], callee=>cname) # the method itself + for r in mi.refs # the references + replacename!((src.code[r])::Expr, callee=>cname) + end + return replacements +end + +""" + methranges = rename_framemethods!(frame) + methranges = rename_framemethods!(recurse, frame) + +Rename the gensymmed methods in `frame` to match those that are currently active. +The issues are described in https://github.com/JuliaLang/julia/issues/30908. +`frame` will be modified in-place as needed. + +Returns a vector of `name=>start:stop` pairs specifying the range of lines in `frame` +at which method definitions occur. In some cases there may be more than one method with +the same name in the `start:stop` range. +""" +function rename_framemethods!(@nospecialize(recurse), frame::Frame, methodinfos, selfcalls, calledby) + src = frame.framecode.src + replacements = Dict{Symbol,Symbol}() + for (callee, caller) in calledby + (!startswith(String(callee), '#') || haskey(replacements, callee)) && continue + idx = findfirst(sc->sc.callee === callee && sc.caller === caller, selfcalls) + idx === nothing && continue + try + set_to_running_name!(recurse, replacements, frame, methodinfos, selfcalls[idx], calledby, callee, caller) + catch err + @warn "skipping callee $callee (called by $caller) due to $err" + end + end + for sc in selfcalls + linetop, linebody, callee, caller = sc.linetop, sc.linebody, sc.callee, sc.caller + cname = get(replacements, callee, nothing) + if cname !== nothing && cname !== callee + replacename!((src.code[linetop].args[3])::CodeInfo, callee=>cname) + end + end + return methodinfos +end + +function rename_framemethods!(@nospecialize(recurse), frame::Frame) + pc0 = frame.pc + methodinfos, selfcalls = identify_framemethod_calls(frame) + calledby = callchain(selfcalls) + rename_framemethods!(recurse, frame, methodinfos, selfcalls, calledby) + frame.pc = pc0 + return methodinfos +end +rename_framemethods!(frame::Frame) = rename_framemethods!(finish_and_return!, frame) + +""" + pctop, isgen = find_name_caller_sig(recurse, frame, pc, name, parentname) + +Scans forward from `pc` in `frame` until a method is found that calls `name`. +`pctop` points to the beginning of that method's signature. +`isgen` is true if `name` corresponds to sa GeneratedFunctionStub. + +Alternatively, this returns `nothing` if `pc` does not appear to point to either +a keyword or generated method. +""" +function find_name_caller_sig(@nospecialize(recurse), frame, pc, name, parentname) + stmt = pc_expr(frame, pc) + while true + pc0 = pc + while !ismethod3(stmt) + pc = next_or_nothing(frame, pc) + pc === nothing && return nothing + stmt = pc_expr(frame, pc) + end + body = stmt.args[3] + if stmt.args[1] !== name && isa(body, CodeInfo) + # This might be the GeneratedFunctionStub for a @generated method + for (i, bodystmt) in enumerate(body.code) + if isexpr(bodystmt, :meta) && (bodystmt::Expr).args[1] === :generated + return signature_top(frame, stmt, pc), true + end + i >= 5 && break # we should find this early + end + if length(body.code) > 1 + bodystmt = body.code[end-1] # the line before the final return + iscallto(bodystmt, name) && return signature_top(frame, stmt, pc), false + end + end + pc = next_or_nothing(frame, pc) + pc === nothing && return nothing + stmt = pc_expr(frame, pc) + end +end + +""" + replacename!(stmts, oldname=>newname) + +Replace a Symbol `oldname` with `newname` in `stmts`. +""" +function replacename!(ex::Expr, pr) + replacename!(ex.args, pr) + return ex +end + +replacename!(src::CodeInfo, pr) = replacename!(src.code, pr) + +function replacename!(args::Vector{Any}, pr) + oldname, newname = pr + for i = 1:length(args) + a = args[i] + if isa(a, Expr) + replacename!(a, pr) + elseif isa(a, CodeInfo) + replacename!(a.code, pr) + elseif isa(a, QuoteNode) && a.value === oldname + args[i] = QuoteNode(newname) + elseif isa(a, Vector{Any}) + replacename!(a, pr) + elseif a === oldname + args[i] = newname + end + end + return args +end + +function get_running_name(@nospecialize(recurse), frame, pc, name, parentname) + nameinfo = find_name_caller_sig(recurse, frame, pc, name, parentname) + if nameinfo === nothing + pc = skip_until(stmt->isexpr(stmt, :method, 3), frame, pc) + pc = next_or_nothing(frame, pc) + return name, pc, nothing + end + pctop, isgen = nameinfo + # Sometimes signature_top---which follows SSAValue links backwards to find the first + # line needed to define the signature---misses out on a SlotNumber assignment. + # Fix https://github.com/timholy/Revise.jl/issues/422 + stmt = pc_expr(frame, pctop) + while pctop > 1 && isa(stmt, SlotNumber) && !isassigned(frame.framedata.locals, pctop) + pctop -= 1 + stmt = pc_expr(frame, pctop) + end # end fix + sigtparent, lastpcparent = signature(recurse, frame, pctop) + sigtparent === nothing && return name, pc, lastpcparent + methparent = whichtt(sigtparent) + methparent === nothing && return name, pc, lastpcparent # caller isn't defined, no correction is needed + if isgen + cname = nameof(methparent.generator.gen) + else + bodyparent = Base.uncompressed_ast(methparent) + bodystmt = bodyparent.code[end-1] + @assert isexpr(bodystmt, :call) + ref = getcallee(bodystmt) + isa(ref, GlobalRef) || @show ref typeof(ref) + @assert isa(ref, GlobalRef) + @assert ref.mod == moduleof(frame) + cname = ref.name + end + return cname, pc, lastpcparent +end + +""" + ret = methoddef!(recurse, signatures, frame; define=true) + ret = methoddef!(signatures, frame; define=true) + +Compute the signature of a method definition. `frame.pc` should point to a +`:method` expression. Upon exit, the new signature will be added to `signatures`. + +There are several possible return values: + + pc, pc3 = ret + +is the typical return. `pc` will point to the next statement to be executed, or be `nothing` +if there are no further statements in `frame`. `pc3` will point to the 3-argument `:method` +expression. + +Alternatively, + + pc = ret + +occurs for "empty method" expressions, e.g., `:(function foo end)`. `pc` will be `nothing`. + +By default the method will be defined (evaluated). You can prevent this by setting `define=false`. +This is recommended if you are simply extracting signatures from code that has already been evaluated. +""" +function methoddef!(@nospecialize(recurse), signatures, frame::Frame, @nospecialize(stmt), pc::Int; define::Bool=true) + framecode, pcin = frame.framecode, pc + if ismethod3(stmt) + pc3 = pc + arg1 = stmt.args[1] + sigt, pc = signature(recurse, frame, stmt, pc) + meth = whichtt(sigt) + if isa(meth, Method) && (meth.sig <: sigt && sigt <: meth.sig) + pc = define ? step_expr!(recurse, frame, stmt, true) : next_or_nothing!(frame) + elseif define + pc = step_expr!(recurse, frame, stmt, true) + meth = whichtt(sigt) + end + if isa(meth, Method) && (meth.sig <: sigt && sigt <: meth.sig) + push!(signatures, meth.sig) + else + if arg1 === false || arg1 === nothing + # If it's anonymous and not defined, define it + pc = step_expr!(recurse, frame, stmt, true) + meth = whichtt(sigt) + isa(meth, Method) && push!(signatures, meth.sig) + return pc, pc3 + else + # guard against busted lookup, e.g., https://github.com/JuliaLang/julia/issues/31112 + code = framecode.src + codeloc = codelocation(code, pc) + loc = linetable(code, codeloc) + ft = Base.unwrap_unionall((Base.unwrap_unionall(sigt)::DataType).parameters[1]) + if !startswith(String((ft.name::Core.TypeName).name), "##") + @warn "file $(loc.file), line $(loc.line): no method found for $sigt" + end + if pc == pc3 + pc = next_or_nothing!(frame) + end + end + end + frame.pc = pc + return pc, pc3 + end + ismethod1(stmt) || error("expected method opening, got ", stmt) + name = stmt.args[1] + name = normalize_defsig(name, frame) + if isa(name, Bool) + error("not valid for anonymous methods") + elseif name === missing + error("given invalid definition: $stmt") + end + name = name::Symbol + while true # methods containing inner methods may need multiple trips through this loop + sigt, pc = signature(recurse, frame, stmt, pc) + stmt = pc_expr(frame, pc) + while !isexpr(stmt, :method, 3) + pc = next_or_nothing(frame, pc) # this should not check define, we've probably already done this once + pc === nothing && return nothing # this was just `function foo end`, signal "no def" + stmt = pc_expr(frame, pc) + end + pc3 = pc + stmt = stmt::Expr + name3 = stmt.args[1] + sigt === nothing && (error("expected a signature"); return next_or_nothing(frame, pc)), pc3 + # Methods like f(x::Ref{<:Real}) that use gensymmed typevars will not have the *exact* + # signature of the active method. So let's get the active signature. + frame.pc = pc + pc = define ? step_expr!(recurse, frame, stmt, true) : next_or_nothing!(frame) + meth = whichtt(sigt) + isa(meth, Method) && push!(signatures, meth.sig) # inner methods are not visible + name === name3 && return pc, pc3 # if this was an inner method we should keep going + stmt = pc_expr(frame, pc) # there *should* be more statements in this frame + end +end +methoddef!(@nospecialize(recurse), signatures, frame::Frame, pc::Int; define=true) = + methoddef!(recurse, signatures, frame, pc_expr(frame, pc), pc; define=define) +function methoddef!(@nospecialize(recurse), signatures, frame::Frame; define=true) + pc = frame.pc + stmt = pc_expr(frame, pc) + if !ismethod(stmt) + pc = next_until!(ismethod, recurse, frame, true) + end + pc === nothing && error("pc at end of frame without finding a method") + methoddef!(recurse, signatures, frame, pc; define=define) +end +methoddef!(signatures, frame::Frame; define=true) = + methoddef!(finish_and_return!, signatures, frame; define=define) + +function methoddefs!(@nospecialize(recurse), signatures, frame::Frame, pc; define=true) + ret = methoddef!(recurse, signatures, frame, pc; define=define) + pc = ret === nothing ? ret : ret[1] + return _methoddefs!(recurse, signatures, frame, pc; define=define) +end +function methoddefs!(@nospecialize(recurse), signatures, frame::Frame; define=true) + ret = methoddef!(recurse, signatures, frame; define=define) + pc = ret === nothing ? ret : ret[1] + return _methoddefs!(recurse, signatures, frame, pc; define=define) +end +methoddefs!(signatures, frame::Frame; define=true) = + methoddefs!(finish_and_return!, signatures, frame; define=define) + +function _methoddefs!(@nospecialize(recurse), signatures, frame::Frame, pc; define=define) + while pc !== nothing + stmt = pc_expr(frame, pc) + if !ismethod(stmt) + pc = next_until!(ismethod, recurse, frame, true) + end + pc === nothing && break + ret = methoddef!(recurse, signatures, frame, pc; define=define) + pc = ret === nothing ? ret : ret[1] + end + return pc +end + +function is_self_call(@nospecialize(stmt), slotnames, argno=1) + if isa(stmt, Expr) + if stmt.head == :call + a = stmt.args[argno] + if isa(a, SlotNumber) || isa(a, Core.SlotNumber) + sn = slotnames[a.id] + if sn == Symbol("#self#") || sn == Symbol("") # allow empty to fix https://github.com/timholy/CodeTracking.jl/pull/48 + return true + end + end + end + end + return false +end + +function has_self_call(src, stmt::Expr) + # Check that it has a #self# call + hasself = false + for i = 2:length(stmt.args) + hasself |= is_self_call(stmt, src.slotnames, i) + end + return hasself +end + +""" + mbody = bodymethod(m::Method) + +Return the "body method" for a method `m`. `mbody` contains the code of the function body +when `m` was defined. +""" +function bodymethod(mkw::Method) + m = mkw + local src + while true + framecode = JuliaInterpreter.get_framecode(m) + fakeargs = Any[nothing for i = 1:(framecode.scope::Method).nargs] + frame = JuliaInterpreter.prepare_frame(framecode, fakeargs, isa(m.sig, UnionAll) ? sparam_ub(m) : Core.svec()) + src = framecode.src + (length(src.code) > 1 && is_self_call(src.code[end-1], src.slotnames)) || break + # Build the optional arg, so we can get its type + pc = frame.pc + while pc < length(src.code) - 1 + pc = step_expr!(frame) + end + val = pc > 1 ? frame.framedata.ssavalues[pc-1] : (src.code[1]::Expr).args[end] + sig = Tuple{(Base.unwrap_unionall(m.sig)::DataType).parameters..., typeof(val)} + m = whichtt(sig) + end + length(src.code) > 1 || return m + stmt = src.code[end-1] + if isexpr(stmt, :call) && (f = (stmt::Expr).args[1]; isa(f, QuoteNode)) + if f.value === (isdefined(Core, :_apply_iterate) ? Core._apply_iterate : Core._apply) + ssaref = stmt.args[end-1] + if isa(ssaref, JuliaInterpreter.SSAValue) + id = ssaref.id + has_self_call(src, src.code[id]) || return m + end + f = stmt.args[end-2] + if isa(f, JuliaInterpreter.SSAValue) + f = src.code[f.id] + end + else + has_self_call(src, stmt) || return m + end + f = f.value + mths = methods(f) + if length(mths) == 1 + return first(mths) + end + end + return m +end diff --git a/packages/LoweredCodeUtils/src/utils.jl b/packages/LoweredCodeUtils/src/utils.jl new file mode 100644 index 0000000..0abcfd3 --- /dev/null +++ b/packages/LoweredCodeUtils/src/utils.jl @@ -0,0 +1,259 @@ +const AnySSAValue = Union{Core.Compiler.SSAValue,JuliaInterpreter.SSAValue} +const AnySlotNumber = Union{Core.Compiler.SlotNumber,JuliaInterpreter.SlotNumber} + +# to circumvent https://github.com/JuliaLang/julia/issues/37342, we inline these `isa` +# condition checks at surface AST level +# https://github.com/JuliaLang/julia/pull/38905 will get rid of the need of these hacks +macro isssa(stmt) + :($(GlobalRef(Core, :isa))($(esc(stmt)), $(GlobalRef(Core.Compiler, :SSAValue))) || + $(GlobalRef(Core, :isa))($(esc(stmt)), $(GlobalRef(JuliaInterpreter, :SSAValue)))) +end +macro issslotnum(stmt) + :($(GlobalRef(Core, :isa))($(esc(stmt)), $(GlobalRef(Core.Compiler, :SlotNumber))) || + $(GlobalRef(Core, :isa))($(esc(stmt)), $(GlobalRef(JuliaInterpreter, :SlotNumber)))) +end + +""" + iscallto(stmt, name) + +Returns `true` is `stmt` is a call expression to `name`. +""" +function iscallto(@nospecialize(stmt), name) + if isa(stmt, Expr) + if stmt.head === :call + a = stmt.args[1] + a === name && return true + is_global_ref(a, Core, :_apply) && stmt.args[2] === name && return true + is_global_ref(a, Core, :_apply_iterate) && stmt.args[3] === name && return true + end + end + return false +end + +""" + getcallee(stmt) + +Returns the function (or Symbol) being called in a :call expression. +""" +function getcallee(@nospecialize(stmt)) + if isa(stmt, Expr) + if stmt.head === :call + a = stmt.args[1] + is_global_ref(a, Core, :_apply) && return stmt.args[2] + is_global_ref(a, Core, :_apply_iterate) && return stmt.args[3] + return a + end + end + error(stmt, " is not a call expression") +end + +function callee_matches(f, mod, sym) + is_global_ref(f, mod, sym) && return true + if isdefined(mod, sym) && isa(f, QuoteNode) + f.value === getfield(mod, sym) && return true # a consequence of JuliaInterpreter.optimize! + end + return false +end + +function rhs(stmt) + isexpr(stmt, :(=)) && return (stmt::Expr).args[2] + return stmt +end + +ismethod(frame::Frame) = ismethod(pc_expr(frame)) +ismethod3(frame::Frame) = ismethod3(pc_expr(frame)) + +ismethod(stmt) = isexpr(stmt, :method) +ismethod1(stmt) = isexpr(stmt, :method, 1) +ismethod3(stmt) = isexpr(stmt, :method, 3) +function ismethod_with_name(src, stmt, target::AbstractString; reentrant::Bool=false) + if reentrant + name = stmt + else + ismethod3(stmt) || return false + name = stmt.args[1] + if name === nothing + name = stmt.args[2] + end + end + isdone = false + while !isdone + if name isa AnySSAValue || name isa AnySlotNumber + name = src.code[name.id] + elseif isexpr(name, :call) && is_quotenode_egal(name.args[1], Core.svec) + name = name.args[2] + elseif isexpr(name, :call) && is_quotenode_egal(name.args[1], Core.apply_type) + for arg in name.args[2:end] + ismethod_with_name(src, arg, target; reentrant=true) && return true + end + isdone = true + elseif isexpr(name, :call) && is_quotenode_egal(name.args[1], UnionAll) + for arg in name.args[2:end] + ismethod_with_name(src, arg, target; reentrant=true) && return true + end + isdone = true + else + isdone = true + end + end + return match(Regex("(^|#)$target(\$|#)"), string(name)) !== nothing +end + + + +# anonymous function types are defined in a :thunk expr with a characteristic CodeInfo +function isanonymous_typedef(stmt) + if isa(stmt, Expr) + stmt.head === :thunk || return false + stmt = stmt.args[1] + end + if isa(stmt, CodeInfo) + src = stmt # just for naming consistency + length(src.code) >= 4 || return false + @static if VERSION ≥ v"1.9.0-DEV.391" + stmt = src.code[end-2] + isexpr(stmt, :(=)) || return false + name = stmt.args[1] + isa(name, Symbol) || return false + else + stmt = src.code[end-1] + isexpr(stmt, :call) || return false + is_global_ref(stmt.args[1], Core, :_typebody!) || return false + name = stmt.args[2]::Symbol + end + return startswith(String(name), "#") + end + return false +end + +function istypedef(stmt) + isa(stmt, Expr) || return false + stmt = rhs(stmt) + isa(stmt, Expr) || return false + @static if all(s->isdefined(Core,s), structdecls) + if stmt.head === :call + f = stmt.args[1] + if isa(f, GlobalRef) + f.mod === Core && f.name ∈ structdecls && return true + end + if isa(f, QuoteNode) + (f.value === Core._structtype || f.value === Core._abstracttype || + f.value === Core._primitivetype) && return true + end + end + end + isanonymous_typedef(stmt) && return true + return false +end + +# Given a typedef at `src.code[idx]`, return the range of statement indices that encompass the typedef. +# The range does not include any constructor methods. +function typedef_range(src::CodeInfo, idx) + stmt = src.code[idx] + istypedef(stmt) || error(stmt, " is not a typedef") + stmt = stmt::Expr + isanonymous_typedef(stmt) && return idx:idx + # Search backwards to the previous :global + istart = idx + while istart >= 1 + isexpr(src.code[istart], :global) && break + istart -= 1 + end + istart >= 1 || error("no initial :global found") + iend, n = idx, length(src.code) + have_typebody = have_equivtypedef = false + while iend <= n + stmt = src.code[iend] + if isa(stmt, Expr) + (stmt.head === :global || stmt.head === :return) && break + if stmt.head === :call + if (is_global_ref(stmt.args[1], Core, :_typebody!) || + isdefined(Core, :_typebody!) && is_quotenode_egal(stmt.args[1], Core._typebody!)) + have_typebody = true + elseif (is_global_ref(stmt.args[1], Core, :_equiv_typedef) || + isdefined(Core, :_equiv_typedef) && is_quotenode_egal(stmt.args[1], Core._equiv_typedef)) + have_equivtypedef = true + # Advance to the type-assignment + while iend <= n + stmt = src.code[iend] + isexpr(stmt, :(=)) && break + iend += 1 + end + end + if have_typebody && have_equivtypedef + iend += 1 # compensate for the `iend-1` in the return + break + end + end + end + is_return(stmt) && break + iend += 1 + end + iend <= n || (@show src; error("no final :global found")) + return istart:iend-1 +end + +""" + nextpc = next_or_nothing(frame, pc) + nextpc = next_or_nothing!(frame) + +Advance the program counter without executing the corresponding line. +If `frame` is finished, `nextpc` will be `nothing`. +""" +next_or_nothing(frame, pc) = pc < nstatements(frame.framecode) ? pc+1 : nothing +function next_or_nothing!(frame) + pc = frame.pc + if pc < nstatements(frame.framecode) + frame.pc = pc = pc + 1 + return pc + end + return nothing +end + +""" + nextpc = skip_until(predicate, frame, pc) + nextpc = skip_until!(predicate, frame) + +Advance the program counter until `predicate(stmt)` return `true`. +""" +function skip_until(predicate, frame, pc) + stmt = pc_expr(frame, pc) + while !predicate(stmt) + pc = next_or_nothing(frame, pc) + pc === nothing && return nothing + stmt = pc_expr(frame, pc) + end + return pc +end +function skip_until!(predicate, frame) + pc = frame.pc + stmt = pc_expr(frame, pc) + while !predicate(stmt) + pc = next_or_nothing!(frame) + pc === nothing && return nothing + stmt = pc_expr(frame, pc) + end + return pc +end + +function sparam_ub(meth::Method) + typs = [] + sig = meth.sig + while sig isa UnionAll + push!(typs, Symbol(sig.var.ub)) + sig = sig.body + end + return Core.svec(typs...) +end + +showempty(list) = isempty(list) ? '∅' : list + +# Smooth the transition between Core.Compiler and Base +rng(bb::Core.Compiler.BasicBlock) = (r = bb.stmts; return Core.Compiler.first(r):Core.Compiler.last(r)) + +function pushall!(dest, src) + for item in src + push!(dest, item) + end + return dest +end diff --git a/packages/LoweredCodeUtils/test/codeedges.jl b/packages/LoweredCodeUtils/test/codeedges.jl new file mode 100644 index 0000000..b747093 --- /dev/null +++ b/packages/LoweredCodeUtils/test/codeedges.jl @@ -0,0 +1,495 @@ +using LoweredCodeUtils +using LoweredCodeUtils.JuliaInterpreter +using LoweredCodeUtils: callee_matches, istypedef, exclude_named_typedefs +using JuliaInterpreter: is_global_ref, is_quotenode +using Test + +function hastrackedexpr(stmt; heads=LoweredCodeUtils.trackedheads) + haseval = false + if isa(stmt, Expr) + if stmt.head === :call + f = stmt.args[1] + haseval = f === :eval || (callee_matches(f, Base, :getproperty) && is_quotenode(stmt.args[2], :eval)) + callee_matches(f, Core, :_typebody!) && return true, haseval + callee_matches(f, Core, :_setsuper!) && return true, haseval + f === :include && return true, haseval + elseif stmt.head === :thunk + any(s->any(hastrackedexpr(s; heads=heads)), stmt.args[1].code) && return true, haseval + elseif stmt.head ∈ heads + return true, haseval + end + end + return false, haseval +end + +function minimal_evaluation(predicate, src::Core.CodeInfo, edges::CodeEdges; kwargs...) + isrequired = fill(false, length(src.code)) + for (i, stmt) in enumerate(src.code) + if !isrequired[i] + isrequired[i], haseval = predicate(stmt) + if haseval + isrequired[edges.succs[i]] .= true + end + end + end + # All tracked expressions are marked. Now add their dependencies. + lines_required!(isrequired, src, edges; kwargs...) + return isrequired +end + +function allmissing(mod::Module, names) + for name in names + isdefined(mod, name) && return false + end + return true +end + +module ModEval end + +module ModSelective end + +@testset "CodeEdges" begin + ex = quote + x = 1 + y = x+1 + a = sin(0.3) + z = x^2 + y + k = rand() + b = 2*a + 5 + end + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + # Check that the result of direct evaluation agrees with selective evaluation + Core.eval(ModEval, ex) + isrequired = lines_required(:x, src, edges) + @test sum(isrequired) == 1 + isdefined(Core, :get_binding_type) * 3 # get_binding_type + convert + typeassert + selective_eval_fromstart!(frame, isrequired) + @test ModSelective.x === ModEval.x + @test allmissing(ModSelective, (:y, :z, :a, :b, :k)) + @test !allmissing(ModSelective, (:x, :y)) # add :y here to test the `all` part of the test itself + # To evaluate z we need to do all the computations for y + isrequired = lines_required(:z, src, edges) + selective_eval_fromstart!(frame, isrequired) + @test ModSelective.y === ModEval.y + @test ModSelective.z === ModEval.z + @test allmissing(ModSelective, (:a, :b, :k)) # ... but not a and b + isrequired = lines_required(length(src.code)-1, src, edges) # this should be the assignment of b + selective_eval_fromstart!(frame, isrequired) + @test ModSelective.a === ModEval.a + @test ModSelective.b === ModEval.b + # Test that we get two separate evaluations of k + @test allmissing(ModSelective, (:k,)) + isrequired = lines_required(:k, src, edges) + selective_eval_fromstart!(frame, isrequired) + @test ModSelective.k != ModEval.k + + # Control-flow + ex = quote + flag2 = true + z2 = 15 + if flag2 + x2 = 5 + a2 = 1 + else + y2 = 7 + a2 = 2 + end + end + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + isrequired = lines_required(:a2, src, edges) + selective_eval_fromstart!(frame, isrequired) + Core.eval(ModEval, ex) + @test ModSelective.a2 === ModEval.a2 == 1 + @test allmissing(ModSelective, (:z2, :x2, :y2)) + # Now do it for the other branch, to make sure it's really sound. + # Also switch up the order of the assignments inside each branch. + ex = quote + flag3 = false + z3 = 15 + if flag3 + a3 = 1 + x3 = 5 + else + a3 = 2 + y3 = 7 + end + end + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + isrequired = lines_required(:a3, src, edges) + selective_eval_fromstart!(frame, isrequired) + Core.eval(ModEval, ex) + @test ModSelective.a3 === ModEval.a3 == 2 + @test allmissing(ModSelective, (:z3, :x3, :y3)) + + ex = quote + if Sys.iswindows() + const ONLY_ON_WINDOWS = true + end + c_os = if Sys.iswindows() + ONLY_ON_WINDOWS + else + false + end + end + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + isrequired = lines_required(:c_os, src, edges) + @test sum(isrequired) >= length(isrequired) - 2 + selective_eval_fromstart!(frame, isrequired) + Core.eval(ModEval, ex) + @test ModSelective.c_os === ModEval.c_os == Sys.iswindows() + + # Capturing dependencies of an `@eval` statement + interpT = Expr(:$, :T) # $T that won't get parsed during file-loading + ex = quote + foo() = 0 + for T in (Float32, Float64) + @eval feval1(::$interpT) = 1 + end + bar() = 1 + end + Core.eval(ModEval, ex) + @test ModEval.foo() == 0 + @test ModEval.bar() == 1 + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + # Mark just the load of Core.eval + haseval(stmt) = (isa(stmt, Expr) && JuliaInterpreter.hasarg(isequal(:eval), stmt.args)) || + (isa(stmt, Expr) && stmt.head === :call && is_quotenode(stmt.args[1], Core.eval)) + isrequired = map(haseval, src.code) + @test sum(isrequired) == 1 + isrequired[edges.succs[findfirst(isrequired)]] .= true # add lines that use Core.eval + lines_required!(isrequired, src, edges) + selective_eval_fromstart!(frame, isrequired) + @test ModSelective.feval1(1.0f0) == 1 + @test ModSelective.feval1(1.0) == 1 + @test_throws MethodError ModSelective.feval1(1) + @test_throws UndefVarError ModSelective.foo() + @test_throws UndefVarError ModSelective.bar() + # Run test from the docs + # Lowered code isn't very suitable to jldoctest (it can vary with each Julia version), + # so better to run it here + ex = quote + s11 = 0 + k11 = 5 + for i = 1:3 + global s11, k11 + s11 += rand(1:5) + k11 += i + end + end + frame = Frame(ModSelective, ex) + JuliaInterpreter.finish_and_return!(frame, true) + @test ModSelective.k11 == 11 + @test 3 <= ModSelective.s11 <= 15 + Core.eval(ModSelective, :(k11 = 0; s11 = -1)) + edges = CodeEdges(frame.framecode.src) + isrequired = lines_required(:s11, frame.framecode.src, edges) + selective_eval_fromstart!(frame, isrequired, true) + @test ModSelective.k11 == 0 + @test 3 <= ModSelective.s11 <= 15 + + # Control-flow in an abstract type definition + ex = :(abstract type StructParent{T, N} <: AbstractArray{T, N} end) + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + # Check that the StructParent name is discovered everywhere it is used + var = edges.byname[:StructParent] + isrequired = minimal_evaluation(hastrackedexpr, src, edges) + selective_eval_fromstart!(frame, isrequired, true) + @test supertype(ModSelective.StructParent) === AbstractArray + # Also check redefinition (it's OK when the definition doesn't change) + Core.eval(ModEval, ex) + frame = Frame(ModEval, ex) + src = frame.framecode.src + edges = CodeEdges(src) + isrequired = minimal_evaluation(hastrackedexpr, src, edges) + selective_eval_fromstart!(frame, isrequired, true) + @test supertype(ModEval.StructParent) === AbstractArray + + # Finding all dependencies in a struct definition + # Nonparametric + ex = :(struct NoParam end) + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + isrequired = minimal_evaluation(stmt->(LoweredCodeUtils.ismethod_with_name(src, stmt, "NoParam"),false), src, edges) # initially mark only the constructor + selective_eval_fromstart!(frame, isrequired, true) + @test isa(ModSelective.NoParam(), ModSelective.NoParam) + # Parametric + ex = quote + struct Struct{T} <: StructParent{T,1} + x::Vector{T} + end + end + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + isrequired = minimal_evaluation(stmt->(LoweredCodeUtils.ismethod_with_name(src, stmt, "Struct"),false), src, edges) # initially mark only the constructor + selective_eval_fromstart!(frame, isrequired, true) + @test isa(ModSelective.Struct([1,2,3]), ModSelective.Struct{Int}) + # Keyword constructor (this generates :copyast expressions) + ex = quote + struct KWStruct + x::Int + y::Float32 + z::String + function KWStruct(; x::Int=1, y::Float32=1.0f0, z::String="hello") + return new(x, y, z) + end + end + end + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + isrequired = minimal_evaluation(stmt->(LoweredCodeUtils.ismethod3(stmt),false), src, edges) # initially mark only the constructor + selective_eval_fromstart!(frame, isrequired, true) + kws = ModSelective.KWStruct(y=5.0f0) + @test kws.y === 5.0f0 + + # Anonymous functions + ex = :(max_values(T::Union{map(X -> Type{X}, Base.BitIntegerSmall_types)...}) = 1 << (8*sizeof(T))) + frame = Frame(ModSelective, ex) + src = frame.framecode.src + edges = CodeEdges(src) + isrequired = fill(false, length(src.code)) + @assert Meta.isexpr(src.code[end-1], :method, 3) + isrequired[end-1] = true + lines_required!(isrequired, src, edges) + selective_eval_fromstart!(frame, isrequired, true) + @test ModSelective.max_values(Int16) === 65536 + + # Avoid redefining types + ex = quote + struct MyNewType + x::Int + MyNewType(y::Int) = new(y) + end + end + Core.eval(ModEval, ex) + frame = Frame(ModEval, ex) + src = frame.framecode.src + edges = CodeEdges(src) + isrequired = minimal_evaluation(stmt->(LoweredCodeUtils.ismethod3(stmt),false), src, edges; norequire=exclude_named_typedefs(src, edges)) # initially mark only the constructor + bbs = Core.Compiler.compute_basic_blocks(src.code) + for (iblock, block) in enumerate(bbs.blocks) + r = LoweredCodeUtils.rng(block) + if iblock == length(bbs.blocks) + @test any(idx->isrequired[idx], r) + else + @test !any(idx->isrequired[idx], r) + end + end + + # https://github.com/timholy/Revise.jl/issues/538 + thk = Meta.lower(ModEval, quote + try + global function v1(x::Float32) + println("F32") + end + catch e + println("caught error") + end + end) + src = thk.args[1] + edges = CodeEdges(src) + lr = lines_required(:v1, src, edges) + idx = findfirst(stmt->Meta.isexpr(stmt, :leave), src.code) + @test lr[idx] + + # https://github.com/timholy/Revise.jl/issues/599 + thk = Meta.lower(Main, quote + mutable struct A + x::Int + A(x) = new(f(x)) + f(x) = x^2 + end + end) + src = thk.args[1] + edges = CodeEdges(src) + idx = findfirst(stmt->Meta.isexpr(stmt, :method), src.code) + lr = lines_required(idx, src, edges; norequire=exclude_named_typedefs(src, edges)) + idx = findfirst(stmt->Meta.isexpr(stmt, :(=)) && Meta.isexpr(stmt.args[2], :call) && is_global_ref(stmt.args[2].args[1], Core, :Box), src.code) + @test lr[idx] + # but make sure we don't break primitivetype & abstracttype (https://github.com/timholy/Revise.jl/pull/611) + if isdefined(Core, :_primitivetype) + thk = Meta.lower(Main, quote + primitive type WindowsRawSocket sizeof(Ptr) * 8 end + end) + src = thk.args[1] + edges = CodeEdges(src) + idx = findfirst(istypedef, src.code) + r = LoweredCodeUtils.typedef_range(src, idx) + @test last(r) == length(src.code) - 1 + end + + @testset "Display" begin + # worth testing because this has proven quite crucial for debugging and + # ensuring that these structures are as "self-documenting" as possible. + io = IOBuffer() + l = LoweredCodeUtils.Links(Int[], [3, 5], LoweredCodeUtils.NamedVar[:hello]) + show(io, l) + str = String(take!(io)) + @test occursin('∅', str) + @test !occursin("GlobalRef", str) + # CodeLinks + ex = quote + s = 0.0 + for i = 1:5 + global s + s += rand() + end + return s + end + lwr = Meta.lower(Main, ex) + src = lwr.args[1] + cl = LoweredCodeUtils.CodeLinks(src) + show(io, cl) + str = String(take!(io)) + @test occursin(r"slot 1:\n preds: ssas: \[\d+, \d+\], slots: ∅, names: ∅;\n succs: ssas: \[\d+, \d+, \d+\], slots: ∅, names: ∅;\n assign @: \[\d+, \d+\]", str) + @test occursin(r"succs: ssas: ∅, slots: \[\d+\], names: ∅;", str) + @test occursin(r"s:\n preds: ssas: \[\d+\], slots: ∅, names: ∅;\n succs: ssas: \[\d+, \d+, \d+\], slots: ∅, names: ∅;\n assign @: \[\d, \d+\]", str) || + occursin(r"s:\n preds: ssas: \[\d+, \d+\], slots: ∅, names: ∅;\n succs: ssas: \[\d+, \d+, \d+\], slots: ∅, names: ∅;\n assign @: \[\d, \d+\]", str) # with global var inference + if Base.VERSION < v"1.8" # changed by global var inference + @test occursin(r"\d+ preds: ssas: \[\d+\], slots: ∅, names: \[:s\];\n\d+ succs: ssas: ∅, slots: ∅, names: \[:s\];", str) + end + LoweredCodeUtils.print_with_code(io, src, cl) + str = String(take!(io)) + if isdefined(Base.IRShow, :show_ir_stmt) + @test occursin(r"slot 1:\n preds: ssas: \[\d+, \d+\], slots: ∅, names: ∅;\n succs: ssas: \[\d+, \d+, \d+\], slots: ∅, names: ∅;\n assign @: \[\d+, \d+\]", str) + @test occursin("# see name s", str) + @test occursin("# see slot 1", str) + if Base.VERSION < v"1.8" # changed by global var inference + @test occursin(r"# preds: ssas: \[\d+\], slots: ∅, names: \[:s\]; succs: ssas: ∅, slots: ∅, names: \[:s\];", str) + end + else + @test occursin("No IR statement printer", str) + end + # CodeEdges + edges = CodeEdges(src) + show(io, edges) + str = String(take!(io)) + @test occursin(r"s: assigned on \[\d, \d+\], depends on \[\d+\], and used by \[\d+, \d+, \d+\]", str) || + occursin(r"s: assigned on \[\d, \d+\], depends on \[\d+, \d+\], and used by \[\d+, \d+, \d+\]", str) # global var inference + if Base.VERSION < v"1.9" + @test (count(occursin("statement $i depends on [1, $(i-1), $(i+1)] and is used by [1, $(i+1)]", str) for i = 1:length(src.code)) == 1) || + (count(occursin("statement $i depends on [4, $(i-1), $(i+4)] and is used by [$(i+2)]", str) for i = 1:length(src.code)) == 1) + end + LoweredCodeUtils.print_with_code(io, src, edges) + str = String(take!(io)) + if isdefined(Base.IRShow, :show_ir_stmt) + @test occursin(r"s: assigned on \[\d, \d+\], depends on \[\d+\], and used by \[\d+, \d+, \d+\]", str) || + occursin(r"s: assigned on \[\d, \d+\], depends on \[\d+, \d+\], and used by \[\d+, \d+, \d+\]", str) + if Base.VERSION < v"1.9" + @test (count(occursin("preds: [1, $(i-1), $(i+1)], succs: [1, $(i+1)]", str) for i = 1:length(src.code)) == 1) || + (count(occursin("preds: [4, $(i-1), $(i+4)], succs: [$(i+2)]", str) for i = 1:length(src.code)) == 1) # global var inference + end + else + @test occursin("No IR statement printer", str) + end + # Works with Frames too + frame = Frame(ModSelective, ex) + edges = CodeEdges(frame.framecode.src) + LoweredCodeUtils.print_with_code(io, frame, edges) + str = String(take!(io)) + if isdefined(Base.IRShow, :show_ir_stmt) + @test occursin(r"s: assigned on \[\d, \d+\], depends on \[\d+\], and used by \[\d+, \d+, \d+\]", str) || + occursin(r"s: assigned on \[\d, \d+\], depends on \[\d, \d+\], and used by \[\d+, \d+, \d+\]", str) # global var inference + if Base.VERSION < v"1.9" + @test (count(occursin("preds: [1, $(i-1), $(i+1)], succs: [1, $(i+1)]", str) for i = 1:length(src.code)) == 1) || + (count(occursin("preds: [4, $(i-1), $(i+4)], succs: [$(i+2)]", str) for i = 1:length(src.code)) == 1) # global var inference + end + else + @test occursin("No IR statement printer", str) + end + end +end + +@testset "selective interpretation of toplevel definitions" begin + gen_mock_module() = Core.eval(@__MODULE__, :(module $(gensym(:LoweredCodeUtilsTestMock)) end)) + function check_toplevel_definition_interprete(ex, defs, undefs) + m = gen_mock_module() + lwr = Meta.lower(m, ex) + src = first(lwr.args) + stmts = src.code + edges = CodeEdges(src) + + isrq = lines_required!(istypedef.(stmts), src, edges) + frame = Frame(m, src) + selective_eval_fromstart!(frame, isrq, #= toplevel =# true) + + for def in defs; @test isdefined(m, def); end + for undef in undefs; @test !isdefined(m, undef); end + end + + @testset "case: $(i), interpret: $(defs), ignore $(undefs)" for (i, ex, defs, undefs) in ( + (1, :(abstract type Foo end), (:Foo,), ()), + + (2, :(struct Foo end), (:Foo,), ()), + + (3, quote + struct Foo + val + end + end, (:Foo,), ()), + + (4, quote + struct Foo{T} + val::T + Foo(v::T) where {T} = new{T}(v) + end + end, (:Foo,), ()), + + (5, :(primitive type Foo 32 end), (:Foo,), ()), + + (6, quote + abstract type Foo end + struct Foo1 <: Foo end + struct Foo2 <: Foo end + end, (:Foo, :Foo1, :Foo2), ()), + + (7, quote + struct Foo + v + Foo(f) = new(f()) + end + + foo = Foo(()->throw("don't interpret me")) + end, (:Foo,), (:foo,)), + + # https://github.com/JuliaDebug/LoweredCodeUtils.jl/issues/47 + (8, quote + struct Foo + b::Bool + Foo(b) = new(b) + end + + foo = Foo(false) + end, (:Foo,), (:foo,)), + + # https://github.com/JuliaDebug/LoweredCodeUtils.jl/pull/48 + # we shouldn't make `add_links!` recur into `QuoteNode`, otherwise the variable + # `bar` will be selected as a requirement for `Bar1` (, which has "bar" field) + (9, quote + abstract type Bar end + struct Bar1 <: Bar + bar + end + + r = (throw("don't interpret me"); rand(10000000000000000)) + bar = Bar1(r) + show(bar) + end, (:Bar, :Bar1), (:r, :bar)) + ) + + check_toplevel_definition_interprete(ex, defs, undefs) + end +end diff --git a/packages/LoweredCodeUtils/test/runtests.jl b/packages/LoweredCodeUtils/test/runtests.jl new file mode 100644 index 0000000..c3913e9 --- /dev/null +++ b/packages/LoweredCodeUtils/test/runtests.jl @@ -0,0 +1,9 @@ +using LoweredCodeUtils +using Test + +# @testset "Ambiguity" begin +# @test isempty(detect_ambiguities(LoweredCodeUtils, LoweredCodeUtils.JuliaInterpreter, Base, Core)) +# end + +include("signatures.jl") +include("codeedges.jl") diff --git a/packages/LoweredCodeUtils/test/signatures.jl b/packages/LoweredCodeUtils/test/signatures.jl new file mode 100644 index 0000000..66923a6 --- /dev/null +++ b/packages/LoweredCodeUtils/test/signatures.jl @@ -0,0 +1,503 @@ +using LoweredCodeUtils +using InteractiveUtils +using JuliaInterpreter +using JuliaInterpreter: finish_and_return! +using Core: CodeInfo +using Base.Meta: isexpr +using Test + +module Lowering +using Parameters +struct Caller end +struct Gen{T} end +end + +# Stuff for https://github.com/timholy/Revise.jl/issues/422 +module Lowering422 +const LVec{N, T} = NTuple{N, Base.VecElement{T}} +const LT{T} = Union{LVec{<:Any, T}, T} +const FloatingTypes = Union{Float32, Float64} +end + +bodymethtest0(x) = 0 +function bodymethtest0(x) + y = 2x + y + x +end +bodymethtest1(x, y=1; z="hello") = 1 +bodymethtest2(x, y=Dict(1=>2); z="hello") = 2 +bodymethtest3(x::T, y=Dict(1=>2); z="hello") where T<:AbstractFloat = 3 +# No kw but has optional args +bodymethtest4(x, y=1) = 4 +bodymethtest5(x, y=Dict(1=>2)) = 5 + +@testset "Signatures" begin + signatures = Set{Any}() + newcode = CodeInfo[] + for ex in (:(f(x::Int8; y=0) = y), + :(f(x::Int16; y::Int=0) = 2), + :(f(x::Int32; y="hello", z::Int=0) = 3), + :(f(x::Int64;) = 4), + :(f(x::Array{Float64,K}; y::Int=0) where K = K), + # Keyword-arg functions that have an anonymous function inside + :(fanon(list; sorted::Bool=true,) = sorted ? sort!(list, by=x->abs(x)) : list), + # Keyword & default positional args + :(g(x, y="hello"; z::Int=0) = 1), + # Return type annotations + :(annot(x, y; z::Bool=false,)::Nothing = nothing), + # Generated methods + quote + @generated function h(x) + if x <: Integer + return :(x ^ 2) + else + return :(x) + end + end + end, + quote + function h(x::Int, y) + if @generated + return y <: Integer ? :(x*y) : :(x+y) + else + return 2x+3y + end + end + end, + :(@generated genkw(; b=2) = nothing), # https://github.com/timholy/Revise.jl/issues/290 + # Generated constructors + quote + function Gen{T}(x) where T + if @generated + return T <: Integer ? :(x^2) : :(2x) + else + return 7x + end + end + end, + # Conditional methods + quote + if 0.8 > 0.2 + fctrue(x) = 1 + else + fcfalse(x) = 1 + end + end, + # Call methods + :((::Caller)(x::String) = length(x)), + ) + Core.eval(Lowering, ex) + frame = Frame(Lowering, ex) + rename_framemethods!(frame) + pc = methoddefs!(signatures, frame; define=false) + push!(newcode, frame.framecode.src) + end + + # Manually add the signature for the Caller constructor, since that was defined + # outside of manual lowering + push!(signatures, Tuple{Type{Lowering.Caller}}) + + nms = names(Lowering; all=true) + modeval, modinclude = getfield(Lowering, :eval), getfield(Lowering, :include) + failed = [] + for fsym in nms + f = getfield(Lowering, fsym) + isa(f, Base.Callable) || continue + (f === modeval || f === modinclude) && continue + for m in methods(f) + if m.sig ∉ signatures + push!(failed, m.sig) + end + end + end + @test isempty(failed) + # Ensure that all names are properly resolved + for code in newcode + Core.eval(Lowering, code) + end + nms2 = names(Lowering; all=true) + @test nms2 == nms + @test Lowering.f(Int8(0)) == 0 + @test Lowering.f(Int8(0); y="LCU") == "LCU" + @test Lowering.f(Int16(0)) == Lowering.f(Int16(0), y=7) == 2 + @test Lowering.f(Int32(0)) == Lowering.f(Int32(0); y=22) == Lowering.f(Int32(0); y=:cat, z=5) == 3 + @test Lowering.f(Int64(0)) == 4 + @test Lowering.f(rand(3,3)) == Lowering.f(rand(3,3); y=5) == 2 + @test Lowering.fanon([1,3,-2]) == [1,-2,3] + @test Lowering.g(0) == Lowering.g(0,"LCU") == Lowering.g(0; z=5) == Lowering.g(0,"LCU"; z=5) == 1 + @test Lowering.annot(0,0) === nothing + @test Lowering.h(2) == 4 + @test Lowering.h(2.0) == 2.0 + @test Lowering.h(2, 3) == 6 + @test Lowering.h(2, 3.0) == 5.0 + @test Lowering.fctrue(0) == 1 + @test_throws UndefVarError Lowering.fcfalse(0) + @test (Lowering.Caller())("Hello, world") == 12 + g = Lowering.Gen{Float64} + @test g(3) == 6 + + # Don't be deceived by inner methods + signatures = [] + ex = quote + function fouter(x) + finner(::Float16) = 2x + return finner(Float16(1)) + end + end + Core.eval(Lowering, ex) + frame = Frame(Lowering, ex) + rename_framemethods!(frame) + methoddefs!(signatures, frame; define=false) + @test length(signatures) == 1 + @test LoweredCodeUtils.whichtt(signatures[1]) == first(methods(Lowering.fouter)) + + # Check output of methoddef! + frame = Frame(Lowering, :(function nomethod end)) + ret = methoddef!(empty!(signatures), frame; define=true) + @test isempty(signatures) + @test ret === nothing + frame = Frame(Lowering, :(function amethod() nothing end)) + ret = methoddef!(empty!(signatures), frame; define=true) + @test !isempty(signatures) + @test isa(ret, NTuple{2,Int}) + + # Anonymous functions in method signatures + ex = :(max_values(T::Union{map(X -> Type{X}, Base.BitIntegerSmall_types)...}) = 1 << (8*sizeof(T))) # base/abstractset.jl + frame = Frame(Base, ex) + rename_framemethods!(frame) + signatures = Set{Any}() + methoddef!(signatures, frame; define=false) + @test length(signatures) == 1 + @test first(signatures) == which(Base.max_values, Tuple{Type{Int16}}).sig + + # define + ex = :(fdefine(x) = 1) + frame = Frame(Lowering, ex) + empty!(signatures) + methoddefs!(signatures, frame; define=false) + @test_throws MethodError Lowering.fdefine(0) + frame = Frame(Lowering, ex) + empty!(signatures) + methoddefs!(signatures, frame; define=true) + @test Lowering.fdefine(0) == 1 + + # define with correct_name! + ex = quote + @generated function generated1(A::AbstractArray{T,N}, val) where {T,N} + ex = Expr(:tuple) + for i = 1:N + push!(ex.args, :val) + end + return ex + end + end + frame = Frame(Lowering, ex) + rename_framemethods!(frame) + empty!(signatures) + methoddefs!(signatures, frame; define=true) + @test length(signatures) == 2 + @test Lowering.generated1(rand(2,2), 3.2) == (3.2, 3.2) + ex = quote + another_kwdef(x, y=1; z="hello") = 333 + end + frame = Frame(Lowering, ex) + rename_framemethods!(frame) + empty!(signatures) + methoddefs!(signatures, frame; define=true) + @test length(signatures) == 5 + @test Lowering.another_kwdef(0) == 333 + ex = :(@generated genkw2(; b=2) = nothing) # https://github.com/timholy/Revise.jl/issues/290 + frame = Frame(Lowering, ex) + # rename_framemethods!(frame) + empty!(signatures) + methoddefs!(signatures, frame; define=true) + @test length(signatures) == 4 + @test Lowering.genkw2() === nothing + + # Test for correct exit (example from base/namedtuples.jl) + ex = quote + function merge(a::NamedTuple{an}, b::NamedTuple{bn}) where {an, bn} + if @generated + names = merge_names(an, bn) + types = merge_types(names, a, b) + vals = Any[ :(getfield($(sym_in(n, bn) ? :b : :a), $(QuoteNode(n)))) for n in names ] + :( NamedTuple{$names,$types}(($(vals...),)) ) + else + names = merge_names(an, bn) + types = merge_types(names, typeof(a), typeof(b)) + NamedTuple{names,types}(map(n->getfield(sym_in(n, bn) ? b : a, n), names)) + end + end + end + frame = Frame(Base, ex) + rename_framemethods!(frame) + empty!(signatures) + stmt = JuliaInterpreter.pc_expr(frame) + if !LoweredCodeUtils.ismethod(stmt) + pc = JuliaInterpreter.next_until!(LoweredCodeUtils.ismethod, frame, true) + end + pc, pc3 = methoddef!(signatures, frame; define=false) # this tests that the return isn't `nothing` + pc, pc3 = methoddef!(signatures, frame; define=false) + @test length(signatures) == 2 # both the GeneratedFunctionStub and the main method + + # With anonymous functions in signatures + ex = :(const BitIntegerType = Union{map(T->Type{T}, Base.BitInteger_types)...}) + frame = Frame(Lowering, ex) + rename_framemethods!(frame) + empty!(signatures) + methoddefs!(signatures, frame; define=false) + @test !isempty(signatures) + + for m in methods(bodymethtest0) + @test bodymethod(m) === m + end + @test startswith(String(bodymethod(first(methods(bodymethtest1))).name), "#") + @test startswith(String(bodymethod(first(methods(bodymethtest2))).name), "#") + @test startswith(String(bodymethod(first(methods(bodymethtest3))).name), "#") + @test bodymethod(first(methods(bodymethtest4))).nargs == 3 # one extra for #self# + @test bodymethod(first(methods(bodymethtest5))).nargs == 3 + m = @which sum([1]; dims=1) + # Issue in https://github.com/timholy/CodeTracking.jl/pull/48 + mbody = bodymethod(m) + @test mbody != m && mbody.file != :none + # varargs keyword methods + m = which(Base.print_shell_escaped, (IO, AbstractString)) + mbody = bodymethod(m) + @test isa(mbody, Method) && mbody != m + + ex = quote + function print_shell_escaped(io::IO, cmd::AbstractString, args::AbstractString...; + special::AbstractString="") + print_shell_word(io, cmd, special) + for arg in args + print(io, ' ') + print_shell_word(io, arg, special) + end + end + end + frame = Frame(Base, ex) + rename_framemethods!(frame) + empty!(signatures) + methoddefs!(signatures, frame; define=false) + @test length(signatures) >= 3 - isdefined(Core, :kwcall) + + ex = :(typedsig(x) = 1) + frame = Frame(Lowering, ex) + methoddefs!(signatures, frame; define=true) + ex = :(typedsig(x::Int) = 2) + frame = Frame(Lowering, ex) + JuliaInterpreter.next_until!(LoweredCodeUtils.ismethod3, frame, true) + empty!(signatures) + methoddefs!(signatures, frame; define=true) + @test first(signatures).parameters[end] == Int + + # Multiple keyword arg methods per frame + # (Revise issue #363) + ex = quote + keywrd1(x; kwarg=false) = 1 + keywrd2(x; kwarg="hello") = 2 + keywrd3(x; kwarg=:stuff) = 3 + end + Core.eval(Lowering, ex) + frame = Frame(Lowering, ex) + rename_framemethods!(frame) + empty!(signatures) + pc, pc3 = methoddef!(signatures, frame; define=false) + @test pc < length(frame.framecode.src.code) + kw2sig = Tuple{typeof(Lowering.keywrd2), Any} + @test kw2sig ∉ signatures + pc = methoddefs!(signatures, frame; define=false) + @test pc === nothing + @test kw2sig ∈ signatures + + # Module-scoping + ex = :(Base.@irrational π 3.14159265358979323846 pi) + frame = Frame(Base.MathConstants, ex) + rename_framemethods!(frame) + empty!(signatures) + methoddefs!(signatures, frame; define=false) + @test !isempty(signatures) + + # Inner methods in structs. Comes up in, e.g., Core.Compiler.Params. + # The body of CustomMS is an SSAValue. + ex = quote + struct MyStructWithMeth + x::Int + global function CustomMS(;x=1) + return new(x) + end + MyStructWithMeth(x) = new(x) + end + end + Core.eval(Lowering, ex) + frame = Frame(Lowering, ex) + rename_framemethods!(frame) + empty!(signatures) + methoddefs!(signatures, frame; define=false) + @test Tuple{typeof(Lowering.CustomMS)} ∈ signatures + + # https://github.com/timholy/Revise.jl/issues/398 + ex = quote + @with_kw struct Items + n::Int + items::Vector{Int} = [i for i=1:n] + end + end + Core.eval(Lowering, ex) + frame = Frame(Lowering, ex) + dct = rename_framemethods!(frame) + ks = collect(filter(k->startswith(String(k), "#Items#"), keys(dct))) + @test length(ks) == 2 + @test dct[ks[1]] == dct[ks[2]] + @test isdefined(Lowering, ks[1]) || isdefined(Lowering, ks[2]) + if !isdefined(Core, :kwcall) + nms = filter(sym->occursin(r"#Items#\d+#\d+", String(sym)), names(Lowering; all=true)) + @test length(nms) == 1 + end + + # https://github.com/timholy/Revise.jl/issues/422 + ex = :(@generated function fneg(x::T) where T<:LT{<:FloatingTypes} + s = """ + %2 = fneg $(llvm_type(T)) %0 + ret $(llvm_type(T)) %2 + """ + return :( + $(Expr(:meta, :inline)); + Base.llvmcall($s, T, Tuple{T}, x) + ) + end) + empty!(signatures) + Core.eval(Lowering422, ex) + frame = Frame(Lowering422, ex) + rename_framemethods!(frame) + pc = methoddefs!(signatures, frame; define=false) + @test typeof(Lowering422.fneg) ∈ Set(Base.unwrap_unionall(sig).parameters[1] for sig in signatures) + + # Scoped names (https://github.com/timholy/Revise.jl/issues/568) + ex = :(f568() = -1) + Core.eval(Lowering, ex) + @test Lowering.f568() == -1 + empty!(signatures) + ex = :(f568() = -2) + frame = Frame(Lowering, ex) + pcstop = findfirst(LoweredCodeUtils.ismethod3, frame.framecode.src.code) + pc = 1 + while pc < pcstop + pc = JuliaInterpreter.step_expr!(finish_and_return!, frame, true) + end + pc = methoddef!(finish_and_return!, signatures, frame, pc; define=true) + @test Tuple{typeof(Lowering.f568)} ∈ signatures + @test Lowering.f568() == -2 + + # Undefined names + # This comes from FileWatching; WindowsRawSocket is only defined on Windows + ex = quote + if Sys.iswindows() + using Base: WindowsRawSocket + function wait(socket::WindowsRawSocket; readable=false, writable=false) + fdw = _FDWatcher(socket, readable, writable) + try + return wait(fdw, readable=readable, writable=writable) + finally + close(fdw, readable, writable) + end + end + end + end + frame = Frame(Lowering, ex) + rename_framemethods!(frame) + + # https://github.com/timholy/Revise.jl/issues/550 + using Pkg + try + # we test with the old version of CBinding, let's do it in an isolated environment + Pkg.activate(; temp=true) + + @info "Adding CBinding to the environment for test purposes" + Pkg.add(; name="CBinding", version="0.9.4") # `@cstruct` isn't defined for v1.0 and above + + m = Module() + Core.eval(m, :(using CBinding)) + + ex = :(@cstruct S { + val::Int8 + }) + empty!(signatures) + Core.eval(m, ex) + frame = Frame(m, ex) + rename_framemethods!(frame) + pc = methoddefs!(signatures, frame; define=false) + @test !isempty(signatures) # really we just need to know that `methoddefs!` completed without getting stuck + finally + Pkg.activate() # back to the original environment + end +end + +# https://github.com/timholy/Revise.jl/issues/643 +module Revise643 + +using LoweredCodeUtils, JuliaInterpreter, Test + +# make sure to not define `foogr` before macro expansion, +# otherwise it will be resolved as `QuoteNode` +macro deffoogr() + gr = GlobalRef(__module__, :foogr) # will be lowered to `GlobalRef` + quote + $gr(args...) = length(args) + end +end +let + ex = quote + @deffoogr + @show foogr(1,2,3) + end + methranges = rename_framemethods!(Frame(@__MODULE__, ex)) + @test haskey(methranges, :foogr) +end + +function fooqn end +macro deffooqn() + sig = :($(GlobalRef(__module__, :fooqn))(args...)) # will be lowered to `QuoteNode` + return Expr(:function, sig, Expr(:block, __source__, :(length(args)))) +end +let + ex = quote + @deffooqn + @show fooqn(1,2,3) + end + methranges = rename_framemethods!(Frame(@__MODULE__, ex)) + @test haskey(methranges, :fooqn) +end + +# define methods in other module +module sandboxgr end +macro deffoogr_sandbox() + gr = GlobalRef(sandboxgr, :foogr_sandbox) # will be lowered to `GlobalRef` + quote + $gr(args...) = length(args) + end +end +let + ex = quote + @deffoogr_sandbox + @show sandboxgr.foogr_sandbox(1,2,3) + end + methranges = rename_framemethods!(Frame(@__MODULE__, ex)) + @test haskey(methranges, :foogr_sandbox) +end + +module sandboxqn; function fooqn_sandbox end; end +macro deffooqn_sandbox() + sig = :($(GlobalRef(sandboxqn, :fooqn_sandbox))(args...)) # will be lowered to `QuoteNode` + return Expr(:function, sig, Expr(:block, __source__, :(length(args)))) +end +let + ex = quote + @deffooqn_sandbox + @show sandboxqn.fooqn_sandbox(1,2,3) + end + methranges = rename_framemethods!(Frame(@__MODULE__, ex)) + @test haskey(methranges, :fooqn_sandbox) +end + +end diff --git a/packages/OrderedCollections/.codecov.yml b/packages/OrderedCollections/.codecov.yml new file mode 100644 index 0000000..69cb760 --- /dev/null +++ b/packages/OrderedCollections/.codecov.yml @@ -0,0 +1 @@ +comment: false diff --git a/packages/OrderedCollections/.github/workflows/CI.yml b/packages/OrderedCollections/.github/workflows/CI.yml new file mode 100644 index 0000000..8d43117 --- /dev/null +++ b/packages/OrderedCollections/.github/workflows/CI.yml @@ -0,0 +1,46 @@ +name: CI +on: + pull_request: + push: + branches: + - master + tags: '*' +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.0' + - '1' +# - 'nightly' + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x64 + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v1 + with: + file: lcov.info diff --git a/packages/OrderedCollections/.github/workflows/Documenter.yml b/packages/OrderedCollections/.github/workflows/Documenter.yml new file mode 100644 index 0000000..586e7ef --- /dev/null +++ b/packages/OrderedCollections/.github/workflows/Documenter.yml @@ -0,0 +1,18 @@ +name: Documenter +on: + push: + branches: [master] + tags: [v*] + pull_request: + +jobs: + Documenter: + name: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/julia-buildpkg@latest + - uses: julia-actions/julia-docdeploy@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/packages/OrderedCollections/.github/workflows/TagBot.yml b/packages/OrderedCollections/.github/workflows/TagBot.yml new file mode 100644 index 0000000..f49313b --- /dev/null +++ b/packages/OrderedCollections/.github/workflows/TagBot.yml @@ -0,0 +1,15 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/packages/OrderedCollections/.gitignore b/packages/OrderedCollections/.gitignore new file mode 100644 index 0000000..49b1c24 --- /dev/null +++ b/packages/OrderedCollections/.gitignore @@ -0,0 +1,4 @@ +doc/build +docs/build/ +docs/site/ +Manifest.toml diff --git a/packages/OrderedCollections/Changelog_DataStructures.md b/packages/OrderedCollections/Changelog_DataStructures.md new file mode 100644 index 0000000..3f3d3dc --- /dev/null +++ b/packages/OrderedCollections/Changelog_DataStructures.md @@ -0,0 +1,292 @@ + +0.7.0 / 2017-09-02 +================== + + * Drop support for Julia v0.5 (and update to v0.6/v0.7 syntax) + * Add some missing things to docs (#317) + * Remove additional v0.6 deprecations + * Fix a "formal" ambiguity on 0.6+ and enable ambiguity tests + * Remove Compat (not needed/used right now) + * Move all tests to testsets + +v0.6.1 / 2017-07-26 +================== + * Fix most of 0.7 depwarns + +v0.6.0 / 2017-07-09 +================== + * Fix depwarn on 0.7 + * Update CI URLs to point to new caching infrastructure + * Re-fix 0.6 depwarns + +v0.5.3 / 2017-02-21 +================== + * Julia v0.6 depwarn, ambiguity, and other misc fixes + * Fix 0.6 typealias depwarn + * Fix 0.6 abstract type declaration depwarn + * Fix 0.6 misc other depwarns + +v0.5.2 / 2017-01-18 +================== + * Julia 0.6 fixes + * Remove recently introduced TypeVars. + * Don't allow failure on nightly + +v0.5.1 / 2017-01-07 +================== + * Temporarily revert removal of HashDict (broke gadfly) + +v0.5.0 / 2017-01-05 +================== + * Changed OrderedDict implementation to Jeff Bezanson's version (from Julia #10116) + * Remove HashDict (no longer needed), refactor Dict-related classes + * Added more Dict-related tests + * Allow OrderedDicts to be sorted + * Fix xor deprecations + +v0.4.6 / 2016-07-28 +================== + * isdefined -> isassigned + +v0.4.5 / 2016-07-28 +================== + * Fixes for Julia v0.5 + * Exception type updates for + * Export complement if not available in Base + * Fix ASCIIString, UTF8String -> String deprecations + * Fix getfield deprecation + * Add RTD badge to Readme + +v0.4.4 / 2016-04-10 +================== + * rename files with underscores for consistency + * OrderedDict: use type parameters for constructor, rather than as parameters + * add various docstrings + * Remove spaces between {} and () in function/constructor definitions + +v0.4.3 / 2016-02-10 +================== + * Many deprecation warnings were deleted (https://github.com/JuliaLang/DataStructures.jl/pull/161) + * Ordered sets now have indexing + * Performance improvements to OrderedDict + + +v0.4.2 / 2016-01-13 +================== + + * Fix OrderedDict constructors (with tests) + * Dead code, tree.jl removal + +v0.4.1 / 2015-12-29 +=================== + + * Updated Changelog + * Merge pull request #156 from JuliaLang/kms/remove-v0.3-part2 + * Replace tuple_or_pair with Pair() or Pair{} + * More thorough removal of v0.3 support + * Updated Changelog.md + +v0.4.1 / 2015-12-29 +================== + + * More thorough removal of v0.3 support + * Replace tuple_or_pair with Pair() or Pair{} + +v0.4.0 / 2015-12-28 +=================== + + * Remove support for Julia 0.3 + +v0.3.14 / 2015-11-14 +==================== + + * OrderedDict: + * Implement merge for OrderedDict + * Serialize and deserialize + * Remove invalid rst and align elements + * Fix #34, implement `==` instead of `isequal` in places + * Define ==(x::Nil, y::Nil) and ==(x::Cons, y::Cons) + +v0.3.13 / 2015-09-18 +==================== + + * Julia v0.4 updates + * Union() -> Union{} + * 0.4 bindings deprecation + * Add operator imports to fix deprecation warnings + * Travis + * Run tests on 0.3, 0.4, and nightly (0.5) + * Enable osx + * (Re)enable codecov + * Add precompile directive + * Switched setindex! to insert! + * Fix Pair usage for OrderedDict + +v0.3.11 / 2015-07-14 +==================== + + * Fix deprecated syntax in OrderedSet test + * Updated README with extra DefaultDict examples + * More formatting updates to README.rst + * Remove syntax deprecation warnings on 0.4 + +v0.3.10 / 2015-06-29 +==================== + + * REQUIRE: bump Julia version to v0.3 + * Fix serialization ambiguity warnings + +v0.3.9 / 2015-05-03 +=================== + + * Fix error on 0.4-dev, allow running tests without installing + +v0.3.8 / 2015-04-18 +=================== + + * Add special OrderedDict deprection for Numbers + * Fix warning about {A, B...} + +v0.3.7 / 2015-04-17 +=================== + + * 0.4 Compat fixes + * Implement nlargest and nsmallest + +v0.3.6 / 2015-03-05 +=================== + + * Updated OrderedSet, OrderedDict tests + * Update OrderedDict, OrderedSet constructors to take iterables + * Use Julia 0.4 syntax + * Added compat support for Julia v0.3 + * Rewrite README in rst format (instead of md) + * Get coverage data generation back up for Coveralls + * Update Travis to use Julia Language Support + * use Base.warn_once() instead of warn() + * Support v0.4 style association construction via Pair operator + * Update syntax to avoid deprecation warnings on Julia 0.4 + * Consistent whitespace + +v0.3.4 / 2014-10-14 +=================== + + * Fix #60 + * Update Dict construction to use new syntax + * Fix signed/unsigned issue in hashindex + * Modernize Travis, Pkg.test compat, coverage, badges + +v0.3.2 / 2014-08-31 +=================== + + * Remove trailing whitespace + * Add more constructors for Trie + * Remove trailing whitespace + +v0.3.1 / 2014-07-14 +=================== + + * Update README + * Deprecate add\! in favor of push\! + +v0.3.0 / 2014-06-10 +=================== + + * Bump REQUIRE to v0.3, for incompatible change in test_throws + +v0.2.15 / 2014-06-10 +==================== + + * Revert "fix `@test_throw` warnings" + +v0.2.14 / 2014-06-02 +==================== + + * Import serialize_type in hashdict.jl + * Add some clarification on code examples + * fix `@test_throw` warnings + * use SVG logo for travis status + * rename run_tests.jl to runtests.jl + +v0.2.13 / 2014-05-08 +==================== + + * Revert "Remove unused code" + * Fix broken tests + +v0.2.12 / 2014-04-26 +==================== + + * Import Base.reverse + * Inserted missing comma + * Avoid stack overflow in length method. Use iterator in show method + * Changed name from add_singleton! to push! + * Update README.md + +v0.2.11 / 2014-04-10 +==================== + + * Update README.md (closes #24) + * Changed the name make_set to add_singleton + * import serialize, deserialize + * Clean up code. Follow Dict interface more closely. + * Added working test of make_set! + * Added make_set! to exports in DataStructures.jl + * Changed length(s.parents) to length(s) + * Added version of make_set! which automatically chooses the new element as the next available one + * Added ! to the name of the make_set function, since it modifies the structure + * Added make_set to add single element as a new disjoint set, with its parent equal to itself + * Implemented list iterator functions + * add list and binary tree. closes #17 + +v0.2.10 / 2014-03-02 +==================== + + * Revert "Update REQUIRE to julia v0.3" + +v0.2.9 / 2014-02-26 +=================== + + * Update REQUIRE to julia v0.3 + * Update README.md + * Fix travis config. Enable testing with releases. + * Change Travis badge url to JuliaLang + * README.md: OrderedDefaultDict -> DefaultOrderedDict + * fix C++ template syntax in README + * Added/updated various dictionary, set variants + * update travis.yml (disable apt-get upgrade) + * add classified counters + * add classified collections + +v0.2.5 / 2013-10-08 +=================== + + * improved benchmark scripts + +0.2.4 / 2013-07-27 +================== + + * add travis logo to readme + * add travis.yml + * use run_tests.jl in the place of test/test_all.jl + * Added 1 missing API call to the documentation + +0.2.3 / 2013-04-21 +================== + + * export in_same_set + +0.2.0 / 2013-04-15 +================== + + * add julia version requirement + * Test ==> Base.Test & add test_all.jl + * add empty REQUIRE file + * Update README.md + * add license + * add readme + * improved interface and added test + * renamed to DataStructures + * add stack and queue (tested) + * add Dequeue (tested) + * Initial commit diff --git a/packages/OrderedCollections/License.md b/packages/OrderedCollections/License.md new file mode 100644 index 0000000..34299fe --- /dev/null +++ b/packages/OrderedCollections/License.md @@ -0,0 +1,7 @@ +Copyright (c) 2013 Dahua Lin + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/packages/OrderedCollections/Project.toml b/packages/OrderedCollections/Project.toml new file mode 100644 index 0000000..9599d63 --- /dev/null +++ b/packages/OrderedCollections/Project.toml @@ -0,0 +1,14 @@ +name = "OrderedCollections" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.4.1" + +[compat] +julia = "0.7, 1" + +[extras] +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Serialization", "Test", "Random"] diff --git a/packages/OrderedCollections/README.md b/packages/OrderedCollections/README.md new file mode 100644 index 0000000..e04baf3 --- /dev/null +++ b/packages/OrderedCollections/README.md @@ -0,0 +1,20 @@ +[![Travis Build Status](https://travis-ci.org/JuliaCollections/OrderedCollections.jl.svg?branch=master)](https://travis-ci.org/JuliaCollections/OrderedCollections.jl) + +[![Appveyor Build Status](https://ci.appveyor.com/api/projects/status/5gw9xok4e58aixsv?svg=true)](https://ci.appveyor.com/project/kmsquire/datastructures-jl) +[![Test Coverage](https://codecov.io/github/JuliaCollections/OrderedCollections.jl/coverage.svg?branch=master)](https://codecov.io/github/JuliaCollections/OrderedCollections.jl?branch=master) + +[![Documentation](https://img.shields.io/badge/docs-latest-blue.svg)](https://juliacollections.github.io/OrderedCollections.jl/latest) + +OrderedCollections.jl +===================== + +This package implements OrderedDicts and OrderedSets, which are similar to containers in base Julia. +However, during iteration the Ordered* containers return items in the order in which they were added to the collection. +It also implements `LittleDict` which is a ordered dictionary, that is much faster than any other `AbstractDict` (ordered or not) for small collections. + +This package was split out from [DataStructures.jl](https://github.com/JuliaCollections/DataStructures.jl). + +Resources +--------- + +- **Documentation**: https://juliacollections.github.io/OrderedCollections.jl/latest diff --git a/packages/OrderedCollections/docs/make.jl b/packages/OrderedCollections/docs/make.jl new file mode 100644 index 0000000..a5d17d8 --- /dev/null +++ b/packages/OrderedCollections/docs/make.jl @@ -0,0 +1,21 @@ +using Documenter +using OrderedCollections + + +makedocs( + format = :html, + sitename = "OrderedCollections.jl", + pages = [ + "index.md", + "ordered_containers.md", + ] +) + +deploydocs( + repo = "github.com/JuliaCollections/OrderedCollections.jl.git", + julia = "0.7", + latest = "master", + target = "build", + deps = nothing, # we use the `format = :html`, without `mkdocs` + make = nothing, # we use the `format = :html`, without `mkdocs` +) diff --git a/packages/OrderedCollections/docs/mkdocs.yml b/packages/OrderedCollections/docs/mkdocs.yml new file mode 100644 index 0000000..4dc9753 --- /dev/null +++ b/packages/OrderedCollections/docs/mkdocs.yml @@ -0,0 +1,24 @@ +site_name: OrderedCollections.jl +repo_url: https://github.com/JuliaCollections/OrderedCollections.jl +site_description: Documentation for the Julia OrderedCollections package +site_author: JuliaCollections + +theme: readthedocs + +extra_css: + - assets/Documenter.css + +extra_javascript: + - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS_HTML + - assets/mathjaxhelper.js + +markdown_extensions: + - extra + - tables + - fenced_code + - mdx_math + +docs_dir: 'build' + +pages: + - Home: index.md diff --git a/packages/OrderedCollections/docs/src/index.md b/packages/OrderedCollections/docs/src/index.md new file mode 100644 index 0000000..fe1341d --- /dev/null +++ b/packages/OrderedCollections/docs/src/index.md @@ -0,0 +1,16 @@ +# OrderedCollections.jl + +This package implements associative containers that preserve the order of insertion: + +- OrderedDict +- OrderedSet +- LittleDict + +## Contents + +```@contents +Pages = [ + "index.md", + "ordered_containers.md", +] +``` diff --git a/packages/OrderedCollections/docs/src/ordered_containers.md b/packages/OrderedCollections/docs/src/ordered_containers.md new file mode 100644 index 0000000..204a353 --- /dev/null +++ b/packages/OrderedCollections/docs/src/ordered_containers.md @@ -0,0 +1,69 @@ +# OrderedSets + +`OrderedSets` are sets whose entries have a particular order. +Order refers to *insertion order*, which allows deterministic +iteration over the set: + +```julia +using Base.MathConstants +s = OrderedSet((π,e,γ,catalan,φ)) +for x in s + println(x) +end +#> π = 3.1415926535897... +#> ℯ = 2.7182818284590... +#> γ = 0.5772156649015... +#> catalan = 0.9159655941772... +#> φ = 1.6180339887498... +``` +All `Set` operations are available for OrderedSets. + +Note that to create an OrderedSet of a particular type, you must +specify the type in curly-braces: + +```julia +# create an OrderedSet of Strings +strs = OrderedSet{AbstractString}() +``` +# OrderedDicts +Similarly, `OrderedDict` are simply dictionaries whose entries have a particular +order. +```julia +d = OrderedDict{Char,Int}() +for c in 'a':'d' + d[c] = c-'a'+1 +end +for x in d + println(x) +end +#> 'a' => 1 +#> 'b' => 2 +#> 'c' => 3 +#> 'd' => 4 +``` +The insertion order is conserved when iterating on the dictionary itself, +its keys (through `keys(d)`), or its values (through `values(d)`). +All standard `Associative` and `Dict` functions are available for `OrderedDicts` + +# LittleDict +```julia +d = LittleDict{Char,Int}() +for c in 'a':'d' + d[c] = c-'a'+1 +end +for x in d + println(x) +end +#> 'a' => 1 +#> 'b' => 2 +#> 'c' => 3 +#> 'd' => 4 +``` +The `LittleDict` acts similarly to the `OrderedDict`. +However for small collections it is much faster. +Indeed the preceeding example (with the io redirected to `devnull`), runs 4x faster in the `LittleDict` version as the earlier `OrderedDict` version. + +```@docs +LittleDict +freeze +``` diff --git a/packages/OrderedCollections/src/OrderedCollections.jl b/packages/OrderedCollections/src/OrderedCollections.jl new file mode 100644 index 0000000..47e1128 --- /dev/null +++ b/packages/OrderedCollections/src/OrderedCollections.jl @@ -0,0 +1,32 @@ +module OrderedCollections + + import Base: <, <=, ==, convert, length, isempty, iterate, delete!, + show, dump, empty!, getindex, setindex!, get, get!, + in, haskey, keys, merge, copy, cat, + push!, pop!, popfirst!, insert!, + union!, delete!, empty, sizehint!, + isequal, hash, + map, map!, reverse, + first, last, eltype, getkey, values, sum, + merge, merge!, lt, Ordering, ForwardOrdering, Forward, + ReverseOrdering, Reverse, Lt, + isless, + union, intersect, symdiff, setdiff, setdiff!, issubset, + searchsortedfirst, searchsortedlast, in, + filter, filter!, ValueIterator, eachindex, keytype, + valtype, lastindex, nextind, + copymutable, emptymutable, dict_with_eltype + + export OrderedDict, OrderedSet, LittleDict + export freeze + + include("dict_support.jl") + include("ordered_dict.jl") + include("little_dict.jl") + include("ordered_set.jl") + include("dict_sorting.jl") + + import Base: similar + @deprecate similar(d::OrderedDict) empty(d) + @deprecate similar(s::OrderedSet) empty(s) +end diff --git a/packages/OrderedCollections/src/dict_sorting.jl b/packages/OrderedCollections/src/dict_sorting.jl new file mode 100644 index 0000000..cfcf918 --- /dev/null +++ b/packages/OrderedCollections/src/dict_sorting.jl @@ -0,0 +1,31 @@ +# Sort for dicts +import Base: sort, sort! + +function sort!(d::OrderedDict; byvalue::Bool=false, args...) + if d.ndel > 0 + rehash!(d) + end + + if byvalue + p = sortperm(d.vals; args...) + else + p = sortperm(d.keys; args...) + end + d.keys = d.keys[p] + d.vals = d.vals[p] + rehash!(d) + return d +end + +sort(d::OrderedDict; args...) = sort!(copy(d); args...) +@deprecate sort(d::Dict; args...) sort!(OrderedDict(d); args...) + +function sort(d::LittleDict; byvalue::Bool=false, args...) + if byvalue + p = sortperm(d.vals; args...) + else + p = sortperm(d.keys; args...) + end + return LittleDict(d.keys[p], d.vals[p]) +end + diff --git a/packages/OrderedCollections/src/dict_support.jl b/packages/OrderedCollections/src/dict_support.jl new file mode 100644 index 0000000..b44f514 --- /dev/null +++ b/packages/OrderedCollections/src/dict_support.jl @@ -0,0 +1,6 @@ +# support functions + +# _tablesz and hashindex are defined in Base, but are not exported, +# so they are redefined here. +_tablesz(x::Integer) = x < 16 ? 16 : one(x)<<((sizeof(x)<<3)-leading_zeros(x-1)) +hashindex(key, sz) = (reinterpret(Int,(hash(key))) & (sz-1)) + 1 diff --git a/packages/OrderedCollections/src/little_dict.jl b/packages/OrderedCollections/src/little_dict.jl new file mode 100644 index 0000000..25e99e6 --- /dev/null +++ b/packages/OrderedCollections/src/little_dict.jl @@ -0,0 +1,268 @@ +const StoreType = Union{<:Tuple, <:Vector} + +""" + LittleDict(keys, vals)<:AbstractDict + +An ordered dictionary type for small numbers of keys. +Rather than using `hash` or some other sophisticated measure +to store the vals in a clever arrangement, +it just keeps everything in a pair of lists. + +While theoretically this has expected time complexity _O(n)_ +(vs the hash-based [`OrderedDict`](@ref)/`Dict`'s expected time complexity _O(1)_, +and the search-tree-based `SortedDict`'s expected time complexity _O(log(n))_), +in practice it is really fast, because it is cache & SIMD friendly. + +It is reasonable to expect it to outperform an `OrderedDict`, +with up to around 30 elements in general; +or with up to around 50 elements if using a `LittleDict` backed by `Tuples` +(see [`freeze`](@ref)) +However, this depends on exactly how long `isequal` and `hash` take, +as well as on how many hash collisions occur etc. + +!!! note + When constructing a `LittleDict` it is faster to pass in the keys and + values each as seperate lists. So if you have them seperately already, + do `LittleDict(ks, vs)` not `LittleDict(zip(ks, vs))`. + Furthermore, key and value lists that are passed as `Tuple`s will not require any + copies to create the `LittleDict`, so `LittleDict(ks::Tuple, vs::Tuple)` + is the fastest constructor of all. +""" +struct LittleDict{K,V,KS<:StoreType,VS<:StoreType} <: AbstractDict{K, V} + keys::KS + vals::VS + + function LittleDict{K,V,KS,VS}(keys,vals) where {K,V,KS,VS} + if length(keys) != length(vals) + throw(ArgumentError( + "Number of keys ($(length(keys))) differs from " * + "number of values ($(length(vals))" + )) + end + K<:eltype(KS) || ArgumentError("Invalid store type $KS, for key type $K") + V<:eltype(VS) || ArgumentError("Invalid store type $VS, for value type $K") + + return new(keys,vals) + end +end + +function LittleDict{K,V}(ks::KS, vs::VS) where {K,V, KS<:StoreType,VS<:StoreType} + return LittleDict{K, V, KS, VS}(ks, vs) +end + +function LittleDict(ks::KS, vs::VS) where {KS<:StoreType,VS<:StoreType} + return LittleDict{eltype(KS), eltype(VS)}(ks, vs) +end + + +# Other iterators should be copied to a Vector +LittleDict(ks, vs) = LittleDict(collect(ks), collect(vs)) + + +function LittleDict{K,V}(itr) where {K,V} + ks = K[] + vs = V[] + for val in itr + if !(val isa Union{Tuple{<:Any, <:Any}, Pair}) + throw(ArgumentError( + "LittleDict(kv): kv needs to be an iterator of tuples or pairs") + ) + end + k, v = val + push!(ks, k) + push!(vs, v) + end + return LittleDict(ks, vs) +end + +LittleDict{K,V}(itr...) where {K,V} = LittleDict{K,V}(itr) +LittleDict(itr...) = LittleDict(itr) +LittleDict(itr::T) where T = LittleDict{kvtype(eltype(T))...}(itr) + +# Avoid contention between the core constructor, and the list of elements +LittleDict(itr1::Pair, itr2::Pair) = LittleDict(first.([itr1, itr2]), last.([itr1,itr2])) +LittleDict(itr1::Pair) = LittleDict([first(itr1)], [last(itr1)]) +LittleDict{K, V}(itr::Pair) where {K, V} = LittleDict{K,V}(K[first(itr)], V[last(itr)]) + +kvtype(::Any) = (Any, Any) +kvtype(::Type{Union{}}) = (Any,Any) + +kvtype(::Type{Pair{K,V}}) where {K,V} = (K,V) +kvtype(::Type{Pair{<:Any,V}}) where {V} = (Any,V) +kvtype(::Type{Pair{K,<:Any}}) where {K} = (K,Any) + +kvtype(::Type{Tuple{K,V}}) where {K,V} = (K,V) + +""" + freeze(dd::AbstractDict) + +Render a dictionary immutable by converting it to a `Tuple`-backed +`LittleDict`. +The `Tuple`-backed `LittleDict` is faster than the `Vector`-backed `LittleDict`, +particularly when the keys are all concretely typed. +""" +function freeze(dd::AbstractDict) + ks = Tuple(keys(dd)) + vs = Tuple(values(dd)) + return LittleDict(ks, vs) +end + +isordered(::Type{<:LittleDict}) = true + +# For now these are internal UnionAlls for dispatch purposes +const UnfrozenLittleDict{K,V} = LittleDict{K,V, Vector{K}, Vector{V}} +const FrozenLittleDict{K,V} = LittleDict{K,V, <:Tuple, <:Tuple} + +##### Methods that all AbstractDicts should implement + +Base.length(dd::LittleDict) = length(dd.keys) + +function Base.getkey(dd::LittleDict, key, default) + if key ∈ dd.keys + return key + else + return default + end +end + +function Base.map!(f, iter::Base.ValueIterator{<:LittleDict}) + dict = iter.dict + vals = dict.vals + for i in 1:length(vals) + @inbounds vals[i] = f(vals[i]) + end + return iter +end + +struct NotFoundSentinel end # Struct to mark not not found +function Base.get(dd::LittleDict, key, default) + @assert length(dd.keys) == length(dd.vals) + for ii in 1:length(dd.keys) + cand = @inbounds dd.keys[ii] + isequal(cand, key) && return @inbounds(dd.vals[ii]) + end + return default +end +function get(default::Base.Callable, dd::LittleDict, key) + got = get(dd, key, NotFoundSentinel()) + if got isa NotFoundSentinel # not found + return default() + else + return got + end +end + +function Base.iterate(dd::LittleDict, ii=1) + ii > length(dd.keys) && return nothing + return (dd.keys[ii] => dd.vals[ii], ii+1) +end + +function merge(d1::LittleDict, others::AbstractDict...) + return merge((x,y)->y, d1, others...) +end + +function merge( + combine::Function, + d::LittleDict, + others::AbstractDict... + ) + K,V = _merge_kvtypes(d, others...) + dc = LittleDict{K,V}(d) + for d2 in others + for (k2,v2) in d2 + got = get(dc, k2, NotFoundSentinel()) + if got isa NotFoundSentinel + add_new!(dc, k2, v2) + else + # GOLDPLATE: ideally we would avoid iterating this twice + # once for get and once for setindex! + dc[k2]=combine(got, v2) + end + end + end + return dc +end + + +Base.empty(dd::LittleDict{K,V}) where {K,V} = LittleDict{K,V}() + +######## Methods that all mutable AbstractDict's should implement + +function Base.sizehint!(dd::UnfrozenLittleDict, sz) + sizehint!(dd.keys, sz) + sizehint!(dd.vals,sz) + return dd +end + +function add_new!(dd::UnfrozenLittleDict{K, V}, key, value) where {K, V} + kk = convert(K, key) + vv = convert(V, value) + + # if we can convert it to the right type, and the dict is unfrozen + # then neither push can fail, so the dict length will remain in sync + push!(dd.keys, kk) + push!(dd.vals, vv) + + return dd +end + + +function Base.setindex!(dd::LittleDict{K,V, <:Any, <:Vector}, value, key) where {K,V} + # Note we only care if the Value store is mutable (<:Vector) + # As we can have immutable keys, if we are setting the value of an existing key + + # Assertion below commented out as by standards of carefully optimised + # setindex! it has huge code (26%), this does mean that if someone has messed + # with the fields of the LittleDict directly, then the @inbounds could be invalid + #@assert length(dd.keys) == length(dd.vals) + + kk = convert(K, key) + vv = convert(V, value) + for ii in 1:length(dd.keys) + cand = @inbounds dd.keys[ii] + if isequal(cand, kk) + @inbounds(dd.vals[ii] = vv) + return dd + end + end + add_new!(dd, key, value) + return dd +end + +function Base.pop!(dd::UnfrozenLittleDict) + pop!(dd.keys) + return pop!(dd.vals) +end + +function Base.pop!(dd::UnfrozenLittleDict, key) + @assert length(dd.keys) == length(dd.vals) + + for ii in 1:length(dd.keys) + cand = @inbounds dd.keys[ii] + if isequal(cand, key) + deleteat!(dd.keys, ii) + val = @inbounds dd.vals[ii] + deleteat!(dd.vals, ii) + return val + end + end +end + +function Base.delete!(dd::UnfrozenLittleDict, key) + pop!(dd, key) + return dd +end + +Base.empty!(dd::UnfrozenLittleDict) = (empty!(dd.keys); empty!(dd.vals); dd) + +function get!(default::Base.Callable, dd::UnfrozenLittleDict, key) + got = get(dd, key, NotFoundSentinel()) + if got isa NotFoundSentinel # not found + val = default() + add_new!(dd, key, val) + return val + else + return got + end +end +get!(dd::UnfrozenLittleDict, key, default) = get!(()->default, dd, key) diff --git a/packages/OrderedCollections/src/ordered_dict.jl b/packages/OrderedCollections/src/ordered_dict.jl new file mode 100644 index 0000000..7da2445 --- /dev/null +++ b/packages/OrderedCollections/src/ordered_dict.jl @@ -0,0 +1,491 @@ +# These can be changed, to trade off better performance for space +const global maxallowedprobe = isdefined(Base, :maxallowedprobe) ? Base.maxallowedprobe : 16 +const global maxprobeshift = isdefined(Base, :maxprobeshift) ? Base.maxprobeshift : 6 + +# OrderedDict + +""" + OrderedDict + +`OrderedDict`s are simply dictionaries whose entries have a particular order. The order +refers to insertion order, which allows deterministic iteration over the dictionary or set. +""" +mutable struct OrderedDict{K,V} <: AbstractDict{K,V} + slots::Array{Int32,1} + keys::Array{K,1} + vals::Array{V,1} + ndel::Int + maxprobe::Int + dirty::Bool + + function OrderedDict{K,V}() where {K,V} + new{K,V}(zeros(Int32,16), Vector{K}(), Vector{V}(), 0, 0, false) + end + function OrderedDict{K,V}(kv) where {K,V} + h = OrderedDict{K,V}() + for (k,v) in kv + h[k] = v + end + return h + end + OrderedDict{K,V}(p::Pair) where {K,V} = setindex!(OrderedDict{K,V}(), p.second, p.first) + function OrderedDict{K,V}(ps::Pair...) where {K,V} + h = OrderedDict{K,V}() + sizehint!(h, length(ps)) + for p in ps + h[p.first] = p.second + end + return h + end + function OrderedDict{K,V}(d::OrderedDict{K,V}) where {K,V} + if d.ndel > 0 + rehash!(d) + end + @assert d.ndel == 0 + new{K,V}(copy(d.slots), copy(d.keys), copy(d.vals), 0, d.maxprobe, false) + end +end +OrderedDict() = OrderedDict{Any,Any}() +OrderedDict(kv::Tuple{}) = OrderedDict() +copy(d::OrderedDict) = OrderedDict(d) + + +# TODO: this can probably be simplified using `eltype` as a THT (Tim Holy trait) +# OrderedDict{K,V}(kv::Tuple{Vararg{Tuple{K,V}}}) = OrderedDict{K,V}(kv) +# OrderedDict{K }(kv::Tuple{Vararg{Tuple{K,Any}}}) = OrderedDict{K,Any}(kv) +# OrderedDict{V }(kv::Tuple{Vararg{Tuple{Any,V}}}) = OrderedDict{Any,V}(kv) +OrderedDict(kv::Tuple{Vararg{Pair{K,V}}}) where {K,V} = OrderedDict{K,V}(kv) + +OrderedDict(kv::AbstractArray{Tuple{K,V}}) where {K,V} = OrderedDict{K,V}(kv) +OrderedDict(kv::AbstractArray{Pair{K,V}}) where {K,V} = OrderedDict{K,V}(kv) +OrderedDict(kv::AbstractDict{K,V}) where {K,V} = OrderedDict{K,V}(kv) + +OrderedDict(ps::Pair{K,V}...) where {K,V} = OrderedDict{K,V}(ps) +OrderedDict(ps::Pair...) = OrderedDict(ps) + +function OrderedDict(kv) + try + dict_with_eltype((K, V) -> OrderedDict{K, V}, kv, eltype(kv)) + catch e + if isempty(methods(iterate, (typeof(kv),))) || + !all(x->isa(x, Union{Tuple,Pair}), kv) + throw(ArgumentError("OrderedDict(kv): kv needs to be an iterator of tuples or pairs")) + else + rethrow(e) + end + end +end + +empty(d::OrderedDict{K,V}) where {K,V} = OrderedDict{K,V}() +empty(d::OrderedDict, ::Type{K}, ::Type{V}) where {K, V} = OrderedDict{K, V}() + +length(d::OrderedDict) = length(d.keys) - d.ndel +isempty(d::OrderedDict) = (length(d) == 0) + +""" + isordered(::Type) + +Property of associative containers, that is `true` if the container type has a +defined order (such as `OrderedDict` and `SortedDict`), and `false` otherwise. +""" +isordered(::Type{T}) where {T<:AbstractDict} = false +isordered(::Type{T}) where {T<:OrderedDict} = true + +# conversion between OrderedDict types +function convert(::Type{OrderedDict{K,V}}, d::AbstractDict) where {K,V} + if !isordered(typeof(d)) + Base.depwarn("Conversion to OrderedDict is deprecated for unordered associative containers (in this case, $(typeof(d))). Use an ordered or sorted associative type, such as SortedDict and OrderedDict.", :convert) + end + h = OrderedDict{K,V}() + for (k,v) in d + ck = convert(K,k) + if !haskey(h,ck) + h[ck] = convert(V,v) + else + error("key collision during dictionary conversion") + end + end + return h +end +convert(::Type{OrderedDict{K,V}},d::OrderedDict{K,V}) where {K,V} = d + +isslotempty(slot_value::Integer) = slot_value == 0 +isslotfilled(slot_value::Integer) = slot_value > 0 +isslotmissing(slot_value::Integer) = slot_value < 0 + +function rehash!(h::OrderedDict{K,V}, newsz = length(h.slots)) where {K,V} + olds = h.slots + keys = h.keys + vals = h.vals + sz = length(olds) + newsz = _tablesz(newsz) + h.dirty = true + count0 = length(h) + if count0 == 0 + resize!(h.slots, newsz) + fill!(h.slots, 0) + resize!(h.keys, 0) + resize!(h.vals, 0) + h.ndel = 0 + return h + end + + slots = zeros(Int32, newsz) + maxprobe = 0 + + if h.ndel > 0 + ndel0 = h.ndel + ptrs = !isbitstype(K) + to = 1 + # TODO: to get the best performance we need to avoid reallocating these. + # This algorithm actually works in place, unless the dict is modified + # due to GC during this process. + newkeys = similar(keys, count0) + newvals = similar(vals, count0) + @inbounds for from = 1:length(keys) + if !ptrs || isassigned(keys, from) + k = keys[from] + hashk = hash(k)%Int + isdeleted = false + if !ptrs + iter = 0 + index = (hashk & (sz-1)) + 1 + while iter <= h.maxprobe + si = olds[index] + si == from && break + # if we find si == 0, then the key was deleted and it's slot was reused/overwritten + (si == -from || si == 0) && (isdeleted = true; break) + index = (index & (sz-1)) + 1 + iter += 1 + end + iter > h.maxprobe && (isdeleted = true) # Another case where the slot was reused/overwritten + end + if !isdeleted + index0 = index = (hashk & (newsz-1)) + 1 + while slots[index] != 0 + index = (index & (newsz-1)) + 1 + end + probe = (index - index0) & (newsz-1) + probe > maxprobe && (maxprobe = probe) + slots[index] = to + newkeys[to] = k + newvals[to] = vals[from] + to += 1 + end + if h.ndel != ndel0 + # if items are removed by finalizers, retry + return rehash!(h, newsz) + end + end + end + h.keys = newkeys + h.vals = newvals + h.ndel = 0 + else + @inbounds for i = 1:count0 + k = keys[i] + index0 = index = hashindex(k, newsz) + while slots[index] != 0 + index = (index & (newsz-1)) + 1 + end + probe = (index - index0) & (newsz-1) + probe > maxprobe && (maxprobe = probe) + slots[index] = i + if h.ndel > 0 + # if items are removed by finalizers, retry + return rehash!(h, newsz) + end + end + end + + h.slots = slots + h.maxprobe = maxprobe + return h +end + +function sizehint!(d::OrderedDict, newsz) + slotsz = (newsz*3)>>1 + oldsz = length(d.slots) + if slotsz <= oldsz + # todo: shrink + # be careful: rehash!() assumes everything fits. it was only designed + # for growing. + return d + end + # grow at least 25% + slotsz = max(slotsz, (oldsz*5)>>2) + rehash!(d, slotsz) +end + +function empty!(h::OrderedDict{K,V}) where {K,V} + fill!(h.slots, 0) + empty!(h.keys) + empty!(h.vals) + h.ndel = 0 + h.dirty = true + return h +end + +# get the index where a key is stored, or -1 if not present +function ht_keyindex(h::OrderedDict{K,V}, key, direct) where {K,V} + slots = h.slots + sz = length(slots) + iter = 0 + maxprobe = h.maxprobe + index = hashindex(key, sz) + keys = h.keys + + @inbounds while iter <= maxprobe + si = slots[index] + isslotempty(si) && break + if isslotfilled(si) && isequal(key, keys[si]) + return ifelse(direct, oftype(index, si), index) + end + + index = (index & (sz-1)) + 1 + iter += 1 + end + + return -1 +end + +# get the index where a key is stored, or -pos if not present +# and the key would be inserted at pos +# This version is for use by setindex! and get! +function ht_keyindex2(h::OrderedDict{K,V}, key) where {K,V} + slots = h.slots + sz = length(slots) + iter = 0 + maxprobe = h.maxprobe + index = hashindex(key, sz) + keys = h.keys + avail = 0 + + @inbounds while iter <= maxprobe + si = slots[index] + if isslotempty(si) + avail < 0 && return avail + return -index + end + + if isslotmissing(si) + avail == 0 && (avail = -index) + elseif isequal(key, keys[si]) + return oftype(index, si) + end + + index = (index & (sz-1)) + 1 + iter += 1 + end + + avail < 0 && return avail + + # If key is not present, may need to keep searching to find slot + maxallowed = max(maxallowedprobe, sz>>maxprobeshift) + @inbounds while iter < maxallowed + if !isslotfilled(slots[index]) + h.maxprobe = iter + return -index + end + index = (index & (sz-1)) + 1 + iter += 1 + end + + rehash!(h, length(h) > 64000 ? sz*2 : sz*4) + + return ht_keyindex2(h, key) +end + +function _setindex!(h::OrderedDict, v, key, index) + hk, hv = h.keys, h.vals + #push!(h.keys, key) + ccall(:jl_array_grow_end, Cvoid, (Any, UInt), hk, 1) + nk = length(hk) + @inbounds hk[nk] = key + #push!(h.vals, v) + ccall(:jl_array_grow_end, Cvoid, (Any, UInt), hv, 1) + @inbounds hv[nk] = v + @inbounds h.slots[index] = nk + h.dirty = true + + sz = length(h.slots) + cnt = nk - h.ndel + # Rehash now if necessary + if h.ndel >= ((3*nk)>>2) || cnt*3 > sz*2 + # > 3/4 deleted or > 2/3 full + rehash!(h, cnt > 64000 ? cnt*2 : cnt*4) + end +end + +function setindex!(h::OrderedDict{K,V}, v0, key0) where {K,V} + key = convert(K, key0) + if !isequal(key, key0) + throw(ArgumentError("$key0 is not a valid key for type $K")) + end + v = convert(V, v0) + + index = ht_keyindex2(h, key) + + if index > 0 + @inbounds h.keys[index] = key + @inbounds h.vals[index] = v + else + _setindex!(h, v, key, -index) + end + + return h +end + +function get!(h::OrderedDict{K,V}, key0, default) where {K,V} + key = convert(K, key0) + if !isequal(key, key0) + throw(ArgumentError("$key0 is not a valid key for type $K")) + end + + index = ht_keyindex2(h, key) + + index > 0 && return h.vals[index] + + v = convert(V, default) + _setindex!(h, v, key, -index) + return v +end + +function get!(default::Base.Callable, h::OrderedDict{K,V}, key0) where {K,V} + key = convert(K, key0) + if !isequal(key, key0) + throw(ArgumentError("$key0 is not a valid key for type $K")) + end + + index = ht_keyindex2(h, key) + + index > 0 && return h.vals[index] + + h.dirty = false + v = convert(V, default()) + if h.dirty # calling default could have dirtied h + index = ht_keyindex2(h, key) + end + if index > 0 + h.keys[index] = key + h.vals[index] = v + else + _setindex!(h, v, key, -index) + end + return v +end + +function getindex(h::OrderedDict{K,V}, key) where {K,V} + index = ht_keyindex(h, key, true) + return (index<0) ? throw(KeyError(key)) : h.vals[index]::V +end + +function get(h::OrderedDict{K,V}, key, default) where {K,V} + index = ht_keyindex(h, key, true) + return (index<0) ? default : h.vals[index]::V +end + +function get(default::Base.Callable, h::OrderedDict{K,V}, key) where {K,V} + index = ht_keyindex(h, key, true) + return (index<0) ? default() : h.vals[index]::V +end + +haskey(h::OrderedDict, key) = (ht_keyindex(h, key, true) >= 0) +in(key, v::Base.KeySet{K,T}) where {K,T<:OrderedDict{K}} = (ht_keyindex(v.dict, key, true) >= 0) + +function getkey(h::OrderedDict{K,V}, key, default) where {K,V} + index = ht_keyindex(h, key, true) + return (index<0) ? default : h.keys[index]::K +end + +function _pop!(h::OrderedDict, index) + @inbounds val = h.vals[h.slots[index]] + _delete!(h, index) + return val +end + +function pop!(h::OrderedDict) + h.ndel > 0 && rehash!(h) + key = h.keys[end] + index = ht_keyindex(h, key, false) + return key => _pop!(h, index) +end + +function popfirst!(h::OrderedDict) + h.ndel > 0 && rehash!(h) + key = h.keys[1] + index = ht_keyindex(h, key, false) + key => _pop!(h, index) +end + +function pop!(h::OrderedDict, key) + index = ht_keyindex(h, key, false) + index > 0 ? _pop!(h, index) : throw(KeyError(key)) +end + +function pop!(h::OrderedDict, key, default) + index = ht_keyindex(h, key, false) + index > 0 ? _pop!(h, index) : default +end + +function _delete!(h::OrderedDict, index) + @inbounds ki = h.slots[index] + @inbounds h.slots[index] = -ki + ccall(:jl_arrayunset, Cvoid, (Any, UInt), h.keys, ki-1) + ccall(:jl_arrayunset, Cvoid, (Any, UInt), h.vals, ki-1) + h.ndel += 1 + h.dirty = true + return h +end + +function delete!(h::OrderedDict, key) + index = ht_keyindex(h, key, false) + if index > 0; _delete!(h, index); end + return h +end + +function iterate(t::OrderedDict) + t.ndel > 0 && rehash!(t) + length(t.keys) < 1 && return nothing + return (Pair(t.keys[1], t.vals[1]), 2) +end +function iterate(t::OrderedDict, i) + length(t.keys) < i && return nothing + return (Pair(t.keys[i], t.vals[i]), i+1) +end + +function _merge_kvtypes(d, others...) + K, V = keytype(d), valtype(d) + for other in others + K = promote_type(K, keytype(other)) + V = promote_type(V, valtype(other)) + end + return (K,V) +end + +function merge(d::OrderedDict, others::AbstractDict...) + K,V = _merge_kvtypes(d, others...) + merge!(OrderedDict{K,V}(), d, others...) +end + +function merge(combine::Function, d::OrderedDict, others::AbstractDict...) + K,V = _merge_kvtypes(d, others...) + merge!(combine, OrderedDict{K,V}(), d, others...) +end + +function Base.map!(f, iter::Base.ValueIterator{<:OrderedDict}) + dict = iter.dict + vals = dict.vals + elements = length(vals) - dict.ndel + elements == 0 && return iter + for i in dict.slots + if i > 0 + @inbounds vals[i] = f(vals[i]) + elements -= 1 + elements == 0 && break + end + end + return iter +end + +last(h::OrderedDict) = h.keys[end] => h.vals[end] \ No newline at end of file diff --git a/packages/OrderedCollections/src/ordered_set.jl b/packages/OrderedCollections/src/ordered_set.jl new file mode 100644 index 0000000..3d3eada --- /dev/null +++ b/packages/OrderedCollections/src/ordered_set.jl @@ -0,0 +1,99 @@ +# ordered sets + +# This was largely copied and modified from Base + + +struct OrderedSet{T} <: AbstractSet{T} + dict::OrderedDict{T,Nothing} + + OrderedSet{T}() where {T} = new{T}(OrderedDict{T,Nothing}()) + OrderedSet{T}(xs) where {T} = union!(new{T}(OrderedDict{T,Nothing}()), xs) +end +OrderedSet() = OrderedSet{Any}() +OrderedSet(xs) = OrderedSet{eltype(xs)}(xs) + + +show(io::IO, s::OrderedSet) = (show(io, typeof(s)); print(io, "("); !isempty(s) && Base.show_vector(io, s,'[',']'); print(io, ")")) + +isempty(s::OrderedSet) = isempty(s.dict) +length(s::OrderedSet) = length(s.dict) + +sizehint!(s::OrderedSet, sz::Integer) = (sizehint!(s.dict, sz); s) + +in(x, s::OrderedSet) = haskey(s.dict, x) + +push!(s::OrderedSet, x) = (s.dict[x] = nothing; s) +pop!(s::OrderedSet, x) = (pop!(s.dict, x); x) +pop!(s::OrderedSet, x, deflt) = pop!(s.dict, x, deflt) == deflt ? deflt : x +delete!(s::OrderedSet, x) = (delete!(s.dict, x); s) + +empty(s::OrderedSet{T}) where {T} = OrderedSet{T}() +copy(s::OrderedSet) = union!(empty(s), s) + +empty!(s::OrderedSet{T}) where {T} = (empty!(s.dict); s) + +emptymutable(s::OrderedSet{T}, ::Type{U}=T) where {T,U} = OrderedSet{U}() +copymutable(s::OrderedSet) = copy(s) + +# NOTE: manually optimized to take advantage of OrderedDict representation +function iterate(s::OrderedSet) + s.dict.ndel > 0 && rehash!(s.dict) + length(s.dict.keys) < 1 && return nothing + return (s.dict.keys[1], 2) +end +function iterate(s::OrderedSet, i) + length(s.dict.keys) < i && return nothing + return (s.dict.keys[i], i+1) +end + +pop!(s::OrderedSet) = pop!(s.dict)[1] +popfirst!(s::OrderedSet) = popfirst!(s.dict)[1] + + + +==(l::OrderedSet, r::OrderedSet) = (length(l) == length(r)) && (l <= r) +<(l::OrderedSet, r::OrderedSet) = (length(l) < length(r)) && (l <= r) +<=(l::OrderedSet, r::OrderedSet) = issubset(l, r) + +function filter!(f::Function, s::OrderedSet) + for x in s + if !f(x) + delete!(s, x) + end + end + return s +end + +const orderedset_seed = UInt === UInt64 ? 0x2114638a942a91a5 : 0xd86bdbf1 +function hash(s::OrderedSet, h::UInt) + h = hash(orderedset_seed, h) + s.dict.ndel > 0 && rehash!(s.dict) + hash(s.dict.keys, h) +end + + +# Deprecated functionality, see +# https://github.com/JuliaCollections/DataStructures.jl/pull/180#issuecomment-400269803 + +function getindex(s::OrderedSet, i::Int) + Base.depwarn("indexing is deprecated for OrderedSet, please rewrite your code to use iteration", :getindex) + s.dict.ndel > 0 && rehash!(s.dict) + return s.dict.keys[i] +end + +function lastindex(s::OrderedSet) + Base.depwarn("indexing is deprecated for OrderedSet, please rewrite your code to use iteration", :lastindex) + s.dict.ndel > 0 && rehash!(s.dict) + return lastindex(s.dict.keys) +end + +function nextind(::OrderedSet, i::Int) + Base.depwarn("indexing is deprecated for OrderedSet, please rewrite your code to use iteration", :lastindex) + return i + 1 # Needed on 0.7 to mimic array indexing. +end + +function keys(s::OrderedSet) + Base.depwarn("indexing is deprecated for OrderedSet, please rewrite your code to use iteration", :lastindex) + s.dict.ndel > 0 && rehash!(s.dict) + return 1:length(s) +end diff --git a/packages/OrderedCollections/test/README b/packages/OrderedCollections/test/README new file mode 100644 index 0000000..27ad8b4 --- /dev/null +++ b/packages/OrderedCollections/test/README @@ -0,0 +1,3 @@ +To run a specific test file, such as test_xxx.jl, from the command line, use + + julia runtests.jl xxx diff --git a/packages/OrderedCollections/test/runtests.jl b/packages/OrderedCollections/test/runtests.jl new file mode 100644 index 0000000..07a9879 --- /dev/null +++ b/packages/OrderedCollections/test/runtests.jl @@ -0,0 +1,25 @@ +using OrderedCollections +using Test +using Random, Serialization + +@test isempty(detect_ambiguities(Base, Core, OrderedCollections)) + +tests = [ + "little_dict", + "ordered_dict", + "ordered_set", + ] + +if length(ARGS) > 0 + tests = ARGS +end + +@testset "OrderedCollections" begin + +for t in tests + fp = joinpath(dirname(@__FILE__), "test_$t.jl") + println("$fp ...") + include(fp) +end + +end # @testset diff --git a/packages/OrderedCollections/test/test_little_dict.jl b/packages/OrderedCollections/test/test_little_dict.jl new file mode 100644 index 0000000..d6edd99 --- /dev/null +++ b/packages/OrderedCollections/test/test_little_dict.jl @@ -0,0 +1,562 @@ +using OrderedCollections, Test +using OrderedCollections: FrozenLittleDict, UnfrozenLittleDict + +@testset "LittleDict" begin + @testset "Type Aliases" begin + FF1 = LittleDict{Int,Int, NTuple{10, Int}, NTuple{10, Int}} + @test FF1 <: FrozenLittleDict{<:Any, <:Any} + @test FF1 <: FrozenLittleDict + @test FF1 <: FrozenLittleDict{Int, Int} + @test !(FF1 <: UnfrozenLittleDict{<:Any, <:Any}) + @test !(FF1 <: UnfrozenLittleDict) + @test !(FF1 <: UnfrozenLittleDict{Int, Int}) + + + UU1 = LittleDict{Int,Int,Vector{Int},Vector{Int}} + @test !(UU1 <: FrozenLittleDict{<:Any, <:Any}) + @test !(UU1 <: FrozenLittleDict) + @test !(UU1 <: FrozenLittleDict{Int, Int}) + @test (UU1 <: UnfrozenLittleDict{<:Any, <:Any}) + @test (UU1 <: UnfrozenLittleDict) + @test (UU1 <: UnfrozenLittleDict{Int, Int}) + + + FU1 = LittleDict{Int,Int,NTuple{10, Int},Vector{Int}} + @test !(FU1 <: FrozenLittleDict{<:Any, <:Any}) + @test !(FU1 <: FrozenLittleDict) + @test !(FU1 <: FrozenLittleDict{Int, Int}) + @test !(FU1 <: UnfrozenLittleDict{<:Any, <:Any}) + @test !(FU1 <: UnfrozenLittleDict) + @test !(FU1 <: UnfrozenLittleDict{Int, Int}) + + UF1 = LittleDict{Int,Int,Vector{Int},NTuple{10,Int}} + @test !(UF1 <: FrozenLittleDict{<:Any, <:Any}) + @test !(UF1 <: FrozenLittleDict) + @test !(UF1 <: FrozenLittleDict{Int, Int}) + @test !(UF1 <: UnfrozenLittleDict{<:Any, <:Any}) + @test !(UF1 <: UnfrozenLittleDict) + @test !(UF1 <: UnfrozenLittleDict{Int, Int}) + end + + @testset "Constructors" begin + @test isa(@inferred(LittleDict()), LittleDict{Any,Any}) + @test isa(@inferred(LittleDict([(1,2.0)])), LittleDict{Int,Float64}) + + @test isa(@inferred(LittleDict([("a",1),("b",2)])), LittleDict{String,Int}) + @test isa(@inferred(LittleDict(Pair(1, 1.0))), LittleDict{Int,Float64}) + @test isa(@inferred(LittleDict(Pair(1, 1.0), Pair(2, 2.0))), + LittleDict{Int,Float64}) + + @test isa(@inferred(LittleDict{Int,Float64}(2=>2.0, 3=>3.0)), + LittleDict{Int,Float64}) + @test isa(@inferred(LittleDict{Int,Float64}(Pair(1, 1), Pair(2, 2))), LittleDict{Int,Float64}) + @test isa(@inferred(LittleDict(Pair(1, 1.0), Pair(2, 2.0), Pair(3, 3.0))), LittleDict{Int,Float64}) + @test LittleDict(()) == LittleDict{Any,Any}() + + @test isa(@inferred(LittleDict([Pair(1, 1.0), Pair(2, 2.0)])), LittleDict{Int,Float64}) + @test_throws ArgumentError LittleDict([1,2,3,4]) + + iter = Iterators.filter(x->x.first>1, [Pair(1, 1.0), Pair(2, 2.0), Pair(3, 3.0)]) + @test @inferred(LittleDict(iter)) == LittleDict{Int,Float64}(2=>2.0, 3=>3.0) + + iter = Iterators.drop(1:10, 1) + @test_throws ArgumentError LittleDict(iter) + + k_iter = Iterators.filter(x->x>1, [1,2,3,4]) + v_iter = Iterators.filter(x->x>1, [1.0,2.0,3.0,4.0]) + @test @inferred(LittleDict(k_iter, v_iter)) isa + LittleDict{Int,Float64, Vector{Int}, Vector{Float64}} + + @test @inferred(LittleDict{Int, Char}(rand(1:100,20), rand('a':'z', 20))) isa + LittleDict{Int,Char,Array{Int,1},Array{Char,1}} + + # Different number of keys and values + @test_throws ArgumentError LittleDict{Int, Char, Vector{Int}, Vector{Char}}([1,2,3], ['a','b']) + end + + + @testset "empty dictionary" begin + d = LittleDict{Char, Int}() + @test length(d) == 0 + @test isempty(d) + @test_throws KeyError d['c'] == 1 + d['c'] = 1 + @test !isempty(d) + @test_throws KeyError d[0.01] + @test isempty(empty(d)) + empty!(d) + @test isempty(d) + @test delete!(d, "foo") == empty(d) # Make sure this does't throw an error + + # access, modification + for c in 'a':'z' + d[c] = c - 'a' + 1 + end + + @test (d['a'] += 1) == 2 + @test 'a' in keys(d) + @test haskey(d, 'a') + @test get(d, 'B', 0) == 0 + @test getkey(d, 'b', nothing) == 'b' + @test getkey(d, 'B', nothing) == nothing + @test !('B' in keys(d)) + @test !haskey(d, 'B') + @test pop!(d, 'a') == 2 + + @test collect(keys(d)) == collect('b':'z') + @test collect(values(d)) == collect(2:26) + @test collect(d) == [Pair(a,i) for (a,i) in zip('b':'z', 2:26)] + end + + @testset "convert" begin + d = LittleDict{Int,Float32}(i=>Float32(i) for i = 1:10) + @test convert(LittleDict{Int,Float32}, d) === d + dc = convert(LittleDict{Int,Float64}, d) + @test dc !== d + @test keytype(dc) == Int + @test valtype(dc) == Float64 + @test keys(dc) == keys(d) + @test collect(values(dc)) == collect(values(d)) + end + + @testset "Issue #60" begin + od60 = LittleDict{Int,Int}() + od60[1] = 2 + + ranges = [2:5, 6:9, 10:13] + for range in ranges + for i = range + od60[i] = i+1 + end + for i = range + delete!( od60, i ) + end + end + od60[14]=15 + + @test od60[14] == 15 + end + + + ############################## + # Copied and modified from Base/test/dict.jl + + # LittleDict + + @testset "LittleDict{Int,Int}" begin + h = LittleDict{Int,Int}() + for i=1:100 + h[i] = i+1 + end + + @test collect(h) == [Pair(x,y) for (x,y) in zip(1:100, 2:101)] + + for i=1:2:100 + delete!(h, i) + end + for i=1:2:100 + h[i] = i+1 + end + + for i=1:100 + @test h[i]==i+1 + end + + for i=1:100 + delete!(h, i) + end + @test isempty(h) + + h[77] = 100 + @test h[77]==100 + @test length(h) == 1 + + for i=1:100 + h[i] = i+1 + end + @test length(h) == 100 + + for i=1:2:50 + delete!(h, i) + end + @test length(h) == 75 + + for i=51:100 + h[i] = i+1 + end + @test length(h) == 75 + + for i=2:2:100 + @test h[i]==i+1 + end + for i=75:100 + @test h[i]==i+1 + end + end + + @testset "LittleDict{Any,Any}" begin + h = LittleDict{Any,Any}([("a", 3)]) + @test h["a"] == 3 + h["a","b"] = 4 + @test h["a","b"] == h[("a","b")] == 4 + h["a","b","c"] = 4 + @test h["a","b","c"] == h[("a","b","c")] == 4 + end + + @testset "KeyError" begin + z = LittleDict() + get_KeyError = false + try + z["a"] + catch _e123_ + get_KeyError = isa(_e123_, KeyError) + end + @test get_KeyError + end + + @testset "filter" begin + _d = LittleDict([("a", 0)]) + v = [k for k in filter(x->length(x)==1, collect(keys(_d)))] + @test isa(v, Vector{String}) + end + + @testset "from tuple/vector/pairs/tuple of pair 1" begin + d = LittleDict(((1, 2), (3, 4))) + d2 = LittleDict([(1, 2), (3, 4)]) + d3 = LittleDict(1 => 2, 3 => 4) + d4 = LittleDict((1 => 2, 3 => 4)) + + @test d[1] === 2 + @test d[3] === 4 + + @test d == d2 == d3 == d4 + @test isa(d, LittleDict{Int,Int}) + @test isa(d2, LittleDict{Int,Int}) + @test isa(d3, LittleDict{Int,Int}) + @test isa(d4, LittleDict{Int,Int}) + end + + @testset "from tuple/vector/pairs/tuple of pair 2" begin + d = LittleDict(((1, 2), (3, "b"))) + d2 = LittleDict([(1, 2), (3, "b")]) + d3 = LittleDict(1 => 2, 3 => "b") + d4 = LittleDict((1 => 2, 3 => "b")) + + @test d2[1] === 2 + @test d2[3] == "b" + + @test d == d2 == d3 == d4 + @test isa(d, LittleDict{Int,Any}) + @test isa(d2, LittleDict{Int,Any}) + @test isa(d3, LittleDict{Int,Any}) + @test isa(d4, LittleDict{Int,Any}) + end + + @testset "from tuple/vector/pairs/tuple of pair 3" begin + d = LittleDict(((1, 2), ("a", 4))) + d2 = LittleDict([(1, 2), ("a", 4)]) + d3 = LittleDict(1 => 2, "a" => 4) + d4 = LittleDict((1 => 2, "a" => 4)) + + @test d2[1] === 2 + @test d2["a"] === 4 + + ## TODO: tuple of tuples doesn't work for mixed tuple types + # @test d == d2 == d3 == d4 + @test d2 == d3 == d4 + # @test isa(d, LittleDict{Any,Int}) + @test isa(d2, LittleDict{Any,Int}) + @test isa(d3, LittleDict{Any,Int}) + @test isa(d4, LittleDict{Any,Int}) + end + + @testset "from tuple/vector/pairs/tuple of pair 4" begin + d = LittleDict(((1, 2), ("a", "b"))) + d2 = LittleDict([(1, 2), ("a", "b")]) + d3 = LittleDict(1 => 2, "a" => "b") + d4 = LittleDict((1 => 2, "a" => "b")) + + @test d[1] === 2 + @test d["a"] == "b" + + @test d == d2 == d3 == d4 + @test isa(d, LittleDict{Any,Any}) + @test isa(d2, LittleDict{Any,Any}) + @test isa(d3, LittleDict{Any,Any}) + @test isa(d4, LittleDict{Any,Any}) + end + + @testset "first" begin + @test_throws ArgumentError first(LittleDict()) + @test first(LittleDict([(:f, 2)])) == Pair(:f,2) + end + + + @testset "iterate" begin + d = LittleDict("a" => [1, 2]) + val1, state1 = iterate(d) + @test val1 == ("a" => [1, 2]) + @test iterate(d, state1) === nothing + end + + + @testset "Failing to add a value but being able to add a key (cf: Issue #1821)" begin + d = LittleDict{String, Vector{Int}}() + d["a"] = [1, 2] + @test_throws MethodError d["b"] = 1 + @test isa(repr(d), AbstractString) # check that printable without error + end + + @testset "Issue #2344" begin + bestkey(d, key) = key + bestkey(d::AbstractDict{K,V}, key) where {K<:AbstractString,V} = string(key) + bar(x) = bestkey(x, :y) + @test bar(LittleDict([(:x, [1,2,5])])) == :y + @test bar(LittleDict([("x", [1,2,5])])) == "y" + end + + @testset "isequal" begin + @test isequal(LittleDict(), LittleDict()) + @test isequal(LittleDict([(1, 1)]), LittleDict([(1, 1)])) + @test !isequal(LittleDict([(1, 1)]), LittleDict()) + @test !isequal(LittleDict([(1, 1)]), LittleDict([(1, 2)])) + @test !isequal(LittleDict([(1, 1)]), LittleDict([(2, 1)])) + + @test isequal(LittleDict(), sizehint!(LittleDict(),96)) + + # Here is what currently happens when dictionaries of different types + # are compared. This is not necessarily desirable. These tests are + # descriptive rather than proscriptive. + @test !isequal(LittleDict([(1, 2)]), LittleDict([("dog", "bone")])) + @test isequal(LittleDict{Int,Int}(), LittleDict{AbstractString,AbstractString}()) + end + + + @testset "data_in" begin + # Generate some data to populate dicts to be compared + data_in = [ (rand(1:1000), randstring(2)) for _ in 1:1001 ] + + # Populate the first dict + d1 = LittleDict{Int, String}() + for (k,v) in data_in + d1[k] = v + end + data_in = collect(d1) + # shuffle the data + for i in 1:length(data_in) + j = rand(1:length(data_in)) + data_in[i], data_in[j] = data_in[j], data_in[i] + end + # Inserting data in different (shuffled) order should result in + # equivalent dict. + d2 = LittleDict{Int, AbstractString}() + for (k,v) in data_in + d2[k] = v + end + + @test isequal(d1, d2) + d3 = copy(d2) + d4 = copy(d2) + # Removing an item gives different dict + delete!(d1, data_in[rand(1:length(data_in))][1]) + @test !isequal(d1, d2) + # Changing a value gives different dict + d3[data_in[rand(1:length(data_in))][1]] = randstring(3) + !isequal(d1, d3) + # Adding a pair gives different dict + d4[1001] = randstring(3) + @test !isequal(d1, d4) + end + + @testset "get!" begin + # get! (get with default values assigned to the given location) + f(x) = x^2 + d = LittleDict(8 => 19) + + @test get!(d, 8, 5) == 19 + @test get!(d, 19, 2) == 2 + + @test get!(d, 42) do # d is updated with f(2) + f(2) + end == 4 + + @test get!(d, 42) do # d is not updated + f(200) + end == 4 + + @test get(d, 13) do # d is not updated + f(4) + end == 16 + + @test d == LittleDict(8=>19, 19=>2, 42=>4) + end + + @testset "Issue #5886" begin + d5886 = LittleDict() + for k5886 in 1:11 + d5886[k5886] = 1 + end + for k5886 in keys(d5886) + # undefined ref if not fixed + d5886[k5886] += 1 + end + end + + @testset "isordered (Issue #216)1" begin + @test OrderedCollections.isordered(LittleDict{Int, String}) + @test !OrderedCollections.isordered(Dict{Int, String}) + end + + @testset "Test merging" begin + a = LittleDict("foo" => 0.0, "bar" => 42.0) + b = LittleDict("フー" => 17, "バー" => 4711) + result = merge(a, b) + @test isa(result, LittleDict{String,Float64}) + + expected = LittleDict("foo" => 0.0, "bar" => 42.0, "フー" => 17, "バー" => 4711) + @test result == expected + + c = LittleDict("a" => 1, "b" => 2, "c" => 3) + result = merge(a, b, c) + @test isa(result, LittleDict{String,Float64}) + + expected = LittleDict( + "foo" => 0.0, "bar" => 42.0, + "フー" => 17, "バー" => 4711, + "a" => 1, "b" => 2, "c" => 3, + ) + @test result == expected + + c = LittleDict("a" => 1, "b" => 2, "foo" => 3) + result = merge(a, b, c) + @test isa(result, LittleDict{String,Float64}) + + expected = LittleDict( + "foo" => 3, "bar" => 42.0, + "フー" => 17, "バー" => 4711, + "a" => 1, "b" => 2, + ) + @test result == expected + end + + @testset "Issue #9295" begin + d = LittleDict() + @test push!(d, 'a'=> 1) === d + @test d['a'] == 1 + @test push!(d, 'b' => 2, 'c' => 3) === d + @test d['b'] == 2 + @test d['c'] == 3 + @test push!(d, 'd' => 4, 'e' => 5, 'f' => 6) === d + @test d['d'] == 4 + @test d['e'] == 5 + @test d['f'] == 6 + @test length(d) == 6 + end + + @testset "Serialization" begin + s = IOBuffer() + od = LittleDict{Char,Int64}() + for c in 'a':'e' + od[c] = c-'a'+1 + end + serialize(s, od) + seek(s, 0) + dd = deserialize(s) + @test isa(dd, OrderedCollections.LittleDict{Char,Int64}) + @test dd == od + close(s) + end + + @testset "Issue #148" begin + d148 = LittleDict( + :gps => [], + :direction => 1:8, + :weather => 1:10 + ) + + d148_2 = LittleDict( + :time => 1:10, + :features => LittleDict( + :gps => 1:5, + :direction => 1:8, + :weather => 1:10 + ) + ) + end + + @testset "Issue #400" begin + @test filter(p->first(p) > 1, LittleDict(1=>2, 3=>4)) isa LittleDict + end + + @testset "Sorting" begin + d = LittleDict(i=>Char(123-i) for i in [4, 8, 1, 7, 9, 3, 10, 2, 6, 5]) + + @test collect(keys(d)) != 1:10 + sd = sort(d) + @test collect(keys(sd)) == 1:10 + @test collect(values(sd)) == collect('z':-1:'q') + @test sort(sd) == sd + sdv = sort(d; byvalue=true) + @test collect(keys(sdv)) == 10:-1:1 + @test collect(values(sdv)) == collect('q':'z') + end + + @testset "Test that LittleDict merge with combiner returns type LittleDict" begin + @test merge(+, LittleDict(:a=>1, :b=>2), LittleDict(:b=>7, :c=>4)) == LittleDict(:a=>1, :b=>9, :c=>4) + @test merge(+, LittleDict(:a=>1, :b=>2), Dict(:b=>7, :c=>4)) isa LittleDict + end + + @testset "issue #27" begin + d = LittleDict{Symbol, Int}(:x=>1) + d1 = LittleDict(:x=>1) + d_wide = LittleDict{Symbol, Number}(:x=>1) + @test d == d1 == d_wide + @test d isa LittleDict{Symbol, Int} + @test d1 isa LittleDict{Symbol, Int} + @test d_wide isa LittleDict{Symbol, Number} + + @test_throws MethodError LittleDict{Char,Char}(:x => 1) + end +end # @testset LittleDict + + +@testset "Frozen LittleDict" begin + + @testset "types" begin + base_dict = LittleDict((10,20,30),("a", "b", "c")) + @test base_dict isa LittleDict{Int, String, <:Tuple, <:Tuple} + + nonfrozen = LittleDict(10=>"a", 20=>"b", 30=>"c") + @test nonfrozen isa LittleDict{Int, String, <:Vector, <:Vector} + + @test base_dict == nonfrozen + + frozen = freeze(nonfrozen) + @test frozen isa LittleDict{Int, String, <:Tuple, <:Tuple} + @test frozen == base_dict + @test frozen === base_dict + end + + @testset "get" begin + fd = LittleDict((10,20,30),("a", "b", "c")) + @test fd[10] == "a" + @test fd[20] == "b" + @test fd[30] == "c" + @test_throws KeyError fd[-1] + end + + @testset "set" begin + fd = LittleDict((10,20,30),("a", "b", "c")) + @test_throws MethodError fd[10] = "ab" + @test_throws MethodError fd[20] = "bb" + @test_throws MethodError fd[30] = "cc" + @test_throws MethodError fd[-1] = "dd" + end + + @testset "map!(f, values(LittleDict))" begin + testdict = LittleDict(:a=>1, :b=>2) + map!(v->v-1, values(testdict)) + @test testdict[:a] == 0 + @test testdict[:b] == 1 +end +end diff --git a/packages/OrderedCollections/test/test_ordered_dict.jl b/packages/OrderedCollections/test/test_ordered_dict.jl new file mode 100644 index 0000000..da8de16 --- /dev/null +++ b/packages/OrderedCollections/test/test_ordered_dict.jl @@ -0,0 +1,464 @@ +using OrderedCollections, Test + +@testset "OrderedDict" begin + + @testset "Constructors" begin + @test isa(@inferred(OrderedDict()), OrderedDict{Any,Any}) + @test isa(@inferred(OrderedDict([(1,2.0)])), OrderedDict{Int,Float64}) + @test isa(@inferred(OrderedDict([("a",1),("b",2)])), OrderedDict{String,Int}) + @test isa(@inferred(OrderedDict(Pair(1, 1.0))), OrderedDict{Int,Float64}) + @test isa(@inferred(OrderedDict(Pair(1, 1.0), Pair(2, 2.0))), OrderedDict{Int,Float64}) + @test isa(@inferred(OrderedDict{Int,Float64}(Pair(1, 1), Pair(2, 2))), OrderedDict{Int,Float64}) + @test isa(@inferred(OrderedDict(Pair(1, 1.0), Pair(2, 2.0), Pair(3, 3.0))), OrderedDict{Int,Float64}) + @test OrderedDict(()) == OrderedDict{Any,Any}() + @test isa(@inferred(OrderedDict([Pair(1, 1.0), Pair(2, 2.0)])), OrderedDict{Int,Float64}) + @test_throws ArgumentError OrderedDict([1,2,3,4]) + iter = Iterators.filter(x->x.first>1, [Pair(1, 1.0), Pair(2, 2.0), Pair(3, 3.0)]) + @test @inferred(OrderedDict(iter)) == OrderedDict{Int,Float64}(2=>2.0, 3=>3.0) + iter = Iterators.drop(1:10, 1) + @test_throws ArgumentError OrderedDict(iter) + end + + @testset "empty dictionary" begin + d = OrderedDict{Char, Int}() + @test length(d) == 0 + @test isempty(d) + @test_throws KeyError d['c'] == 1 + d['c'] = 1 + @test !isempty(d) + @test_throws KeyError d[0.01] + @test isempty(empty(d)) + empty!(d) + @test isempty(d) + + # access, modification + for c in 'a':'z' + d[c] = c - 'a' + 1 + end + + @test (d['a'] += 1) == 2 + @test 'a' in keys(d) + @test haskey(d, 'a') + @test get(d, 'B', 0) == 0 + @test getkey(d, 'b', nothing) == 'b' + @test getkey(d, 'B', nothing) == nothing + @test !('B' in keys(d)) + @test !haskey(d, 'B') + @test pop!(d, 'a') == 2 + + @test collect(keys(d)) == collect('b':'z') + @test collect(values(d)) == collect(2:26) + @test collect(d) == [Pair(a,i) for (a,i) in zip('b':'z', 2:26)] + end + + @testset "convert" begin + d = OrderedDict{Int,Float32}(i=>Float32(i) for i = 1:10) + @test convert(OrderedDict{Int,Float32}, d) === d + dc = convert(OrderedDict{Int,Float64}, d) + @test dc !== d + @test keytype(dc) == Int + @test valtype(dc) == Float64 + @test keys(dc) == keys(d) + @test collect(values(dc)) == collect(values(d)) + end + + @testset "Issue #60" begin + od60 = OrderedDict{Int,Int}() + od60[1] = 2 + + ranges = [2:5, 6:9, 10:13] + for range in ranges + for i = range + od60[i] = i+1 + end + for i = range + delete!( od60, i ) + end + end + od60[14]=15 + + @test od60[14] == 15 + end + + + ############################## + # Copied and modified from Base/test/dict.jl + + # OrderedDict + + @testset "OrderedDict{Int,Int}" begin + h = OrderedDict{Int,Int}() + for i=1:10000 + h[i] = i+1 + end + + @test collect(h) == [Pair(x,y) for (x,y) in zip(1:10000, 2:10001)] + + for i=1:2:10000 + delete!(h, i) + end + for i=1:2:10000 + h[i] = i+1 + end + + for i=1:10000 + @test h[i]==i+1 + end + + for i=1:10000 + delete!(h, i) + end + @test isempty(h) + + h[77] = 100 + @test h[77]==100 + + for i=1:10000 + h[i] = i+1 + end + + for i=1:2:10000 + delete!(h, i) + end + + for i=10001:20000 + h[i] = i+1 + end + + for i=2:2:10000 + @test h[i]==i+1 + end + + for i=10000:20000 + @test h[i]==i+1 + end + end + + @testset "OrderedDict{Any,Any}" begin + h = OrderedDict{Any,Any}([("a", 3)]) + @test h["a"] == 3 + h["a","b"] = 4 + @test h["a","b"] == h[("a","b")] == 4 + h["a","b","c"] = 4 + @test h["a","b","c"] == h[("a","b","c")] == 4 + end + + @testset "KeyError" begin + z = OrderedDict() + get_KeyError = false + try + z["a"] + catch _e123_ + get_KeyError = isa(_e123_, KeyError) + end + @test get_KeyError + end + + @testset "filter" begin + _d = OrderedDict([("a", 0)]) + v = [k for k in filter(x->length(x)==1, collect(keys(_d)))] + @test isa(v, Vector{String}) + end + + @testset "from tuple/vector/pairs/tuple of pair 1" begin + d = OrderedDict(((1, 2), (3, 4))) + d2 = OrderedDict([(1, 2), (3, 4)]) + d3 = OrderedDict(1 => 2, 3 => 4) + d4 = OrderedDict((1 => 2, 3 => 4)) + + @test d[1] === 2 + @test d[3] === 4 + + @test d == d2 == d3 == d4 + @test isa(d, OrderedDict{Int,Int}) + @test isa(d2, OrderedDict{Int,Int}) + @test isa(d3, OrderedDict{Int,Int}) + @test isa(d4, OrderedDict{Int,Int}) + end + + @testset "from tuple/vector/pairs/tuple of pair 2" begin + d = OrderedDict(((1, 2), (3, "b"))) + d2 = OrderedDict([(1, 2), (3, "b")]) + d3 = OrderedDict(1 => 2, 3 => "b") + d4 = OrderedDict((1 => 2, 3 => "b")) + + @test d2[1] === 2 + @test d2[3] == "b" + + ## TODO: tuple of tuples doesn't work for mixed tuple types + # @test d == d2 == d3 == d4 + # @test isa(d, OrderedDict{Int,Any}) + @test d2 == d3 == d4 + @test isa(d2, OrderedDict{Int,Any}) + @test isa(d3, OrderedDict{Int,Any}) + @test isa(d4, OrderedDict{Int,Any}) + end + + @testset "from tuple/vector/pairs/tuple of pair 3" begin + d = OrderedDict(((1, 2), ("a", 4))) + d2 = OrderedDict([(1, 2), ("a", 4)]) + d3 = OrderedDict(1 => 2, "a" => 4) + d4 = OrderedDict((1 => 2, "a" => 4)) + + @test d2[1] === 2 + @test d2["a"] === 4 + + ## TODO: tuple of tuples doesn't work for mixed tuple types + # @test d == d2 == d3 == d4 + @test d2 == d3 == d4 + # @test isa(d, OrderedDict{Any,Int}) + @test isa(d2, OrderedDict{Any,Int}) + @test isa(d3, OrderedDict{Any,Int}) + @test isa(d4, OrderedDict{Any,Int}) + end + + @testset "from tuple/vector/pairs/tuple of pair 4" begin + d = OrderedDict(((1, 2), ("a", "b"))) + d2 = OrderedDict([(1, 2), ("a", "b")]) + d3 = OrderedDict(1 => 2, "a" => "b") + d4 = OrderedDict((1 => 2, "a" => "b")) + + @test d[1] === 2 + @test d["a"] == "b" + + @test d == d2 == d3 == d4 + @test isa(d, OrderedDict{Any,Any}) + @test isa(d2, OrderedDict{Any,Any}) + @test isa(d3, OrderedDict{Any,Any}) + @test isa(d4, OrderedDict{Any,Any}) + end + + @testset "first" begin + @test_throws ArgumentError first(OrderedDict()) + @test first(OrderedDict([(:f, 2)])) == Pair(:f,2) + end + + @testset "last" begin + @test last(OrderedDict([(:f, 2)])) == Pair(:f,2) + end + + @testset "Issue #1821" begin + d = OrderedDict{String, Vector{Int}}() + d["a"] = [1, 2] + @test_throws MethodError d["b"] = 1 + @test isa(repr(d), AbstractString) # check that printable without error + end + + @testset "Issue #2344" begin + bestkey(d, key) = key + bestkey(d::AbstractDict{K,V}, key) where {K<:AbstractString,V} = string(key) + bar(x) = bestkey(x, :y) + @test bar(OrderedDict([(:x, [1,2,5])])) == :y + @test bar(OrderedDict([("x", [1,2,5])])) == "y" + end + + @testset "isequal" begin + @test isequal(OrderedDict(), OrderedDict()) + @test isequal(OrderedDict([(1, 1)]), OrderedDict([(1, 1)])) + @test !isequal(OrderedDict([(1, 1)]), OrderedDict()) + @test !isequal(OrderedDict([(1, 1)]), OrderedDict([(1, 2)])) + @test !isequal(OrderedDict([(1, 1)]), OrderedDict([(2, 1)])) + + @test isequal(OrderedDict(), sizehint!(OrderedDict(),96)) + + # Here is what currently happens when dictionaries of different types + # are compared. This is not necessarily desirable. These tests are + # descriptive rather than proscriptive. + @test !isequal(OrderedDict([(1, 2)]), OrderedDict([("dog", "bone")])) + @test isequal(OrderedDict{Int,Int}(), OrderedDict{AbstractString,AbstractString}()) + end + + @testset "data_in" begin + # Generate some data to populate dicts to be compared + data_in = [ (rand(1:1000), randstring(2)) for _ in 1:1001 ] + + # Populate the first dict + d1 = OrderedDict{Int, String}() + for (k,v) in data_in + d1[k] = v + end + data_in = collect(d1) + # shuffle the data + for i in 1:length(data_in) + j = rand(1:length(data_in)) + data_in[i], data_in[j] = data_in[j], data_in[i] + end + # Inserting data in different (shuffled) order should result in + # equivalent dict. + d2 = OrderedDict{Int, AbstractString}() + for (k,v) in data_in + d2[k] = v + end + + @test isequal(d1, d2) + d3 = copy(d2) + d4 = copy(d2) + # Removing an item gives different dict + delete!(d1, data_in[rand(1:length(data_in))][1]) + @test !isequal(d1, d2) + # Changing a value gives different dict + d3[data_in[rand(1:length(data_in))][1]] = randstring(3) + !isequal(d1, d3) + # Adding a pair gives different dict + d4[1001] = randstring(3) + @test !isequal(d1, d4) + end + + @testset "get!" begin + # get! (get with default values assigned to the given location) + f(x) = x^2 + d = OrderedDict(8 => 19) + + @test get!(d, 8, 5) == 19 + @test get!(d, 19, 2) == 2 + + @test get!(d, 42) do # d is updated with f(2) + f(2) + end == 4 + + @test get!(d, 42) do # d is not updated + f(200) + end == 4 + + @test get(d, 13) do # d is not updated + f(4) + end == 16 + + @test d == OrderedDict(8=>19, 19=>2, 42=>4) + end + + @testset "Issue #5886" begin + d5886 = OrderedDict() + for k5886 in 1:11 + d5886[k5886] = 1 + end + for k5886 in keys(d5886) + # undefined ref if not fixed + d5886[k5886] += 1 + end + end + + @testset "Issue #216" begin + @test OrderedCollections.isordered(OrderedDict{Int, String}) + @test !OrderedCollections.isordered(Dict{Int, String}) + end + + @testset "Test merging" begin + a = OrderedDict("foo" => 0.0, "bar" => 42.0) + b = OrderedDict("フー" => 17, "バー" => 4711) + @test isa(merge(a, b), OrderedDict{String,Float64}) + end + + @testset "Issue #9295" begin + d = OrderedDict() + @test push!(d, 'a'=> 1) === d + @test d['a'] == 1 + @test push!(d, 'b' => 2, 'c' => 3) === d + @test d['b'] == 2 + @test d['c'] == 3 + @test push!(d, 'd' => 4, 'e' => 5, 'f' => 6) === d + @test d['d'] == 4 + @test d['e'] == 5 + @test d['f'] == 6 + @test length(d) == 6 + end + + @testset "Serialization" begin + s = IOBuffer() + od = OrderedDict{Char,Int64}() + for c in 'a':'e' + od[c] = c-'a'+1 + end + serialize(s, od) + seek(s, 0) + dd = deserialize(s) + @test isa(dd, OrderedCollections.OrderedDict{Char,Int64}) + @test dd == od + close(s) + end + + @testset "Issue #148" begin + d148 = OrderedDict( + :gps => [], + :direction => 1:8, + :weather => 1:10 + ) + + d148_2 = OrderedDict( + :time => 1:10, + :features => OrderedDict( + :gps => 1:5, + :direction => 1:8, + :weather => 1:10 + ) + ) + end + + @testset "Issue #400" begin + @test filter(p->first(p) > 1, OrderedDict(1=>2, 3=>4)) isa OrderedDict + end + + @testset "Issue #30" begin + d = OrderedDict(:a=>1, :b=>2) + d1 = OrderedDict(k=>v for (k,v) in d) + @test keytype(d1) == keytype(d) + @test valtype(d1) == valtype(d) + end + + @testset "Sorting" begin + d = Dict(i=>Char(123-i) for i = 1:10) + @test collect(keys(d)) != 1:10 + sd = sort!(OrderedDict(d)) + @test collect(keys(sd)) == 1:10 + @test collect(values(sd)) == collect('z':-1:'q') + @test sort(sd) == sd + sdv = sort!(OrderedDict(d); byvalue=true) + @test collect(keys(sdv)) == 10:-1:1 + @test collect(values(sdv)) == collect('q':'z') + end + + @testset "Test that OrderedDict merge with combiner returns type OrderedDict" begin + @test merge(+, OrderedDict(:a=>1, :b=>2), OrderedDict(:b=>7, :c=>4)) == OrderedDict(:a=>1, :b=>9, :c=>4) + @test merge(+, OrderedDict(:a=>1, :b=>2), Dict(:b=>7, :c=>4)) isa OrderedDict + end + + @testset "map!(f, values(OrderedDict))" begin + testdict = OrderedDict(:a=>1, :b=>2) + map!(v->v-1, values(testdict)) + @test testdict[:a] == 0 + @test testdict[:b] == 1 + end + + @testset "Issue #47" begin + @test eltype(OrderedDict(String => :string, SubString => :substring)) == Pair{Type,Symbol} + @test eltype(OrderedDict(:string => String, :substring => SubString)) == Pair{Symbol,Type} + @test eltype(OrderedDict(String => String, SubString => SubString)) == Pair{Type,Type} + + @test eltype(OrderedDict(tuple(String => :string, SubString => :substring))) == Pair{Type,Symbol} + @test eltype(OrderedDict(tuple(:string => String, :substring => SubString))) == Pair{Symbol,Type} + @test eltype(OrderedDict(tuple(String => String, SubString => SubString))) == Pair{Type,Type} + end + + @testset "Issue #71" begin + od = OrderedDict(Dict(i=>0 for i=1:158)) + sort!(od) + @test od[158] == 0 + end + + @testset "Issue #71b" begin + # This is actually a simplified version of #60, which was triggered while fixing #71 + # It doesn't actually fail on previous versions of OrderedCollections + od = OrderedDict{Int,Int}(13=>13) + delete!( od, 13 ) + od[14]=14 + @test od[14] == 14 + end + + @testset "ordered access" begin + od = OrderedDict(:a=>1, :b=>2, :c=>3) + @test popfirst!(od) == (:a => 1) + @test :a ∉ keys(od) + @test pop!(od) == (:c => 3) + @test :c ∉ keys(od) + end +end # @testset OrderedDict diff --git a/packages/OrderedCollections/test/test_ordered_set.jl b/packages/OrderedCollections/test/test_ordered_set.jl new file mode 100644 index 0000000..5dd6964 --- /dev/null +++ b/packages/OrderedCollections/test/test_ordered_set.jl @@ -0,0 +1,242 @@ +using OrderedCollections, Test + +@testset "OrderedSet" begin + + @testset "Constructors" begin + @test isa(OrderedSet(), OrderedSet{Any}) + @test isa(OrderedSet([1,2,3]), OrderedSet{Int}) + @test isa(OrderedSet{Int}([3]), OrderedSet{Int}) + data_in = (1, "banana", ()) + s = OrderedSet(data_in) + data_out = collect(s) + @test isa(data_out, Array{Any,1}) + @test tuple(data_out...) === data_in + @test tuple(data_in...) === tuple(s...) + @test length(data_out) == length(data_in) + end + + @testset "hash" begin + s1 = OrderedSet{String}(["bar", "foo"]) + s2 = OrderedSet{String}(["foo", "bar"]) + s3 = OrderedSet{String}(["baz"]) + @test hash(s1) != hash(s2) + @test hash(s1) != hash(s3) + end + + @testset "isequal" begin + @test isequal(OrderedSet(), OrderedSet()) + @test !isequal(OrderedSet(), OrderedSet([1])) + @test isequal(OrderedSet{Any}(Any[1,2]), OrderedSet{Int}([1,2])) + @test !isequal(OrderedSet{Any}(Any[1,2]), OrderedSet{Int}([1,2,3])) + + @test isequal(OrderedSet{Int}(), OrderedSet{AbstractString}()) + @test !isequal(OrderedSet{Int}(), OrderedSet{AbstractString}([""])) + @test !isequal(OrderedSet{AbstractString}(), OrderedSet{Int}([0])) + @test !isequal(OrderedSet{Int}([1]), OrderedSet{AbstractString}()) + @test isequal(OrderedSet{Any}([1,2,3]), OrderedSet{Int}([1,2,3])) + @test isequal(OrderedSet{Int}([1,2,3]), OrderedSet{Any}([1,2,3])) + @test !isequal(OrderedSet{Any}([1,2,3]), OrderedSet{Int}([1,2,3,4])) + @test !isequal(OrderedSet{Int}([1,2,3]), OrderedSet{Any}([1,2,3,4])) + @test !isequal(OrderedSet{Any}([1,2,3,4]), OrderedSet{Int}([1,2,3])) + @test !isequal(OrderedSet{Int}([1,2,3,4]), OrderedSet{Any}([1,2,3])) + end + + @testset "eltype, empty" begin + s1 = empty(OrderedSet([1,"hello"])) + @test isequal(s1, OrderedSet()) + @test eltype(s1) === Any + s2 = empty(OrderedSet{Float32}([2.0f0,3.0f0,4.0f0])) + @test isequal(s2, OrderedSet()) + @test eltype(s2) === Float32 + end + + @testset "show" begin + @test endswith(sprint(show, OrderedSet()), "OrderedSet{Any}()") + @test endswith(sprint(show, OrderedSet(['a'])), "OrderedSet{Char}(['a'])") + end + + @testset "Core Functionality" begin + s = OrderedSet(); push!(s,1); push!(s,2); push!(s,3) + @test !isempty(s) + @test in(1,s) + @test in(2,s) + @test length(s) == 3 + push!(s,1); push!(s,2); push!(s,3) + @test length(s) == 3 + @test pop!(s,1) == 1 + @test !in(1,s) + @test in(2,s) + @test length(s) == 2 + @test_throws KeyError pop!(s,1) + @test pop!(s,1,:foo) == :foo + @test length(delete!(s,2)) == 1 + @test !in(1,s) + @test !in(2,s) + @test pop!(s) == 3 + @test length(s) == 0 + @test isempty(s) + end + + @testset "copy" begin + data_in = (1,2,9,8,4) + s = OrderedSet(data_in) + c = copy(s) + @test isequal(s,c) + v = pop!(s) + @test !in(v,s) + @test in(v,c) + push!(s,100) + push!(c,200) + @test !in(100,c) + @test !in(200,s) + end + + @testset "sizehint!, empty" begin + s = OrderedSet([1]) + @test isequal(sizehint!(s, 10), OrderedSet([1])) + @test isequal(empty!(s), OrderedSet()) + # TODO: rehash + end + + @testset "iterate" begin + for data_in in ((7,8,4,5), + ("hello", 23, 2.7, (), [], (1,8))) + s = OrderedSet(data_in) + + s_new = OrderedSet() + for el in s + push!(s_new, el) + end + @test isequal(s, s_new) + + t = tuple(s...) + + @test t === data_in + @test length(t) == length(s) + for (e,f) in zip(t,s) + @test e === f + end + end + end + + @testset "union" begin + @test isequal(union(OrderedSet([1])),OrderedSet([1])) + s = ∪(OrderedSet([1,2]), OrderedSet([3,4])) + @test isequal(s, OrderedSet([1,2,3,4])) + s = union(OrderedSet([5,6,7,8]), OrderedSet([7,8,9])) + @test isequal(s, OrderedSet([5,6,7,8,9])) + s = OrderedSet([1,3,5,7]) + union!(s,(2,3,4,5)) + # TODO: order is not the same, so isequal should return false... + @test isequal(s,OrderedSet([1,2,3,4,5,7])) + end + + @testset "intersect" begin + @test isequal(intersect(OrderedSet([1])),OrderedSet([1])) + s = ∩(OrderedSet([1,2]), OrderedSet([3,4])) + @test isequal(s, OrderedSet()) + s = intersect(OrderedSet([5,6,7,8]), OrderedSet([7,8,9])) + @test isequal(s, OrderedSet([7,8])) + @test isequal(intersect(OrderedSet([2,3,1]), OrderedSet([4,2,3]), OrderedSet([5,4,3,2])), OrderedSet([2,3])) + end + + @testset "setdiff" begin + @test isequal(setdiff(OrderedSet([1,2,3]), OrderedSet()), OrderedSet([1,2,3])) + @test isequal(setdiff(OrderedSet([1,2,3]), OrderedSet([1])), OrderedSet([2,3])) + @test isequal(setdiff(OrderedSet([1,2,3]), Set([1])), OrderedSet([2,3])) + @test isequal(setdiff(OrderedSet([1,2,3]), OrderedSet([1,2])), OrderedSet([3])) + @test isequal(setdiff(OrderedSet([1,2,3]), Set([1,2])), OrderedSet([3])) + @test isequal(setdiff(OrderedSet([1,2,3]), OrderedSet([1,2,3])), OrderedSet()) + @test isequal(setdiff(OrderedSet([1,2,3]), OrderedSet([4])), OrderedSet([1,2,3])) + @test isequal(setdiff(OrderedSet([1,2,3]), OrderedSet([4,1])), OrderedSet([2,3])) + s = OrderedSet([1,3,5,7]) + setdiff!(s,(3,5)) + @test isequal(s,OrderedSet([1,7])) + s = OrderedSet([1,2,3,4]) + setdiff!(s, OrderedSet([2,4,5,6])) + @test isequal(s,OrderedSet([1,3])) + end + + @testset "ordering" begin + @test OrderedSet() < OrderedSet([1]) + @test OrderedSet([1]) < OrderedSet([1,2]) + @test !(OrderedSet([3]) < OrderedSet([1,2])) + @test !(OrderedSet([3]) > OrderedSet([1,2])) + @test OrderedSet([1,2,3]) > OrderedSet([1,2]) + @test !(OrderedSet([3]) <= OrderedSet([1,2])) + @test !(OrderedSet([3]) >= OrderedSet([1,2])) + @test OrderedSet([1]) <= OrderedSet([1,2]) + @test OrderedSet([1,2]) <= OrderedSet([1,2]) + @test OrderedSet([1,2]) >= OrderedSet([1,2]) + @test OrderedSet([1,2,3]) >= OrderedSet([1,2]) + @test !(OrderedSet([1,2,3]) >= OrderedSet([1,2,4])) + @test !(OrderedSet([1,2,3]) <= OrderedSet([1,2,4])) + end + + @testset "issubset, symdiff" begin + for (l,r) in ((OrderedSet([1,2]), OrderedSet([3,4])), + (OrderedSet([5,6,7,8]), OrderedSet([7,8,9])), + (OrderedSet([1,2]), OrderedSet([3,4])), + (OrderedSet([5,6,7,8]), OrderedSet([7,8,9])), + (OrderedSet([1,2,3]), OrderedSet()), + (OrderedSet([1,2,3]), OrderedSet([1])), + (OrderedSet([1,2,3]), OrderedSet([1,2])), + (OrderedSet([1,2,3]), OrderedSet([1,2,3])), + (OrderedSet([1,2,3]), OrderedSet([4])), + (OrderedSet([1,2,3]), OrderedSet([4,1]))) + @test issubset(intersect(l,r), l) + @test issubset(intersect(l,r), r) + @test issubset(l, union(l,r)) + @test issubset(r, union(l,r)) + @test isequal(union(intersect(l,r),symdiff(l,r)), union(l,r)) + end + @test ⊆(OrderedSet([1]), OrderedSet([1,2])) + + @test ⊊(OrderedSet([1]), OrderedSet([1,2])) + @test !⊊(OrderedSet([1]), OrderedSet([1])) + @test ⊈(OrderedSet([1]), OrderedSet([2])) + + @test symdiff(OrderedSet([1,2,3,4]), OrderedSet([2,4,5,6])) == OrderedSet([1,3,5,6]) + + @test isequal(symdiff(OrderedSet([1,2,3,4]), OrderedSet([2,4,5,6])), OrderedSet([1,3,5,6])) + + end + + @testset "filter" begin + s = OrderedSet([1,2,3,4]) + @test isequal(filter(isodd,s), OrderedSet([1,3])) + filter!(isodd, s) + @test isequal(s, OrderedSet([1,3])) + end + + @testset "first" begin + @test_throws ArgumentError first(OrderedSet()) + @test first(OrderedSet([2])) == 2 + end + + @testset "empty set" begin + d = OrderedSet{Char}() + @test length(d) == 0 + @test isempty(d) + @test !('c' in d) + push!(d, 'c') + @test !isempty(d) + empty!(d) + @test isempty(d) + end + + @testset "access, modification" begin + d = OrderedSet{Char}() + + for c in 'a':'z' + push!(d, c) + end + + for c in 'a':'z' + @test c in d + end + + @test collect(d) == collect('a':'z') + end + +end # @testset OrderedSet diff --git a/packages/Revise/.codecov.yml b/packages/Revise/.codecov.yml new file mode 100644 index 0000000..69cb760 --- /dev/null +++ b/packages/Revise/.codecov.yml @@ -0,0 +1 @@ +comment: false diff --git a/packages/Revise/.github/workflows/CompatHelper.yml b/packages/Revise/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..fa47d7d --- /dev/null +++ b/packages/Revise/.github/workflows/CompatHelper.yml @@ -0,0 +1,16 @@ +name: CompatHelper +on: + schedule: + - cron: '00 20 * * *' + workflow_dispatch: +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} + run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/packages/Revise/.github/workflows/Documenter.yml b/packages/Revise/.github/workflows/Documenter.yml new file mode 100644 index 0000000..8faa2e2 --- /dev/null +++ b/packages/Revise/.github/workflows/Documenter.yml @@ -0,0 +1,18 @@ +name: Documenter +on: + push: + branches: [master] + tags: [v*] + pull_request: + +jobs: + Documenter: + name: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/julia-buildpkg@latest + - uses: julia-actions/julia-docdeploy@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/packages/Revise/.github/workflows/TagBot.yml b/packages/Revise/.github/workflows/TagBot.yml new file mode 100644 index 0000000..f49313b --- /dev/null +++ b/packages/Revise/.github/workflows/TagBot.yml @@ -0,0 +1,15 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/packages/Revise/.github/workflows/ci.yml b/packages/Revise/.github/workflows/ci.yml new file mode 100644 index 0000000..21724a0 --- /dev/null +++ b/packages/Revise/.github/workflows/ci.yml @@ -0,0 +1,94 @@ +name: CI +on: + pull_request: + push: + branches: + - master + tags: '*' +jobs: + test: + name: Install & test Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.6' # LTS + - '1' # current stable + - 'nightly' + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x64 + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + show-versioninfo: ${{ matrix.version == 'nightly' }} + - uses: julia-actions/cache@v1 + - uses: julia-actions/julia-buildpkg@latest + # Revise's tests need significant customization + # Populate the precompile cache with an extraneous file, to catch issues like in #460 + - name: populate_compiled + if: ${{ matrix.os != 'windows-latest' }} + run: julia -e 'include(joinpath("test", "populate_compiled.jl"))' + - uses: julia-actions/julia-runtest@latest + - name: filewatching + if: ${{ matrix.os == 'ubuntu-latest' && matrix.version != '1.0' }} + run: julia --project -e 'using Pkg; Pkg.build(); Pkg.test(; test_args=["REVISE_TESTS_WATCH_FILES"], coverage=true)' + - name: extra tests + if: ${{ matrix.os != 'windows-latest' && matrix.version != '1.0' }} + run: | + echo $TERM + # Tests for when using polling + echo "Polling" + julia --project --code-coverage=user -e ' + ENV["JULIA_REVISE_POLL"]="1" + using Pkg, Revise + include(joinpath(dirname(pathof(Revise)), "..", "test", "polling.jl")) + ' + # The REPL wasn't initialized, so the "Methods at REPL" tests didn't run. Pick those up now. + echo "Methods at REPL" + TERM="xterm" julia --project --code-coverage=user -e ' + using InteractiveUtils, REPL, Revise + @async(Base.run_main_repl(true, true, false, true, false)) + while !isdefined(Base, :active_repl_backend) sleep(0.1) end + pushfirst!(Base.active_repl_backend.ast_transforms, Revise.revise_first) + include(joinpath("test", "runtests.jl")) + if Base.VERSION.major == 1 && Base.VERSION.minor >= 9 + REPL.eval_user_input(:(exit()), Base.active_repl_backend, Main) + else + REPL.eval_user_input(:(exit()), Base.active_repl_backend) + end' "Methods at REPL" + # Tests for out-of-process updates to manifest + echo "Switch version" + bash test/envs/use_exputils/setup.sh + julia --project --code-coverage=user test/envs/use_exputils/switch_version.jl + # We also need to pick up the Git tests, but for that we need to `dev` the package + echo "Git tests" + julia --code-coverage=user -e ' + using Pkg; Pkg.develop(PackageSpec(path=".")) + include(joinpath("test", "runtests.jl")) + ' "Git" + # Check #664 + echo "Test #664" + TERM="xterm" julia --startup-file=no --project test/start_late.jl + # Check #697 + echo "Test #697" + dn=$(mktemp -d) + ver=$(julia -e 'println(VERSION)') + curl -s -L https://github.com/JuliaLang/julia/archive/refs/tags/v$ver.tar.gz --output - | tar -xz -C $dn + julia --project test/juliadir.jl "$dn/julia-$ver" + + # # Running out of inotify storage (see #26) + # - name: inotify + # if: ${{ matrix.os == 'ubuntu-latest' }} + # run: echo 4 | sudo tee -a /proc/sys/fs/inotify/max_user_watches; julia --project --code-coverage=user -e 'using Pkg, Revise; cd(joinpath(dirname(pathof(Revise)), "..", "test")); include("inotify.jl")' + - uses: julia-actions/julia-processcoverage@latest + - uses: codecov/codecov-action@v3 + with: + file: lcov.info diff --git a/packages/Revise/.gitignore b/packages/Revise/.gitignore new file mode 100644 index 0000000..d943ffe --- /dev/null +++ b/packages/Revise/.gitignore @@ -0,0 +1,14 @@ +*.jl.cov +*.jl.*.cov +*.jl.mem +deps/silence.txt +deps/build.log +docs/build +docs/site +*.aux +*.log +docs/src/figures/diagram.pdf +Manifest.toml +src/aliasnewstruct.jl +.vscode +test/envs/use_exputils/Project.toml diff --git a/packages/Revise/LICENSE.md b/packages/Revise/LICENSE.md new file mode 100644 index 0000000..1ba89be --- /dev/null +++ b/packages/Revise/LICENSE.md @@ -0,0 +1,22 @@ +The Revise.jl package is licensed under the MIT "Expat" License: + +> Copyright (c) 2017: Tim Holy. +> +> Permission is hereby granted, free of charge, to any person obtaining a copy +> of this software and associated documentation files (the "Software"), to deal +> in the Software without restriction, including without limitation the rights +> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is +> furnished to do so, subject to the following conditions: +> +> The above copyright notice and this permission notice shall be included in all +> copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. +> diff --git a/packages/Revise/NEWS.md b/packages/Revise/NEWS.md new file mode 100644 index 0000000..0fe8bf2 --- /dev/null +++ b/packages/Revise/NEWS.md @@ -0,0 +1,153 @@ +# News + +This file describes only major changes, and does not include bug fixes, +cleanups, or minor enhancements. + +## Revise 3.3 + +* Upgrade to JuliaInterpreter 0.9 and drop support for Julia prior to 1.6 (the new LTS). + +## Revise 3.2 + +* Switch to synchronous processing of new packages and `@require` blocks. + This is motivated by changes in Julia designed to make code-loading threadsafe. + There are small (100-200ms) increases in latency on first use, but more guarantees that + Revise's workqueue will finish before new operations commence. + +## Revise 3.0 + +* Latencies at startup and upon first subsequent package load are greatly reduced. +* Support for selective evaluation: by default, `includet` will use a mode in which only + method definitions, not "data," are revised. By default, packages still + re-evaluate every changed expression, but packages can opt out of this behavior + by defining `__revise_mode__ = :evalmeth`. See the documentation for details. + This change should make `includet` more resistant to long latencies and other bad behavior. +* Evaluations now happen in order of dependency: if PkgA depends on PkgB, + PkgB's evaluations will occur before PkgA's. Likewise, if a package loads `"file1.jl"` before + `"file2.jl"`, `"file1.jl`"'s evaluations will be processed first. +* Duplicating a method and then deleting one copy no longer risks deleting the method from your + session--method deletion happens only when the final copy is removed. +* Error handling has been extensively reworked. Messages and stacktraces should be more consistent + with the error reporting of Julia itself. Only the first error in each file is shown. + Users are reminded of outstanding revision errors only by changing the prompt color to yellow. +* By default, Revise no longer tracks its own code or that of its dependencies. + Call `Revise.add_revise_deps()` (before making any changes) if you want Revise to track its + own code. + +## Revise 2.7 + +* Add framework for user callbacks +* Faster startup and revision, depending on Julia version + +## Revise 2.6 + +* Starting with Julia 1.5 it will be possible to run Revise with just `using Revise` + in your `startup.jl` file. Older Julia versions will still need the + backend-stealing code. + +## Revise 2.5 + +* Allow previously reported errors to be re-reported with `Revise.errors()` + +## Revise 2.4 + +* Automatic tracking of methods and included files in `@require` blocks + (needs Requires 1.0.0 or higher) + +## Revise 2.3 + +* When running code (e.g., with `includet`), execute lines that "do work" rather than + "define methods" using the compiler. The greatly improves performance in + work-intensive cases. +* When analyzing code to compute method signatures, omit expressions that don't contribute + to signatures. By skipping initialization code this leads to improved safety and + performance. +* Switch to an O(N) algorithm for renaming frame methods to match their running variants. +* Support addition and deletion of source files. +* Improve handling and printing of errors. + +## Revise 2.2 + +* Revise now warns you when the source files are not synchronized with running code. + (https://github.com/timholy/Revise.jl/issues/317) + +## Revise 2.1 + +New features: + +* Add `entr` for re-running code any time a set of dependent files and/or + packages change. + +## Revise 2.0 + +Revise 2.0 is a major rewrite with +[JuliaInterpreter](https://github.com/JuliaDebug/JuliaInterpreter.jl) +at its foundation. + +Breaking changes: + +* Most of the internal data structures have changed + +* The ability to revise code in Core.Compiler has regressed until technical + issues are resolved in JuliaInterpreter. + +* In principle, code that cannot be evaluated twice (e.g., library initialization) + could be problematic. + +New features: + +* Revise now (re)evaluates top-level code to extract method signatures. This allows + Revise to identify methods defined by code, e.g., by an `@eval` block. + Moreover, Revise can identify important changes external to the definition, e.g., + if + + ```julia + for T in (Float16, Float32, Float32) + @eval foo(::Type{$T}) = 1 + end + ``` + + gets revised to + + ```julia + for T in (Float32, Float32) + @eval foo(::Type{$T}) = 1 + end + ``` + + then Revise correctly deletes the `Float16` method of `foo`. ([#243]) + +* Revise handles all method deletions before enacting any new definitions. + As a consequence, moving methods from one file to another is more robust. + ([#243]) + +* Revise was split, with a new package + [CodeTracking](https://github.com/timholy/CodeTracking.jl) + designed to be the "query" interface for Revise. ([#245]) + +* Line numbers in method lists are corrected for moving code (requires Julia 1.2 or higher) + ([#278]) + +## Revise 1.0 (changes compared to the 0.7 branch) + +Breaking changes: + +* The internal structure has changed from using absolute paths for + individual files to a package-level organization that uses + `Base.PkgId` keys and relative paths ([#217]). + +New features: + +* Integration with Julia package manager. Revise now follows switches + from `dev`ed packages to `free`d packages, and also follows + version-upgrades of `free`d packages ([#217]). + +* Tracking code in Julia's standard libraries even for users who + download Julia binaries. Users of Rebugger will be able to step into + such methods ([#222]). + +[#217]: https://github.com/timholy/Revise.jl/pull/217 +[#222]: https://github.com/timholy/Revise.jl/pull/222 +[#243]: https://github.com/timholy/Revise.jl/pull/243 +[#245]: https://github.com/timholy/Revise.jl/pull/245 +[#278]: https://github.com/timholy/Revise.jl/pull/278 diff --git a/packages/Revise/Project.toml b/packages/Revise/Project.toml new file mode 100644 index 0000000..c6afe85 --- /dev/null +++ b/packages/Revise/Project.toml @@ -0,0 +1,44 @@ +name = "Revise" +uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" +version = "3.5.0" + +[deps] +CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +JuliaInterpreter = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" +LoweredCodeUtils = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +Requires = "ae029012-a4dd-5104-9daa-d747884805df" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[compat] +CodeTracking = "1.2" +JuliaInterpreter = "0.9" +LoweredCodeUtils = "2.3" +OrderedCollections = "1" +# Exclude Requires-1.1.0 - see https://github.com/JuliaPackaging/Requires.jl/issues/94 +Requires = "~1.0, ^1.1.1" +julia = "1.6" + +[extras] +CatIndices = "aafaddc9-749c-510e-ac4f-586e18779b91" +EndpointRanges = "340492b5-2a47-5f55-813d-aca7ddf97656" +EponymTuples = "97e2ac4a-e175-5f49-beb1-4d6866a6cdc3" +Example = "7876af07-990d-54b4-ab0e-23690620f79a" +IndirectArrays = "9b13fd28-a010-5f03-acff-a1bbcff69959" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +MappedArrays = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Requires = "ae029012-a4dd-5104-9daa-d747884805df" +RoundingIntegers = "d5f540fe-1c90-5db3-b776-2e2f362d9394" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +UnsafeArrays = "c4a57d5a-5b31-53a6-b365-19f8c011fbd6" + +[targets] +test = ["CatIndices", "EndpointRanges", "EponymTuples", "Example", "IndirectArrays", "InteractiveUtils", "MacroTools", "MappedArrays", "Random", "Requires", "RoundingIntegers", "Test", "UnsafeArrays"] diff --git a/packages/Revise/README.md b/packages/Revise/README.md new file mode 100644 index 0000000..05d21cf --- /dev/null +++ b/packages/Revise/README.md @@ -0,0 +1,31 @@ +
Revise.jl
+ +[![Build Status](https://github.com/timholy/Revise.jl/workflows/CI/badge.svg)](https://github.com/timholy/Revise.jl/actions?query=workflow%3A%22CI%22+branch%3Amaster) +[![codecov.io](http://codecov.io/github/timholy/Revise.jl/coverage.svg?branch=master)](http://codecov.io/github/timholy/Revise.jl?branch=master) + +`Revise.jl` allows you to modify code and use the changes without restarting Julia. +With Revise, you can be in the middle of a session and then update packages, switch git branches, +and/or edit the source code in the editor of your choice; any changes will typically be incorporated +into the very next command you issue from the REPL. +This can save you the overhead of restarting Julia, loading packages, and waiting for code to JIT-compile. + +See the [documentation](https://timholy.github.io/Revise.jl/stable): + +[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://timholy.github.io/Revise.jl/stable) + +In particular, most users will probably want to alter their `.julia/config/startup.jl` file +to run Revise automatically, as described in the [Configuration section](https://timholy.github.io/Revise.jl/stable/config/#Using-Revise-by-default-1) of the documentation. + +## Credits + +Revise became possible because of Jameson Nash's fix of [Julia issue 265](https://github.com/JuliaLang/julia/issues/265). +[Julia for VSCode](https://www.julia-vscode.org/) and [Juno](http://junolab.org/) are IDEs that offer an editor-based mechanism for achieving a subset of +Revise's aims. + +## Major releases + +- Both the current 3.x and 2.x release cycles use JuliaInterpreter to step through your module-defining code. +- The 1.x release cycle does not use JuliaInterpreter, but does integrate with Pkg.jl. Try this if the more recent releases give you trouble. (But please report the problems first!) +- For Julia 0.6 [see this branch](https://github.com/timholy/Revise.jl/tree/v0.6). However, you really shouldn't be using Julia 0.6 anymore! + +See the [NEWS](NEWS.md) for additional information. diff --git a/packages/Revise/docs/Project.toml b/packages/Revise/docs/Project.toml new file mode 100644 index 0000000..f2a273e --- /dev/null +++ b/packages/Revise/docs/Project.toml @@ -0,0 +1,5 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" + +[compat] +Documenter = "~0.24" diff --git a/packages/Revise/docs/make.jl b/packages/Revise/docs/make.jl new file mode 100644 index 0000000..fe6f908 --- /dev/null +++ b/packages/Revise/docs/make.jl @@ -0,0 +1,25 @@ +using Documenter, Revise + +makedocs( + modules = [Revise], + clean = false, + format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"), + sitename = "Revise.jl", + authors = "Tim Holy", + linkcheck = !("skiplinks" in ARGS), + pages = [ + "Home" => "index.md", + "config.md", + "cookbook.md", + "limitations.md", + "debugging.md", + "internals.md", + "user_reference.md", + "dev_reference.md", + ], +) + +deploydocs( + repo = "github.com/timholy/Revise.jl.git", + push_preview = true, +) diff --git a/packages/Revise/docs/src/assets/logo.png b/packages/Revise/docs/src/assets/logo.png new file mode 100644 index 0000000..350f98a Binary files /dev/null and b/packages/Revise/docs/src/assets/logo.png differ diff --git a/packages/Revise/docs/src/assets/logo.svg b/packages/Revise/docs/src/assets/logo.svg new file mode 100644 index 0000000..774f7db --- /dev/null +++ b/packages/Revise/docs/src/assets/logo.svg @@ -0,0 +1,135 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/Revise/docs/src/config.md b/packages/Revise/docs/src/config.md new file mode 100644 index 0000000..b43a2ea --- /dev/null +++ b/packages/Revise/docs/src/config.md @@ -0,0 +1,166 @@ +# Configuration + +!!! compat + These instructions are applicable only for Julia 1.5 and higher. If you are running an older version of Julia, upgrading to at least 1.6 is recommended. If you cannot upgrade, see the documentation for Revise 3.2.x or earlier. + +## Using Revise by default + +If you like Revise, you can ensure that every Julia session uses it by +launching it from your `~/.julia/config/startup.jl` file. +Note that using Revise adds a small latency at Julia startup, generally about 0.7s when you first launch Julia and another 0.25s for your first package load. +Users should weigh this penalty against whatever benefit they may derive from not having to restart their entire session. + +This can be as simple as adding + +```julia +using Revise +``` +as the first line in your `startup.jl`. If you have a Unix terminal available, simply run +```bash +mkdir -p ~/.julia/config/ && echo "using Revise" >> ~/.julia/config/startup.jl +``` + +If you use different package environments and do not always have Revise available, + +```julia +try + using Revise +catch e + @warn "Error initializing Revise" exception=(e, catch_backtrace()) +end +``` + +is recommended instead. + +### Using Revise automatically within Jupyter/IJulia + +If you want Revise to launch automatically within IJulia, then you should also create a `.julia/config/startup_ijulia.jl` file with the contents + +```julia +try + @eval using Revise +catch e + @warn "Error initializing Revise" exception=(e, catch_backtrace()) +end +``` +or simply run +```bash +mkdir -p ~/.julia/config/ && tee -a ~/.julia/config/startup_ijulia.jl << END +try + @eval using Revise +catch e + @warn "Error initializing Revise" exception=(e, catch_backtrace()) +end +END +``` + +## Configuring the revise mode + +By default, in packages all changes are tracked, but with `includet` only method definitions are tracked. +This behavior can be overridden by defining a variable `__revise_mode__` in the module(s) containing +your methods and/or data. `__revise_mode__` must be a `Symbol` taking one of the following values: + +- `:eval`: evaluate everything (the default for packages) +- `:evalmeth`: evaluate changes to method definitions (the default for `includet`) + This should work even for quite complicated method definitions, such as those that might + be made within a `for`-loop and `@eval` block. +- `:evalassign`: evaluate method definitions and assignment statements. A top-level expression + `a = Int[]` would be evaluated, but `push!(a, 1)` would not because the latter is not an assignment. +- `:sigs`: do not implement any changes, only scan method definitions for their signatures so that + their location can be updated as changes to the file(s) are made. + +If you're using `includet` from the REPL, you can enter `__revise_mode__ = :eval` to set +it throughout `Main`. `__revise_mode__` can be set independently in each module. + +## Optional global configuration + +Revise can be configured by setting environment variables. These variables have to be +set before you execute `using Revise`, because these environment variables are parsed +only during execution of Revise's `__init__` function. + +There are several ways to set these environment variables: + +- If you are [Using Revise by default](@ref) then you can include statements like + `ENV["JULIA_REVISE"] = "manual"` in your `.julia/config/startup.jl` file prior to + the line containing `using Revise`. +- On Unix systems, you can set variables in your shell initialization script + (e.g., put lines like `export JULIA_REVISE=manual` in your + [`.bashrc` file](http://www.linuxfromscratch.org/blfs/view/svn/postlfs/profile.html) + if you use `bash`). +- On Unix systems, you can launch Julia from the Unix prompt as `$ JULIA_REVISE=manual julia` + to set options for just that session. + +The function of specific environment variables is described below. + +### Manual revision: JULIA_REVISE + +By default, Revise processes any modified source files every time you enter +a command at the REPL. +However, there might be times where you'd prefer to exert manual control over +the timing of revisions. `Revise` looks for an environment variable +`JULIA_REVISE`, and if it is set to anything other than `"auto"` it +will require that you manually call `revise()` to update code. + +### User scripts: JULIA\_REVISE\_INCLUDE + +By default, `Revise` only tracks files that have been required as a consequence of +a `using` or `import` statement; files loaded by `include` are not +tracked, unless you explicitly use `includet` or `Revise.track(filename)`. However, you can turn on +automatic tracking by setting the environment variable `JULIA_REVISE_INCLUDE` to the +string `"1"` (e.g., `JULIA_REVISE_INCLUDE=1` in a bash script). + +!!! note + Most users should avoid setting `JULIA_REVISE_INCLUDE`. + Try `includet` instead. + +## Configurations for fixing errors + +### No space left on device + +!!! note + This applies only to Linux + +Revise needs to be notified by your filesystem about changes to your code, +which means that the files that define your modules need to be watched for updates. +Some systems impose limits on the number of files and directories that can be +watched simultaneously; if this limit is hit, on Linux this can result in a fairly cryptic +error like + +```sh +ERROR: start_watching (File Monitor): no space left on device (ENOSPC) +``` + +The cure is to increase the number of files that can be watched, by executing + +```sh +echo 65536 | sudo tee -a /proc/sys/fs/inotify/max_user_watches +``` + +at the Linux prompt. (The maximum value is 524288, +which will allocate half a gigabyte of RAM to file-watching). +For more information see [issue #26](https://github.com/timholy/Revise.jl/issues/26). + +Changing the value this way may not last through the next reboot, +but [you can also change it permanently](https://askubuntu.com/questions/716431/inotify-max-user-watches-value-resets-on-reboot-how-to-change-it-permanently). + +### Polling and NFS-mounted code directories: JULIA\_REVISE\_POLL + +!!! note + This applies only to Unix systems with code on network-mounted drives + +`Revise` works by monitoring your filesystem for changes to the files that define your code. +On most operating systems, Revise can work "passively" and wait to be signaled +that one or more watched directories has changed. + +Unfortunately, a few file systems (notably, the Unix-based Network File System NFS) don't support this approach. In such cases, Revise needs to "actively" check each file periodically to see whether it has changed since the last check. This active process is called [polling](https://en.wikipedia.org/wiki/Polling_(computer_science)). +You turn on polling by setting the environment variable `JULIA_REVISE_POLL` to the +string `"1"` (e.g., `JULIA_REVISE_POLL=1` in a bash script). + +!!! warning + If you're using polling, you may have to wait several seconds before changes take effect. + Polling is *not* recommended unless you have no other alternative. + +!!! note + NFS stands for [Network File System](https://en.wikipedia.org/wiki/Network_File_System) and is typically only used to mount shared network drives on *Unix* file systems. + Despite similarities in the acronym, NTFS, the standard [filesystem on Windows](https://en.wikipedia.org/wiki/NTFS), is completely different from NFS; Revise's default configuration should work fine on Windows without polling. + However, WSL2 users currently need polling due to [this bug](https://github.com/JuliaLang/julia/issues/37029). diff --git a/packages/Revise/docs/src/cookbook.md b/packages/Revise/docs/src/cookbook.md new file mode 100644 index 0000000..5b468b1 --- /dev/null +++ b/packages/Revise/docs/src/cookbook.md @@ -0,0 +1,213 @@ +# Revise usage: a cookbook + +## Package-centric usage + +For code that might be useful more than once, it's often a good idea to put it in +a package. +Revise cooperates with the package manager to enforce its distinction between +["versioned" and "under development" packages](https://julialang.github.io/Pkg.jl/v1/managing-packages/); +packages that you want to modify and have tracked by `Revise` should be `dev`ed rather than `add`ed. + +!!! note + You should never modify package files in your `.julia/packages` directory, + because this breaks the "contract" that such package files correspond to registered versions of the code. + In recent versions of Julia, the source files in `.julia/packages` are read-only, + and you should leave them this way. + + In keeping with this spirit, Revise is designed to avoid tracking changes in such files. + The correct way to make and track modifications is to `dev` the package. + +For creating packages, the author recommends [PkgTemplates.jl](https://github.com/invenia/PkgTemplates.jl). +A fallback is to use "plain" `Pkg` commands. +Both options are described below. + +### PkgTemplates + +!!! note + Because PkgTemplates integrates nicely with [`git`](https://git-scm.com/), + this approach might require you to do some configuration. + (Once you get things set up, you shouldn't have to do this part ever again.) + PkgTemplates needs you to configure your `git` user name and email. + Some instructions on configuration are [here](https://docs.github.com/en/github/getting-started-with-github/set-up-git) + and [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup). + It's also helpful to sign up for a [GitHub account](https://github.com/) + and set git's `github.user` variable. + The [PkgTemplates documentation](https://invenia.github.io/PkgTemplates.jl/stable/) + may also be useful. + + If you struggle with this part, consider trying the "plain" `Pkg` variant below. + +!!! note + If the current directory in your Julia session is itself a package folder, PkgTemplates + will use it as the parent environment (project) for your new package. + To reduce confusion, before trying the commands below it may help to first ensure you're in a + a "neutral" directory, for example by typing `cd()` at the Julia prompt. + +Let's create a new package, `MyPkg`, to play with. + +```julia +julia> using PkgTemplates + +julia> t = Template() +Template: + → User: timholy + → Host: github.com + → License: MIT (Tim Holy 2019) + → Package directory: ~/.julia/dev + → Minimum Julia version: v1.0 + → SSH remote: No + → Add packages to main environment: Yes + → Commit Manifest.toml: No + → Plugins: None + +julia> t("MyPkg") +Generating project MyPkg: + /home/tim/.julia/dev/MyPkg/Project.toml + /home/tim/.julia/dev/MyPkg/src/MyPkg.jl +[lots more output suppressed] +``` + +In the first few lines you can see the location of your new package, here +the directory `/home/tim/.julia/dev/MyPkg`. + +Before doing anything else, let's try it out: + +```julia +julia> using Revise # you must do this before loading any revisable packages + +julia> using MyPkg +[ Info: Precompiling MyPkg [102b5b08-597c-4d40-b98a-e9249f4d01f4] + +julia> MyPkg.greet() +Hello World! +``` + +(It's perfectly fine if you see a different string of digits and letters after the "Precompiling MyPkg" message.) +You'll note that Julia found your package without you having to take any extra steps. + +*Without* quitting this Julia session, open the `MyPkg.jl` file in an editor. +You might be able to open it with + +```julia +julia> edit(pathof(MyPkg)) +``` + +although that might require [configuring your EDITOR environment variable](https://askubuntu.com/questions/432524/how-do-i-find-and-set-my-editor-environment-variable). + +You should see something like this: + +```julia +module MyPkg + +greet() = print("Hello World!") + +end # module +``` + +This is the basic package created by PkgTemplates. Let's modify `greet` to return +a different message: + +```julia +module MyPkg + +greet() = print("Hello, revised World!") + +end # module +``` + +Now go back to that same Julia session, and try calling `greet` again. +After a pause (while Revise's internal code compiles), you should see + +```julia +julia> MyPkg.greet() +Hello, revised World! +``` + +From this point forward, revisions should be fast. You can modify `MyPkg.jl` +quite extensively without quitting the Julia session, although there are some [Limitations](@ref). + + +### Using Pkg + +[Pkg](https://julialang.github.io/Pkg.jl/v1/) works similarly to `PkgTemplates`, +but requires less configuration while also doing less on your behalf. +Let's create a blank `MyPkg` using `Pkg`. (If you tried the `PkgTemplates` version +above, you might first have to delete the package with `Pkg.rm("MyPkg")` following by +a complete removal from your `dev` directory.) + +```julia +julia> using Pkg + +julia> cd(Pkg.devdir()) # take us to the standard "development directory" + +(v1.2) pkg> generate MyPkg +Generating project MyPkg: + MyPkg/Project.toml + MyPkg/src/MyPkg.jl + +(v1.2) pkg> dev MyPkg +[ Info: resolving package identifier `MyPkg` as a directory at `~/.julia/dev/MyPkg`. +... +``` + +For the line starting `(v1.2) pkg>`, hit the `]` key at the beginning of the line, +then type `generate MyPkg`. +The next line, `dev MyPkg`, is necessary to tell `Pkg` about the existence of this new package. + +Now you can do the following: +```julia +julia> using MyPkg +[ Info: Precompiling MyPkg [efe7ebfe-4313-4388-9b6c-3590daf47143] + +julia> edit(pathof(MyPkg)) +``` +and the rest should be similar to what's above under `PkgTemplates`. +Note that with this approach, `MyPkg` has not been set up for version +control. + +!!! note + If you `add` instead of `dev` the package, the package manager will make a copy of the `MyPkg` files in your `.julia/packages` directory. + This will be the "official" version of the files, and Revise will not track changes. + + +## `includet` usage + +The alternative to creating packages is to manually load individual source files. +This approach is intended for early stages of development; +if you want to track multiple files and/or have some files include other files, +you should consider switching to the package style above. + +Open your editor and create a file like this: + +```julia +mygreeting() = "Hello, world!" +``` + +Save it as `mygreet.jl` in some directory. Here we will assume it's being saved in `/tmp/`. + +Now load the code with `includet`, which stands for "include and track": + +```julia +julia> using Revise + +julia> includet("/tmp/mygreet.jl") + +julia> mygreeting() +"Hello, world!" +``` + +Now, in your editor modify `mygreeting` to do this: + +```julia +mygreeting() = "Hello, revised world!" +``` + +and then try it in the same session: + +```julia +julia> mygreeting() +"Hello, revised world!" +``` + +As described above, the first revision you make may be very slow, but later revisions +should be fast. diff --git a/packages/Revise/docs/src/debugging.md b/packages/Revise/docs/src/debugging.md new file mode 100644 index 0000000..93dcc9c --- /dev/null +++ b/packages/Revise/docs/src/debugging.md @@ -0,0 +1,226 @@ +# Debugging Revise + +## Handling errors + +Revise attempts to make error reports mimic Julia's own stacktraces, and as a consequence it has +to prevent stacktraces from containing lots of lines pointing to Revise's own code. +If you're trying to debug a Revise error, you'd probably prefer to see the entire stacktrace. +You can uncomment the obvious commented-out line in [`Revise.trim_toplevel!`](@ref). + +## The logging framework + +If Revise isn't behaving the way you expect it to, it can be useful to examine the +decisions it made. +Revise supports Julia's [Logging framework](https://docs.julialang.org/en/v1/stdlib/Logging/) +and can optionally record its decisions in a format suitable for later inspection. +What follows is a simple series of steps you can use to turn on logging, capture messages, +and then submit them with a bug report. +Alternatively, more advanced developers may want to examine the logs themselves to determine +the source of Revise's error, and for such users a few tips about interpreting the log +messages are also provided below. + +### Turning on logging + +Currently, the best way to turn on logging is within a running Julia session: + +```jldoctest; setup=(using Revise) +julia> rlogger = Revise.debug_logger() +Revise.ReviseLogger(Revise.LogRecord[], Debug) +``` +You'll use `rlogger` at the end to retrieve the logs. + +Now carry out the series of julia commands and code edits that reproduces the problem. + +### Capturing the logs and submitting them with your bug report + +Once all the revisions have been triggered and the mistake has been reproduced, +it's time to capture the logs. +To capture all the logs, use + +```julia +julia> using Base.CoreLogging: Debug + +julia> logs = filter(r->r.level==Debug, rlogger.logs); +``` + +You can capture just the changes that Revise made to running code with + +```julia +julia> logs = Revise.actions(rlogger) +``` + +You can either let these print to the console and copy/paste the text output into the +issue, or if they are extensive you can save `logs` to a file: + +```julia +open("/tmp/revise.logs", "w") do io + for log in logs + println(io, log) + end +end +``` + +Then you can upload the logs somewhere (e.g., https://gist.github.com/) and link the url in your bug report. +To assist in the resolution of the bug, please also specify additional relevant information such as the name of the function that was misbehaving after revision and/or any error messages that your received. + +See also [A complete debugging demo](@ref) below. + +### Logging by default + +If you suspect a bug in Revise but have difficulty isolating it, you can include the lines + +```julia + # Turn on logging + Revise.debug_logger() +``` + +within the `Revise` block of your `~/.julia/config/startup.jl` file. +This will ensure that you always log Revise's actions. +Then carry out your normal Julia development. +If a Revise-related problem arises, executing these lines + +```julia +rlogger = Revise.debug_logger() +using Base.CoreLogging: Debug +logs = filter(r->r.level==Debug, rlogger.logs) +open("/tmp/revise.logs", "w") do io + for log in logs + println(io, log) + end +end +``` + +within the same session will generate the `/tmp/revise.logs` file that +you can submit with your bug report. +(What makes this possible is that a second call to `Revise.debug_logger()` returns +the same logger object created by the first call--it is not necessary to hold +on to `rlogger`.) + +### The structure of the logs + +For those who want to do a little investigating on their own, it may be helpful to +know that Revise's core decisions are captured in the group called "Action," and they come in three +flavors: + +- log entries with message `"Eval"` signify a call to `eval`; for these events, + keyword `:deltainfo` has value `(mod, expr)` where `mod` is the module of evaluation + and `expr` is a [`Revise.RelocatableExpr`](@ref) containing the expression + that was evaluated. +- log entries with message `"DeleteMethod"` signify a method deletion; for these events, + keyword `:deltainfo` has value `(sigt, methsummary)` where `sigt` is the signature of the + method that Revise *intended* to delete and `methsummary` is a [`MethodSummary`](@ref) of the + method that Revise actually found to delete. +- log entries with message `"LineOffset"` correspond to updates to Revise's own internal + estimates of how far a given method has become displaced from the line number it + occupied when it was last evaluated. For these events, `:deltainfo` has value + `(sigt, newlineno, oldoffset=>newoffset)`. + +If you're debugging mistakes in method creation/deletion, the `"LineOffset"` events +may be distracting; by default [`Revise.actions`](@ref) excludes these events. + +Note that Revise records the time of each revision, which can sometimes be useful in +determining which revisions occur in conjunction with which user actions. +If you want to make use of this, it can be handy to capture the start time with `tstart = time()` +before commencing on a session. + +See [`Revise.debug_logger`](@ref) for information on groups besides "Action." + + +### A complete debugging demo + +From within Revise's `test/` directory, try the following: + +```julia +julia> rlogger = Revise.debug_logger(); + +shell> cp revisetest.jl /tmp/ + +julia> includet("/tmp/revisetest.jl") + +julia> ReviseTest.cube(3) +81 + +shell> cp revisetest_revised.jl /tmp/revisetest.jl + +julia> ReviseTest.cube(3) +27 + +julia> rlogger.logs +julia> rlogger.logs +9-element Array{Revise.LogRecord,1}: + Revise.LogRecord(Debug, DeleteMethod, Action, Revise_4ac0f476, "/home/tim/.julia/dev/Revise/src/Revise.jl", 226, (time=1.557996459055345e9, deltainfo=(Tuple{typeof(Main.ReviseTest.cube),Any}, MethodSummary(:cube, :ReviseTest, Symbol("/tmp/revisetest.jl"), 7, Tuple{typeof(Main.ReviseTest.cube),Any})))) + Revise.LogRecord(Debug, DeleteMethod, Action, Revise_4ac0f476, "/home/tim/.julia/dev/Revise/src/Revise.jl", 226, (time=1.557996459167895e9, deltainfo=(Tuple{typeof(Main.ReviseTest.Internal.mult3),Any}, MethodSummary(:mult3, :Internal, Symbol("/tmp/revisetest.jl"), 12, Tuple{typeof(Main.ReviseTest.Internal.mult3),Any})))) + Revise.LogRecord(Debug, DeleteMethod, Action, Revise_4ac0f476, "/home/tim/.julia/dev/Revise/src/Revise.jl", 226, (time=1.557996459167956e9, deltainfo=(Tuple{typeof(Main.ReviseTest.Internal.mult4),Any}, MethodSummary(:mult4, :Internal, Symbol("/tmp/revisetest.jl"), 13, Tuple{typeof(Main.ReviseTest.Internal.mult4),Any})))) + Revise.LogRecord(Debug, Eval, Action, Revise_9147188b, "/home/tim/.julia/dev/Revise/src/Revise.jl", 276, (time=1.557996459259605e9, deltainfo=(Main.ReviseTest, :(cube(x) = begin + #= /tmp/revisetest.jl:7 =# + x ^ 3 + end)))) + Revise.LogRecord(Debug, Eval, Action, Revise_9147188b, "/home/tim/.julia/dev/Revise/src/Revise.jl", 276, (time=1.557996459330512e9, deltainfo=(Main.ReviseTest, :(fourth(x) = begin + #= /tmp/revisetest.jl:9 =# + x ^ 4 + end)))) + Revise.LogRecord(Debug, LineOffset, Action, Revise_fb38a7f7, "/home/tim/.julia/dev/Revise/src/Revise.jl", 296, (time=1.557996459331061e9, deltainfo=(Any[Tuple{typeof(mult2),Any}], :(#= /tmp/revisetest.jl:11 =#) => :(#= /tmp/revisetest.jl:13 =#)))) + Revise.LogRecord(Debug, Eval, Action, Revise_9147188b, "/home/tim/.julia/dev/Revise/src/Revise.jl", 276, (time=1.557996459391182e9, deltainfo=(Main.ReviseTest.Internal, :(mult3(x) = begin + #= /tmp/revisetest.jl:14 =# + 3x + end)))) + Revise.LogRecord(Debug, LineOffset, Action, Revise_fb38a7f7, "/home/tim/.julia/dev/Revise/src/Revise.jl", 296, (time=1.557996459391642e9, deltainfo=(Any[Tuple{typeof(unchanged),Any}], :(#= /tmp/revisetest.jl:18 =#) => :(#= /tmp/revisetest.jl:19 =#)))) + Revise.LogRecord(Debug, LineOffset, Action, Revise_fb38a7f7, "/home/tim/.julia/dev/Revise/src/Revise.jl", 296, (time=1.557996459391695e9, deltainfo=(Any[Tuple{typeof(unchanged2),Any}], :(#= /tmp/revisetest.jl:20 =#) => :(#= /tmp/revisetest.jl:21 =#)))) +``` + +You can see that Revise started by deleting three methods, followed by evaluating three new versions of those methods. Interspersed are various changes to the line numbering. + +In rare cases it might be helpful to independently record the sequence of edits to the file. +You can make copies `cp editedfile.jl > /tmp/version1.jl`, edit code, `cp editedfile.jl > /tmp/version2.jl`, +etc. +`diff version1.jl version2.jl` can be used to capture a compact summary of the changes +and pasted into the bug report. + +## Debugging problems with paths + +During certain types of usage you might receive messages like + +```julia +Warning: /some/system/path/stdlib/v1.0/SHA/src is not an existing directory, Revise is not watching +``` + +Unless you've just deleted that directory, this indicates that some of Revise's functionality is broken. + +In the majority of cases, failures come down to Revise having trouble locating source +code on your drive. +This problem should be fixable, because Revise includes functionality +to update its links to source files, as long as it knows what to do. + +One of the best approaches is to run Revise's own tests via `pkg> test Revise`. +Here are some possible test warnings and errors, and steps you might take to fix them: + +- `Base & stdlib file paths: Test Failed at /some/path... Expression: isfile(Revise.basesrccache)` + This failure is quite serious, and indicates that you will be unable to access code in `Base`. + To fix this, look for a file called `"base.cache"` somewhere in your Julia install + or build directory (for the author, it is at `/home/tim/src/julia-1.0/usr/share/julia/base.cache`). + Now compare this with the value of `Revise.basesrccache`. + (If you're getting this failure, presumably they are different.) + An important "top level" directory is `Sys.BINDIR`; if they differ already at this level, + consider adding a symbolic link from the location pointed at by `Sys.BINDIR` to the + corresponding top-level directory in your actual Julia installation. + You'll know you've succeeded in specifying it correctly when, after restarting + Julia, `Revise.basesrccache` points to the correct file and `Revise.juliadir` + points to the directory that contains `base/`. + If this workaround is not possible or does not succeed, please + [file an issue](https://github.com/timholy/Revise.jl/issues) with a description of + why you can't use it and/or + + details from `versioninfo` and information about how you obtained your Julia installation; + + the values of `Revise.basesrccache` and `Revise.juliadir`, and the actual paths to `base.cache` + and the directory containing the running Julia's `base/`; + + what you attempted when trying to fix the problem; + + if possible, your best understanding of why this failed to fix it. +- `skipping Core.Compiler tests due to lack of git repo`: this likely indicates + that you downloaded a Julia binary rather than building Julia from source. + While Revise should be able to access the code in `Base` and standard libraries, + at the current time it is not possible for Revise to access julia's Core.Compiler module + unless you clone Julia's repository and build it from source. +- `skipping git tests because Revise is not under development`: this warning should be + harmless. Revise has built-in functionality for extracting source code using `git`, + and it uses itself (i.e., its own git repository) for testing purposes. + These tests run only if you have checked out Revise for development (`pkg> dev Revise`) + or on the continuous integration servers (Travis and Appveyor). diff --git a/packages/Revise/docs/src/dev_reference.md b/packages/Revise/docs/src/dev_reference.md new file mode 100644 index 0000000..82c36c0 --- /dev/null +++ b/packages/Revise/docs/src/dev_reference.md @@ -0,0 +1,166 @@ +# Developer reference + +## Internal global variables + +### Configuration-related variables + +These are set during execution of Revise's `__init__` function. + +```@docs +Revise.watching_files +Revise.polling_files +Revise.tracking_Main_includes +``` + +### Path-related variables + +```@docs +Revise.juliadir +Revise.basesrccache +Revise.basebuilddir +``` + +### Internal state management + +```@docs +Revise.pkgdatas +Revise.watched_files +Revise.revision_queue +Revise.NOPACKAGE +Revise.queue_errors +Revise.included_files +``` + +The following are specific to user callbacks (see [`Revise.add_callback`](@ref)) and +the implementation of [`entr`](@ref): + +```@docs +Revise.revision_event +Revise.user_callbacks_queue +Revise.user_callbacks_by_file +Revise.user_callbacks_by_key +``` + +## Types + +```@docs +Revise.RelocatableExpr +Revise.ModuleExprsSigs +Revise.FileInfo +Revise.PkgData +Revise.WatchList +Revise.TaskThunk +Revise.ReviseEvalException +MethodSummary +``` + +## Function reference + +### Functions called when you load a new package + +```@docs +Revise.watch_package +Revise.parse_pkg_files +Revise.init_watching +``` + +### Monitoring for changes + +These functions get called on each directory or file that you monitor for revisions. +These block execution until the file(s) are updated, so you should only call them from +within an `@async` block. +They work recursively: once an update has been detected and execution resumes, +they schedule a revision (see [`Revise.revision_queue`](@ref)) and +then call themselves on the same directory or file to wait for the next set of changes. + +```@docs +Revise.revise_dir_queued +Revise.revise_file_queued +``` + +The following functions support user callbacks, and are used in the implementation of `entr` +but can be used more broadly: + +```@docs +Revise.add_callback +Revise.remove_callback +``` + +### Evaluating changes (revising) and computing diffs + +[`revise`](@ref) is the primary entry point for implementing changes. Additionally, + +```@docs +Revise.revise_file_now +``` + +### Caching the definition of methods + +```@docs +Revise.get_def +``` + +### Parsing source code + +```@docs +Revise.parse_source +Revise.parse_source! +``` + +### Lowered source code + +Much of the "brains" of Revise comes from doing analysis on lowered code. +This part of the package is not as well documented. + +```@docs +Revise.minimal_evaluation! +Revise.methods_by_execution! +Revise.CodeTrackingMethodInfo +``` + +### Modules and paths + +```@docs +Revise.modulefiles +``` + +### Handling errors + +```@docs +Revise.trim_toplevel! +``` + +In current releases of Julia, hitting Ctrl-C from the REPL can stop tasks running in the background. +This risks stopping Revise's ability to watch for changes in files and directories. +Revise has a work-around for this problem. + +```@docs +Revise.throwto_repl +``` + +### Git integration + +```@docs +Revise.git_source +Revise.git_files +Revise.git_repo +``` + +### Distributed computing + +```@docs +Revise.init_worker +``` + +## Teaching Revise about non-julia source codes +Revise can be made to work for transpilers from non-Julia languages to Julia with a little effort. +For example, if you wrote a transpiler from C to Julia, you can define a `struct CFile` +which overrides enough of the common `String` methods (`abspath`,`isabspath`, `joinpath`, `normpath`,`isfile`,`findfirst`, and `String`), +it will be supported by Revise if you define a method like +``` +function Revise.parse_source!(mod_exprs_sigs::Revise.ModuleExprsSigs, file::CFile, mod::Module; kwargs...) + ex = # julia Expr returned from running transpiler + Revise.process_source!(mod_exprs_sigs, ex, file, mod; kwargs...) +end + +``` diff --git a/packages/Revise/docs/src/figures/diagram.png b/packages/Revise/docs/src/figures/diagram.png new file mode 100644 index 0000000..b48d344 Binary files /dev/null and b/packages/Revise/docs/src/figures/diagram.png differ diff --git a/packages/Revise/docs/src/figures/diagram.tex b/packages/Revise/docs/src/figures/diagram.tex new file mode 100644 index 0000000..fe5d4a5 --- /dev/null +++ b/packages/Revise/docs/src/figures/diagram.tex @@ -0,0 +1,52 @@ +% compile with +% pdflatex -shell-escape diagram + +\documentclass{article} +\usepackage[paperheight=6cm,paperwidth=12cm,margin=0cm]{geometry} +\usepackage{tikz} +\usetikzlibrary{calc} + +\pagestyle{empty} + +\begin{document} + + \begin{tikzpicture}[ + font=\sffamily, + every matrix/.style={ampersand replacement=\&,column sep=1cm,row sep=1cm}, + object/.style={draw,thick,rounded corners,fill=blue!20}, + to/.style={->,shorten >=1pt,semithick,font=\sffamily\footnotesize}, + ] + % every node/.style={align=center}] + + % Position the nodes + \matrix{ + \& \node[object] (src) {Source text}; \& \\ + \& \node[object] (exprs) {Expressions}; \& \\ + \node[object] (sigts) {Method signatures}; \& \& \node[object] (lowered) {Lowered expressions}; \\ + \& \node[object] (methods) {Methods}; \& \\ + }; + + % Draw the arrows and labels + %\draw[to] (src) -- node[midway,left] {\texttt{Meta.parse}} (exprs); + %\draw[to] (exprs) -- node[midway,right] {\texttt{print}} (src); + \path [-latex] (src.south west) edge node[midway,left] {\texttt{Meta.parse}} (exprs.north west); + \path[dashed] [-latex] (exprs.north east) edge node[midway,right] {\texttt{print}} (src.south east); + + \path [-latex] (exprs.south east) edge node[midway,right] {\texttt{Meta.lower}} (lowered.north west); + \path [-latex] (lowered.south west) edge node[midway,right,color=red] {\texttt{eval}} (methods.north east); + \path [-latex] (methods.east) edge [bend right=20] node[midway,right] {\texttt{uncompress\_ast}} (lowered.south); + \path [-latex] (methods.north west) edge node[midway,right] {\texttt{meth.sig}} (sigts.south east); + \path [-latex] (sigts.south) edge node[midway,left] {\texttt{which}} (methods.west); + +v \path [-latex] (exprs.south west) edge node[midway,left,color=green] {\texttt{ExprsSigs}} (sigts.north east); + \path [-latex] (lowered.west) edge (sigts.east); + +% \path[dashed] [-latex] (methods.south west) edge [bend left=100] node[pos=0.75,left] {file,line} (src.west); + %\draw[dashed,->] (methods) to[out=180,in=-90] ($(sigts)-(2,0)$) to[out=90,in=180] node[midway,left] {file,line} (src); + \path [-latex] (sigts.north west) edge [bend left=20] node[pos=0.4,left,color=green] {\texttt{\tiny CodeTracking.method\_info}} (exprs.west); + + \end{tikzpicture} + + Base \qquad {\color{red} Base destructive} \qquad {\color{green} Revise} + +\end{document} diff --git a/packages/Revise/docs/src/index.md b/packages/Revise/docs/src/index.md new file mode 100644 index 0000000..62984e3 --- /dev/null +++ b/packages/Revise/docs/src/index.md @@ -0,0 +1,212 @@ +# Introduction to Revise + +`Revise.jl` may help you keep your Julia sessions running longer, reducing the +need to restart when you make changes to code. +With Revise, you can be in the middle of a session and then edit source code, +update packages, switch git branches, and/or stash/unstash code; +typically, the changes will be incorporated into the very next command you issue from the REPL. +This can save you the overhead of restarting, loading packages, and waiting for code to JIT-compile. + +Using Revise also improves your experience when using the +[debuggers](https://julialang.org/blog/2019/03/debuggers/). +Revise will keep track of changed locations of your methods in file, and ensure that the +debugger displays the source code of what you're actually debugging. + +!!! note "Automatically loading Revise" + + Many users automatically load Revise on startup. + On versions of Julia older than 1.5, this is slightly more involved + than just adding `using Revise` to `.julia/config/startup.jl`: see + [Using Revise by default](@ref) for details. + +## Installation + +You can obtain Revise using Julia's Pkg REPL-mode (hitting `]` as the first character of the command prompt): + +```julia +(v1.0) pkg> add Revise +``` + +or with `using Pkg; Pkg.add("Revise")`. + +## Usage example + +We'll make changes to Julia's "Example" package (a trivial package designed to +illustrate the file and directory organization of typical packages). +We have to "develop" it in order to make changes: + +```julia +(v1.0) pkg> dev Example +[...output related to installation...] + +``` +Now we load Revise (if we haven't already done so) and Example: +```julia +julia> using Revise # importantly, this must come before `using Example` + +julia> using Example + +julia> hello("world") +"Hello, world" +``` + +Now we're going to check that the `Example` module currently lacks a function named `f`: + +```julia +julia> Example.f() +ERROR: UndefVarError: f not defined +``` + +But say we really want `f`, so let's add it. +You can either navigate to the source code (at `.julia/dev/Example/src/Example.jl`) +in an editor manually, or you can use Julia to open it for you: + +```julia +julia> edit(hello) # opens Example.jl in the editor you have configured +``` + +Now, add a function `f() = π` and save the file. +Go back to the REPL (the *same* REPL, don't restart Julia) and try this: + +```julia +julia> Example.f() +π = 3.1415926535897... +``` + +Voila! Even though we'd loaded Example before adding this function, +Revise noticed the change and inserted it into our running session. + +!!! warning + Revise's first revision has latency of several seconds--it's compiling all of its internal code, which includes a complete [Julia interpreter](https://github.com/JuliaDebug/JuliaInterpreter.jl) and all of Revise's parse/diff/patch/cache machinery. + After your first revision, future revisions will generally be fast enough that they will seem nearly instantaneous. (There are exceptions, but they occur + only in specific circumstances, for example when Revise's own code gets [invalidated](https://julialang.org/blog/2020/08/invalidations/) by your changes.) + +Now suppose we realize we've made a horrible mistake: that `f` method will mess up everything, because it's part of a more complicated dispatch process and incorrectly intercepts certain `f` calls. +No problem, just delete `f` in your editor, save the file, and you're back to this: + +```julia +julia> Example.f() +ERROR: UndefVarError: f not defined +``` + +all without restarting Julia. +While you can evaluate *new* methods without Revise using [inline evaluation](https://www.julia-vscode.org/docs/stable/userguide/runningcode/#Julia:-Execute-Code-Block-(AltEnter)-1) through your IDE, +method *deletion* is just one example of a change that can only be made easily by Revise. + +If you need more examples, see [Revise usage: a cookbook](@ref). + +## Other key features of Revise + +Revise updates its internal paths when you change versions of a package. +To try this yourself, first re-insert that definition of `f` in the `dev` version of +`Example` and save the file. +Now try toggling back and forth between the `dev` and released versions of `Example`: + +```julia +(v1.0) pkg> free Example # switch to the released version of Example + +julia> Example.f() +ERROR: UndefVarError: f not defined + +(v1.0) pkg> dev Example + +julia> Example.f() +π = 3.1415926535897... +``` + +Revise is not tied to any particular editor. +(The [EDITOR or JULIA_EDITOR](https://docs.julialang.org/en/v1/stdlib/InteractiveUtils/#InteractiveUtils.edit-Tuple{AbstractString,Integer}) environment variables can be used to specify your preference for which editor gets launched by Julia's `edit` function.) + +If you don't want to have to remember to say `using Revise` each time you start +Julia, see [Using Revise by default](@ref). + +## What Revise can track + +Revise is fairly ambitious: if all is working, subject to a few [Limitations](@ref) you should be able to track changes to + +- any package that you load with `import` or `using` +- any script you load with [`includet`](@ref) (see [Configuring the revise mode](@ref) for important default restrictions on `includet`) +- any file defining `Base` julia itself (with `Revise.track(Base)`) +- any of Julia's standard libraries (with, e.g., `using Unicode; Revise.track(Unicode)`) +- any file defining `Core.Compiler` (with `Revise.track(Core.Compiler)`) + +The last one requires that you clone Julia and build it yourself from source. + +## Secrets of Revise "wizards" + +Revise can assist with methodologies like +[test-driven development](https://en.wikipedia.org/wiki/Test-driven_development). +While it's often desirable to write the test first, sometimes when fixing a bug +it's very difficult to write a good test until you understand the bug better. +Often that means basically fixing the bug before your write the test. +With Revise, you can + +- fix the bug while simultaneously developing a high-quality test +- verify that your test passes with the fixed code +- `git stash` your fix and check that your new test fails on the old code, + thus verifying that your test captures the essence of the former bug (if it doesn't fail, + you need a better test!) +- `git stash pop`, test again, commit, and submit + +all without restarting your Julia session. + +## Other Revise workflows + +Revise can be used to perform work when files update. +For example, let's say you want to regenerate a set of web pages whenever your code changes. +Suppose you've placed your Julia code in a package called `MyWebCode`, +and the pages depend on "file.js" and all files in the "assets/" directory; then + +```julia +entr(["file.js", "assets"], [MyWebCode]) do + build_webpages(args...) +end +``` + +will execute `build_webpages(args...)` whenever you save updates to the listed files +or `MyWebCode`. + +If you want to regenerate the web page as soon as any change is detected, not +only in `MyWebCode` but also in any package tracked by Revise, you can provide +the `all` keyword argument to [`entr`](@ref): + +```julia +entr(["file.js", "assets"]; all=true) do + build_webpages(args...) +end +``` + +## Taking advantage of Revise in other packages + +To make it easier for other packages to benefit from Revise without needing to add it +as a dependency or understand Revise's internals, Revise interfaces with +[CodeTracking](https://github.com/timholy/CodeTracking.jl), +which is a small package acting as Revise's "query" interface. + +## What else do I need to know? + +Except in cases of problems (see below), that's it! +Revise is a tool that runs in the background, and when all is well it should be +essentially invisible, except that you don't have to restart Julia so often. + +Revise can also be used as a "library" by developers who want to add other new capabilities +to Julia; the sections [How Revise works](@ref) and [Developer reference](@ref) are +particularly relevant for them. + +## If Revise doesn't work as expected + +If Revise isn't working for you, here are some steps to try: + +- See [Configuration](@ref) for information on customization options. + In particular, some file systems (like [NFS](https://en.wikipedia.org/wiki/Network_File_System)) and current users of [WSL2](https://devblogs.microsoft.com/commandline/announcing-wsl-2/) might require special options. +- Revise can't handle all kinds of code changes; for more information, + see the section on [Limitations](@ref). +- Try running `test Revise` from the Pkg REPL-mode. + If tests pass, check the documentation to make sure you understand how Revise should work. + If they fail (especially if it mirrors functionality that you need and isn't working), see + [Debugging problems with paths](@ref) for one set of suggestions. + +If you still encounter problems, please [file an issue](https://github.com/timholy/Revise.jl/issues). +Especially if you think Revise is making mistakes in adding or deleting methods, please +see the page on [Debugging Revise](@ref) for information about how to attach logs +to your bug report. diff --git a/packages/Revise/docs/src/internals.md b/packages/Revise/docs/src/internals.md new file mode 100644 index 0000000..e421ea7 --- /dev/null +++ b/packages/Revise/docs/src/internals.md @@ -0,0 +1,388 @@ +# How Revise works + +In addition to the material below, see these talks: +- [JuliaCon 2018](https://www.youtube.com/watch?v=KuM0AGaN09s) +- [JuliaCon 2019](https://www.youtube.com/watch?v=gXDI4DSp04c) + +Revise is based on the fact that you can change functions even when +they are defined in other modules. +Here's an example showing how you do that manually (without using Revise): + +```julia +julia> convert(Float64, π) +3.141592653589793 + +julia> # That's too hard, let's make life easier for students + +julia> @eval Base convert(::Type{Float64}, x::Irrational{:π}) = 3.0 +convert (generic function with 714 methods) + +julia> convert(Float64, π) +3.0 +``` + +Revise removes some of the tedium of manually copying and pasting code +into `@eval` statements. +To decrease the amount of re-JITting +required, Revise avoids reloading entire modules; instead, it takes care +to `eval` only the *changes* in your package(s), much as you would if you were +doing it manually. +Importantly, changes are detected in a manner that is independent of the specific +line numbers in your code, so that you don't have to re-evaluate just +because code moves around within the same file. +(One unfortunate side effect is that line numbers may become inaccurate in backtraces, +but Revise takes pains to correct these, see below.) + +Conceptually, Revise implements +[`diff` and `patch`](https://acloudguru.com/blog/engineering/introduction-using-diff-and-patch/) +for a running Julia session. Schematically, Revise's inner loop (`revise()`) looks like this: + +```julia +for def in setdiff(oldexprs, newexprs) + # `def` is an expression that defines a method. + # It was in `oldexprs`, but is no longer present in `newexprs`--delete the method. + delete_methods_corresponding_to_defexpr(mod, def) +end +for def in setdiff(newexprs, oldexprs) + # `def` is an expression for a new or modified method. Instantiate it. + Core.eval(mod, def) +end +``` + +In somewhat greater detail, Revise uses the following overall strategy: + +- add callbacks to Base so that Revise gets notified when new + packages are loaded or new files `include`d +- prepare source-code caches for every new file. These caches + will allow Revise to detect changes when files are updated. For precompiled + packages this happens on an as-needed basis, using the cached + source in the `*.ji` file. For non-precompiled packages, Revise parses + the source for each `include`d file immediately so that the initial state is + known and changes can be detected. +- monitor the file system for changes to any of the dependent files; + it immediately appends any updates to a list of file names that need future + processing +- intercept the REPL's backend to ensure that the list of + files-to-be-revised gets processed each time you execute a new + command at the REPL +- when a revision is triggered, the source file(s) are re-parsed, and + a diff between the cached version and the new version is + created. `eval` the diff in the appropriate module(s). +- replace the cached version of each source file with the new version, so that + further changes are `diff`ed against the most recent update. + +## The structure of Revise's internal representation + +![diagram](figures/diagram.png) + +**Figure notes**: Nodes represent primary objects in Julia's compilation pipeline. +Arrows and their labels represent functions or data structures that allow you to move from one node to another. +Red ("destructive") paths force recompilation of dependent functions. + +Revise bridges between text files (your source code) and compiled code. +Revise consequently maintains data structures that parallel Julia's own internal +processing of code. +When dealing with a source-code file, you start with strings, parse them to obtain Julia +expressions, evaluate them to obtain Julia objects, and (where appropriate, +e.g., for methods) compile them to machine code. +This will be called the *forward workflow*. +Revise sets up a few key structures that allow it to progress from files to modules +to Julia expressions and types. + +Revise also sets up a *backward workflow*, proceeding from compiled code to Julia +types back to Julia expressions. +This workflow is useful, for example, when dealing with errors: the stack traces +displayed by Julia link from the compiled code back to the source files. +To make this possible, Julia builds "breadcrumbs" into compiled code that store the +filename and line number at which each expression was found. +However, these links are static, meaning they are set up once (when the code is compiled) +and are not updated when the source file changes. +Because trivial manipulations to source files (e.g., the insertion of blank lines +and/or comments) can change the line number of an expression without necessitating +its recompilation, Revise implements a way of correcting these line numbers before +they are displayed to the user. +The same problem presents when using a [debugger](https://julialang.org/blog/2019/03/debuggers/), in that one wants the debugger to display the correct code (at the correct line number) even after modifications have been made to the file. +This capability requires that Revise proceed backward from the compiled objects to +something resembling the original text file. + +### Terminology + +A few convenience terms are used throughout: *definition*, +*signature-expression*, and *signature-type*. +These terms are illustrated using the following example: + +```@raw html +

function print_item(io::IO, item, ntimes::Integer=1, pre::String="")
+    print(io, pre)
+    for i = 1:ntimes
+        print(io, item)
+    end
+end

+``` + +This represents the *definition* of a method. +Definitions are stored as expressions, using a [`Revise.RelocatableExpr`](@ref). +The highlighted portion is the *signature-expression*, specifying the name, argument names +and their types, and (if applicable) type-parameters of the method. + +From the signature-expression we can generate one or more *signature-types*. +Since this function has two default arguments, this signature-expression generates +three signature-types, each corresponding to a different valid way of calling +this method: + +```julia +Tuple{typeof(print_item),IO,Any} # print_item(io, item) +Tuple{typeof(print_item),IO,Any,Integer} # print_item(io, item, 2) +Tuple{typeof(print_item),IO,Any,Integer,String} # print_item(io, item, 2, " ") +``` + +In Revise's internal code, a definition is often represented with a variable `def`, and a signature-type with `sigt`. +Recent versions of Revise do not make extensive use of signature expressions. + +### Computing signatures + +Since version 2.0, Revise works primarily with lowered-code representations, specifically using the lowered code to compute method signatures (if you don't know about lowered code, see [this tutorial](https://juliadebug.github.io/JuliaInterpreter.jl/stable/ast/)). +There are several reasons that make this an attractive approach, of which the most important are: + +- keyword-argument methods get "expanded" to multiple methods handling various ways of populating the arguments. The lowered code lists all of them, which ensures that Revise knows about them all. (There are some challenges regarding "gensymmed" names, see [LoweredCodeUtils](https://github.com/JuliaDebug/LoweredCodeUtils.jl) and [julia#30908](https://github.com/JuliaLang/julia/issues/30908), but in short LoweredCodeUtils "fixes" those difficulties.) +- for methods generated by code, the only really reliable mechanism to compute all the signatures is to step through the code that generates the methods. That is performed using [JuliaInterpreter](https://github.com/JuliaDebug/JuliaInterpreter.jl). + +As an example, suppose the following code is part of your module definition: +``` +for T in (Float16, Float32, Float64) + @eval sizefloat(x::$T) = sizeof($T) +end +``` +!!! clarification + + This is equivalent to the following explicit definitions: + ``` + sizefloat(x::Float16) = 2 + sizefloat(x::Float32) = 4 + sizefloat(x::Float64) = 8 + ``` + +If you replace the loop with `for T in (Float32, Float64)`, then Revise should delete the method for `Float16`. But this implies that Revise can deduce all the method-signatures created by this block, which essentially requires "simulating" the block that defines the methods. (In simple cases there are other approaches, but for [complex cases](https://github.com/JuliaLang/julia/blob/c7e4b9929b3b6ee89d47ce1320ef2de14c4ecf85/base/atomics.jl#L415-L430) stepping through the code seems to be the only viable answer.) + +Because lowered code is far simpler than ordinary Julia code, it is much easier to interpret. Let's look briefly at a method definition: + +``` +floatwins(x::AbstractFloat, y::Integer) = x +``` + +which has lowered representation approximately equal to + +``` +CodeInfo( +│ $(Expr(:method, :floatwins)) +│ %2 = Core.Typeof(floatwins) +│ %3 = Core.svec(%2, AbstractFloat, Integer) +│ %4 = Core.svec() +│ %5 = Core.svec(%3, %4) +│ $(Expr(:method, :floatwins, :(%5), CodeInfo(quote + return x +end))) +└── return floatwins +) +``` + +(I've edited this lightly for clarity.) As one steps through this, the first line tells us we're about to define a method for the function `floatwins`. Lines 2-5 compute the signature, in the representation `svec(sig, params)`, where here `sig = svec(typeof(floatwins), AbstractFloat, Integer)` and `params = svec()`. +(This example has no type parameters, which is why `params` is empty.) + +What Revise does is steps through the first 5 of these lines, and when it encounters the `Expr(:method, :floatwins, :(%5), CodeInfo(...))` statement, +it pulls out the signature (the `%5`, which refers to the result computed on the 5th line) and records this as a method generated by this block of code. (It does not, however, evaluate the `Expr(:method, ...)` expression as a whole, because that would force it to be recompiled.) Stepping through this code ensures that Revise can compute the exact signature, no matter how this method is defined at the level of ordinary Julia code. + +Unfortunately, modules sometimes contain code blocks that perhaps shouldn't be interpreted: + +```julia +init_c_library() # library crashes if we call this twice +``` + +Starting with version 2.3, Revise attempts to avoid interpreting any code not necessary for signature computation. +If you are just tracking changes, Revise will skip over such blocks; if you're loading a file with `includet` for the first time, Revise will execute such blocks in compiled mode. + +Revise achieves this by computing [backedges](https://juliadebug.github.io/LoweredCodeUtils.jl/stable/edges/), essentially a set of links encoding the dependencies among different lines of the lowered code. +For the `floatwins` example above, the backedges would represent the fact that line 2 has one direct dependant, line 3 (which uses `%2`), that lines 3 and 4 both have line 5 as their dependents, and line 5 has line 6 as a dependent. As a consequence, to (nearly) execute line 6, we have to execute lines 2-5, because they set up the signature. If an interdependent block doesn't contain any `:method` or related (`:struct_type`, `:eval`) expressions, then it doesn't need to interpret the block at all. + +As should be evident, the lowered code makes it much easier to analyze the graph of these dependencies. There are, however, a few tricky cases. +For example, any code inside an `@eval` might, or might not, expand into lowered code that contains a `:method` expression. Because Revise can't reliably predict what it will look like after expansion, Revise will execute any code in (or needed for) an `@eval` block. As a consequence, even after version 2.3 Revise may sometimes interpret more code than is strictly necessary. + +!!! note + + If Revise executes code that still shouldn't be run twice, one good solution is to put all initialization inside your module's [`__init__` function](https://docs.julialang.org/en/v1/manual/modules/#Module-initialization-and-precompilation-1). + For files that you track with `includet`, you can also split "code that defines methods" into a separate file from "code that does work," and have Revise track only the method-defining file. + However, starting with version 2.3 Revise should be fairly good at doing this on its own; such manual interventions should not be necessary in most cases. + +### Core data structures and representations + +Most of Revise's magic comes down to just three internal variables: + +- [`Revise.watched_files`](@ref): encodes information used by the filesystem (`FileWatching`) + to detect changes in source files. +- [`Revise.revision_queue`](@ref): a list of "work to do," containing the files that have been + modified since the last code update. +- [`Revise.pkgdatas`](@ref): the central repository of parsed code, used to "diff" for changes + and then "patch" the running session. + +Two "maps" are central to Revise's inner workings: `ExprsSigs` maps link +definition=>signature-types (the forward workflow), while `CodeTracking` (specifically, +its internal variable `method_info`) links from +signature-type=>definition (the backward workflow). +Concretely, `CodeTracking.method_info` is just an `IdDict` mapping `sigt=>(locationinfo, def)`. +Of note, a stack frame typically contains a link to a method, which stores the equivalent +of `sigt`; consequently, this information allows one to look up the corresponding +`locationinfo` and `def`. (When methods move, the location information stored by CodeTracking +gets updated by Revise.) + +Some additional notes about Revise's `ExprsSigs` maps: + +- For expressions that do not define a method, it is just `def=>nothing` +- For expressions that do define a method, it is `def=>[sigt1, ...]`. + `[sigt1, ...]` is the list of signature-types generated from `def` (often just one, + but more in the case of methods with default arguments or keyword arguments). +- They are represented as an `OrderedDict` so as to preserve the sequence in which expressions + occur in the file. + This can be important particularly for updating macro definitions, which affect the + expansion of later code. + The order is maintained so as to match the current ordering of the source-file, + which is not necessarily the same as the ordering when these expressions were last + `eval`ed. +- Each key in the map (the definition `RelocatableExpr`) is the most recently + `eval`ed version of the expression. + This has an important consequence: the line numbers in the `def` (which are still present, + even though not used for equality comparisons) correspond to the ones in compiled code. + Any discrepancy with the current line numbers in the file is handled through updates to + the location information stored by `CodeTracking`. + +`ExprsSigs` are organized by module and then file, so that one can map +`filename`=>`module`=>`def`=>`sigts`. +Importantly, single-file modules can be "reconstructed" from the keys of the corresponding +`ExprsSigs` (and multi-file modules from a collection of such items), since they hold +the complete ordered set of expressions that would be `eval`ed to define the module. + +The global variable that holds all this information is [`Revise.pkgdatas`](@ref), organized +into a dictionary of [`Revise.PkgData`](@ref) objects indexed by Base Julia's `PkgId` +(a unique identifier for packages). + +### An example + +Consider a module, `Items`, defined by the following two source files: + +`Items.jl`: + +```julia +__precompile__(false) + +module Items + +include("indents.jl") + +function print_item(io::IO, item, ntimes::Integer=1, pre::String=indent(item)) + print(io, pre) + for i = 1:ntimes + print(io, item) + end +end + +end +``` + +`indents.jl`: + +```julia +indent(::UInt16) = 2 +indent(::UInt8) = 4 +``` + +If you create this as a mini-package and then say `using Revise, Items`, you can start +examining internal variables in the following manner: + +```julia +julia> id = Base.PkgId(Items) +Items [b24a5932-55ed-11e9-2a88-e52f99e65a0d] + +julia> pkgdata = Revise.pkgdatas[id] +PkgData(Items [b24a5932-55ed-11e9-2a88-e52f99e65a0d]: + "src/Items.jl": FileInfo(Main=>ExprsSigs(<1 expressions>, <0 signatures>), Items=>ExprsSigs(<2 expressions>, <3 signatures>), ) + "src/indents.jl": FileInfo(Items=>ExprsSigs(<2 expressions>, <2 signatures>), ) +``` + +(Your specific UUID may differ.) + +Path information is stored in `pkgdata.info`: +```julia +julia> pkgdata.info +PkgFiles(Items [b24a5932-55ed-11e9-2a88-e52f99e65a0d]): + basedir: "/tmp/pkgs/Items" + files: ["src/Items.jl", "src/indents.jl"] +``` + +`basedir` is the only part using absolute paths; everything else is encoded relative +to that location. This facilitates, e.g., switching between `develop` and `add` mode in the +package manager. + +`src/indents.jl` is particularly simple: + +```julia +julia> pkgdata.fileinfos[2] +FileInfo(Items=>ExprsSigs with the following expressions: + :(indent(::UInt16) = begin + 2 + end) + :(indent(::UInt8) = begin + 4 + end), ) +``` + +This is just a summary; to see the actual `def=>sigts` map, do the following: + +```julia +julia> pkgdata.fileinfos[2].modexsigs[Items] +OrderedCollections.OrderedDict{Revise.RelocatableExpr,Union{Nothing, Array{Any,1}}} with 2 entries: + :(indent(::UInt16) = begin… => Any[Tuple{typeof(indent),UInt16}] + :(indent(::UInt8) = begin… => Any[Tuple{typeof(indent),UInt8}] +``` + +These are populated now because we specified `__precompile__(false)`, which forces +Revise to defensively parse all expressions in the package in case revisions are made +at some future point. +For precompiled packages, each `pkgdata.fileinfos[i]` can instead rely on the `cachefile` +(another field stored in the [`Revise.FileInfo`](@ref)) as a record of the state of the file +at the time the package was loaded; as a consequence, Revise can defer parsing the source +file(s) until they are updated. + +`Items.jl` is represented with a bit more complexity, +`"Items.jl"=>Dict(Main=>map1, Items=>map2)`. +This is because `Items.jl` contains one expression (the `__precompile__` statement) +that is `eval`ed in `Main`, +and other expressions that are `eval`ed in `Items`. + +### Revisions and computing diffs + +When the file system notifies Revise that a file has been modified, Revise re-parses +the file and assigns the expressions to the appropriate modules, creating a +[`Revise.ModuleExprsSigs`](@ref) `mexsnew`. +It then compares `mexsnew` against `mexsref`, the reference object that is synchronized to +code as it was `eval`ed. +The following actions are taken: + +- if a `def` entry in `mexsref` is equal to one in `mexsnew`, the expression is "unchanged" + except possibly for line number. The `locationinfo` in `CodeTracking` is updated as needed. +- if a `def` entry in `mexsref` is not present in `mexsnew`, that entry is deleted and + any corresponding methods are also deleted. +- if a `def` entry in `mexsnew` is not present in `mexsref`, it is `eval`ed and then added to + `mexsref`. + +Technically, a new `mexsref` is generated every time to ensure that the expressions are +ordered as in `mexsnew`; however, conceptually this is better thought of as an updating of +`mexsref`, after which `mexsnew` is discarded. + +Note that one consequence is that modifying a method causes two actions, the deletion of +the original followed by `eval`ing a new version. +During revision, all method deletions are performed first, followed by all the new `eval`ed methods. +This ensures that if a method gets moved from `fileB.jl` to `fileA.jl`, Revise doesn't mistakenly +redefine and then delete the method simply because `fileA.jl` got processed before `fileB.jl`. + +### Internal API + +You can find more detail about Revise's inner workings in the [Developer reference](@ref). diff --git a/packages/Revise/docs/src/limitations.md b/packages/Revise/docs/src/limitations.md new file mode 100644 index 0000000..65da2ab --- /dev/null +++ b/packages/Revise/docs/src/limitations.md @@ -0,0 +1,91 @@ +# Limitations + +There are some kinds of changes that Revise (or often, Julia itself) cannot incorporate into a running Julia session: + +- changes to type definitions or `const`s +- conflicts between variables and functions sharing the same name +- removal of `export`s + +These kinds of changes require that you restart your Julia session. + +During early stages of development, it's quite common to want to change type definitions. You can work around Julia's/Revise's limitations by temporary renaming: + +```julia +# 1st version +struct FooStruct1 + bar::Int +end +FooStruct = FooStruct1 +function processFoo(foo::FooStruct) + @info foo.bar +end +``` +and then the type can be updated like +```julia +# 2nd version +struct FooStruct2 # change version here + bar::Int + str::String +end +FooStruct = FooStruct2 # change version here +function processFoo(foo::FooStruct) # no need to change this + @info foo.bar +end +``` +This works as long as the new type name doesn't conflict with an existing name; within a session you need to change the name each time you change the definition. + +Once your development has converged on a solution, it's best to switch to the "permanent" name: in the example above, `FooStruct` is a non-constant global variable, and if used internally in a function there will be consequent performance penalties. Switching to the permanent name will force you to restart your session. + +In addition, some situations may require special handling: + +### Macros and generated functions + +If you change a macro definition or methods that get called by `@generated` functions +outside their `quote` block, these changes will not be propagated to functions that have +already evaluated the macro or generated function. + +You may explicitly call `revise(MyModule)` to force reevaluating every definition in module +`MyModule`. +Note that when a macro changes, you have to revise all of the modules that *use* it. + +### Distributed computing (multiple workers) and anonymous functions + +Revise supports changes to code in worker processes. +The code must be loaded in the main process in which Revise is running. + +Revise cannot handle changes in anonymous functions used in `remotecall`s. +Consider the following module definition: + +```julia +module ParReviseExample +using Distributed + +greet(x) = println("Hello, ", x) + +foo() = for p in workers() + remotecall_fetch(() -> greet("Bar"), p) +end + +end # module +``` + +Changing the remotecall to `remotecall_fetch((x) -> greet("Bar"), p, 1)` will fail, +because the new anonymous function is not defined on all workers. +The workaround is to write the code to use named functions, e.g., + +```julia +module ParReviseExample +using Distributed + +greet(x) = println("Hello, ", x) +greetcaller() = greet("Bar") + +foo() = for p in workers() + remotecall_fetch(greetcaller, p) +end + +end # module +``` + +and the corresponding edit to the code would be to modify it to `greetcaller(x) = greet("Bar")` +and `remotecall_fetch(greetcaller, p, 1)`. diff --git a/packages/Revise/docs/src/user_reference.md b/packages/Revise/docs/src/user_reference.md new file mode 100644 index 0000000..c986459 --- /dev/null +++ b/packages/Revise/docs/src/user_reference.md @@ -0,0 +1,30 @@ +# User reference + +There are really only six functions that most users would be expected to call manually: +`revise`, `includet`, `Revise.track`, `entr`, `Revise.retry`, and `Revise.errors`. +Other user-level constructs might apply if you want to debug Revise or +prevent it from watching specific packages, or for fine-grained handling of callbacks. + +```@docs +revise +Revise.track +includet +entr +Revise.retry +Revise.errors +``` + +### Revise logs (debugging Revise) + +```@docs +Revise.debug_logger +Revise.actions +Revise.diffs +``` + +### Prevent Revise from watching specific packages + +```@docs +Revise.dont_watch_pkgs +Revise.silence +``` diff --git a/packages/Revise/images/revise-logo.png b/packages/Revise/images/revise-logo.png new file mode 100644 index 0000000..350f98a Binary files /dev/null and b/packages/Revise/images/revise-logo.png differ diff --git a/packages/Revise/images/revise-logo.svg b/packages/Revise/images/revise-logo.svg new file mode 100644 index 0000000..774f7db --- /dev/null +++ b/packages/Revise/images/revise-logo.svg @@ -0,0 +1,135 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/Revise/images/revise-wordmark.png b/packages/Revise/images/revise-wordmark.png new file mode 100644 index 0000000..f4eb0f9 Binary files /dev/null and b/packages/Revise/images/revise-wordmark.png differ diff --git a/packages/Revise/images/revise-wordmark.svg b/packages/Revise/images/revise-wordmark.svg new file mode 100644 index 0000000..8c39fb7 --- /dev/null +++ b/packages/Revise/images/revise-wordmark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/Revise/src/Revise.jl b/packages/Revise/src/Revise.jl new file mode 100644 index 0000000..efda9ab --- /dev/null +++ b/packages/Revise/src/Revise.jl @@ -0,0 +1,46 @@ +""" +Revise.jl tracks source code changes and incorporates the changes to a running Julia session. + +Revise.jl works behind-the-scenes. To track a package, e.g. `Example`: +```julia +(@v1.6) pkg> dev Example # make a development copy of the package +[...pkg output omitted...] + +julia> using Revise # this must come before the package under development + +julia> using Example + +[...develop the package...] # Revise.jl will automatically update package functionality to match code changes + +``` + +Functions in Revise.jl that may come handy in special circumstances: +- `Revise.track`: track updates to `Base` Julia itself or `Core.Compiler` +- `includet`: load a file and track future changes. Intended for small, quick works +- `entr`: call an additional function whenever code updates +- `revise`: evaluate any changes in `Revise.revision_queue` or every definition in a module +- `Revise.retry`: perform previously-failed revisions. Useful in cases of order-dependent errors +- `Revise.errors`: report the errors represented in `Revise.queue_errors` +""" +module Revise + +# We use a code structure where all `using` and `import` +# statements in the package that load anything other than +# a Julia base or stdlib package are located in this file here. +# Nothing else should appear in this file here, apart from +# the `include("packagedef.jl")` statement, which loads what +# we would normally consider the bulk of the package code. +# This somewhat unusual structure is in place to support +# the VS Code extension integration. + +using OrderedCollections, CodeTracking, JuliaInterpreter, LoweredCodeUtils + +using CodeTracking: PkgFiles, basedir, srcfiles, line_is_decl, basepath +using JuliaInterpreter: whichtt, is_doc_expr, step_expr!, finish_and_return!, get_return, + @lookup, moduleof, scopeof, pc_expr, is_quotenode_egal, + linetable, codelocs, LineTypes, is_GotoIfNot, isassign, isidentical +using LoweredCodeUtils: next_or_nothing!, trackedheads, callee_matches + +include("packagedef.jl") + +end # module diff --git a/packages/Revise/src/callbacks.jl b/packages/Revise/src/callbacks.jl new file mode 100644 index 0000000..b40e5ad --- /dev/null +++ b/packages/Revise/src/callbacks.jl @@ -0,0 +1,173 @@ +# Globals needed to support `entr` and other callbacks + +""" + Revise.revision_event + +This `Condition` is used to notify `entr` that one of the watched files has changed. +""" +const revision_event = Condition() + +""" + Revise.user_callbacks_queue + +Global variable, `user_callbacks_queue` holds `key` values for which the +file has changed but the user hooks have not yet been called. +""" +const user_callbacks_queue = Set{Any}() + +""" + Revise.user_callbacks_by_file + +Global variable, maps files (identified by their absolute path) to the set of +callback keys registered for them. +""" +const user_callbacks_by_file = Dict{String, Set{Any}}() + +""" + Revise.user_callbacks_by_key + +Global variable, maps callback keys to user hooks. +""" +const user_callbacks_by_key = Dict{Any, Any}() + + + +""" + key = Revise.add_callback(f, files, modules=nothing; key=gensym()) + +Add a user-specified callback, to be executed during the first run of +`revise()` after a file in `files` or a module in `modules` is changed on the +file system. If `all` is set to `true`, also execute the callback whenever any +file already monitored by Revise changes. In an interactive session like the +REPL, Juno or Jupyter, this means the callback executes immediately before +executing a new command / cell. + +You can use the return value `key` to remove the callback later +(`Revise.remove_callback`) or to update it using another call +to `Revise.add_callback` with `key=key`. +""" +function add_callback(f, files, modules=nothing; all=false, key=gensym()) + fix_trailing(path) = isdir(path) ? joinpath(path, "") : path # insert a trailing '/' if missing, see https://github.com/timholy/Revise.jl/issues/470#issuecomment-633298553 + + remove_callback(key) + + files = map(fix_trailing, map(abspath, files)) + init_watching(files) + + # in case the `all` kwarg was set: + # add all files which are already known to Revise + if all + for pkgdata in values(pkgdatas) + append!(files, joinpath.(Ref(basedir(pkgdata)), srcfiles(pkgdata))) + end + end + + if modules !== nothing + for mod in modules + track(mod) # Potentially needed for modules like e.g. Base + id = PkgId(mod) + pkgdata = pkgdatas[id] + for file in srcfiles(pkgdata) + absname = joinpath(basedir(pkgdata), file) + push!(files, absname) + end + end + end + + # There might be duplicate entries in `files`, but it shouldn't cause any + # problem with the sort of things we do here + for file in files + cb = get!(Set, user_callbacks_by_file, file) + push!(cb, key) + end + user_callbacks_by_key[key] = f + + return key +end + +""" + Revise.remove_callback(key) + +Remove a callback previously installed by a call to `Revise.add_callback(...)`. +See its docstring for details. +""" +function remove_callback(key) + for cbs in values(user_callbacks_by_file) + delete!(cbs, key) + end + delete!(user_callbacks_queue, key) + delete!(user_callbacks_by_key, key) + + # possible future work: we may stop watching (some of) these files + # now. But we don't really keep track of what background tasks are running + # and Julia doesn't have an ergonomic way of task cancellation yet (see + # e.g. + # https://github.com/JuliaLang/Juleps/blob/master/StructuredConcurrency.md + # so we'll omit this for now. The downside is that in pathological cases, + # this may exhaust inotify resources. + + nothing +end + +function process_user_callbacks!(keys = user_callbacks_queue; throw=false) + try + # use (a)sync so any exceptions get nicely collected into CompositeException + @sync for key in keys + f = user_callbacks_by_key[key] + @async Base.invokelatest(f) + end + catch err + if throw + rethrow(err) + else + @warn "[Revise] Ignoring callback errors" err + end + finally + empty!(keys) + end +end + +""" + entr(f, files; all=false, postpone=false, pause=0.02) + entr(f, files, modules; all=false, postpone=false, pause=0.02) + +Execute `f()` whenever files or directories listed in `files`, or code in `modules`, updates. +If `all` is `true`, also execute `f()` as soon as code updates are detected in +any module tracked by Revise. + +`entr` will process updates (and block your command line) until you press Ctrl-C. +Unless `postpone` is `true`, `f()` will be executed also when calling `entr`, +regardless of file changes. The `pause` is the period (in seconds) that `entr` +will wait between being triggered and actually calling `f()`, to handle +clusters of modifications, such as those produced by saving files in certain +text editors. + +# Example + +```julia +entr(["/tmp/watched.txt"], [Pkg1, Pkg2]) do + println("update") +end +``` +This will print "update" every time `"/tmp/watched.txt"` or any of the code defining +`Pkg1` or `Pkg2` gets updated. +""" +function entr(f::Function, files, modules=nothing; all=false, postpone=false, pause=0.02) + yield() + postpone || f() + key = add_callback(files, modules; all=all) do + sleep(pause) + f() + end + try + while true + wait(revision_event) + revise(throw=true) + end + catch err + isa(err, InterruptException) || rethrow(err) + finally + remove_callback(key) + end + nothing +end diff --git a/packages/Revise/src/git.jl b/packages/Revise/src/git.jl new file mode 100644 index 0000000..e0d5327 --- /dev/null +++ b/packages/Revise/src/git.jl @@ -0,0 +1,84 @@ +""" + repo, repo_path = git_repo(path::AbstractString) + +Return the `repo::LibGit2.GitRepo` containing the file or directory `path`. +`path` does not necessarily need to be the top-level directory of the +repository. Also returns the `repo_path` of the top-level directory for the +repository. +""" +function git_repo(path::AbstractString) + if isfile(path) + path = dirname(path) + end + while true + # check if we are at the repo root + git_dir = joinpath(path, ".git") + if ispath(git_dir) + return LibGit2.GitRepo(path), path + end + # traverse to parent folder + previous = path + path = dirname(path) + if previous == path + return nothing, path + end + end +end + +function git_tree(repo::LibGit2.GitRepo, commit="HEAD") + return LibGit2.GitTree(repo, "$commit^{tree}") +end +function git_tree(path::AbstractString, commit="HEAD") + repo, _ = git_repo(path) + return git_tree(repo, commit) +end + +""" + files = git_files(repo) + +Return the list of files checked into `repo`. +""" +function git_files(repo::LibGit2.GitRepo) + status = LibGit2.GitStatus(repo; + status_opts=LibGit2.StatusOptions(flags=LibGit2.Consts.STATUS_OPT_INCLUDE_UNMODIFIED)) + files = String[] + for i = 1:length(status) + e = status[i] + dd = unsafe_load(e.head_to_index) + push!(files, unsafe_string(dd.new_file.path)) + end + return files +end +Base.keys(tree::LibGit2.GitTree) = git_files(tree.owner) + +""" + Revise.git_source(file::AbstractString, reference) + +Read the source-text for `file` from a git commit `reference`. +The reference may be a string, Symbol, or `LibGit2.Tree`. + +# Example: + + Revise.git_source("/path/to/myfile.jl", "HEAD") + Revise.git_source("/path/to/myfile.jl", :abcd1234) # by commit SHA +""" +function git_source(file::AbstractString, reference) + fullfile = abspath(file) + tree = git_tree(fullfile, reference) + # git uses Unix-style paths even on Windows + filepath = replace(relpath(fullfile, LibGit2.path(tree.owner)), + Base.Filesystem.path_separator_re=>'/') + return git_source(filepath, tree) +end + +function git_source(file::AbstractString, tree::LibGit2.GitTree) + local blob + blob = tree[file] + if blob === nothing + # assume empty tree when tracking new files + src = "" + else + src = LibGit2.content(blob) + end + return src +end diff --git a/packages/Revise/src/loading.jl b/packages/Revise/src/loading.jl new file mode 100644 index 0000000..c6ace75 --- /dev/null +++ b/packages/Revise/src/loading.jl @@ -0,0 +1,76 @@ +function pkg_fileinfo(id::PkgId) + origin = get(Base.pkgorigins, id, nothing) + origin === nothing && return nothing, nothing, nothing + cachepath = origin.cachepath + cachepath === nothing && return nothing, nothing, nothing + provides, includes_requires, required_modules = try + Base.parse_cache_header(cachepath; srcfiles_only=true) + catch + return nothing, nothing, nothing + end + includes, _ = includes_requires + for (pkgid, buildid) in provides + if pkgid.uuid === id.uuid && pkgid.name == id.name + return cachepath, includes, first.(required_modules) + end + end +end + +function parse_pkg_files(id::PkgId) + pkgdata = get(pkgdatas, id, nothing) + if pkgdata === nothing + pkgdata = PkgData(id) + end + modsym = Symbol(id.name) + if use_compiled_modules() + cachefile, includes, reqs = pkg_fileinfo(id) + if cachefile !== nothing + @assert includes !== nothing + @assert reqs !== nothing + pkgdata.requirements = reqs + for chi in includes + mod = Base.root_module(id) + for mpath in chi.modpath + mod = getfield(mod, Symbol(mpath))::Module + end + fname = relpath(chi.filename, pkgdata) + # For precompiled packages, we can read the source later (whenever we need it) + # from the *.ji cachefile. + push!(pkgdata, fname=>FileInfo(mod, cachefile)) + end + CodeTracking._pkgfiles[id] = pkgdata.info + return pkgdata + end + end + # Non-precompiled package(s). Here we rely on the `include` callbacks to have + # already populated `included_files`; all we have to do is collect the relevant + # files. + # To reduce compiler latency, use runtime dispatch for `queue_includes!`. + # `queue_includes!` requires compilation of the whole parsing/expression-splitting infrastructure, + # and it's better to wait to compile it until we actually need it. + Base.invokelatest(queue_includes!, pkgdata, id) + return pkgdata +end + +function modulefiles(mod::Module) + function keypath(filename) + filename = fixpath(filename) + return get(src_file_key, filename, filename) + end + parentfile = String(first(methods(getfield(mod, :eval))).file) + id = PkgId(mod) + if id.name == "Base" || Symbol(id.name) ∈ stdlib_names + parentfile = normpath(Base.find_source_file(parentfile)) + if !startswith(parentfile, juliadir) + parentfile = replace(parentfile, fallback_juliadir()=>juliadir) + end + filedata = Base._included_files + included_files = filter(mf->mf[1] == mod, filedata) + return keypath(parentfile), [keypath(mf[2]) for mf in included_files] + end + use_compiled_modules() || return nothing, nothing # FIXME: support non-precompiled packages + _, filedata, reqs = pkg_fileinfo(id) + filedata === nothing && return nothing, nothing + included_files = filter(mf->mf.id == id, filedata) + return keypath(parentfile), [keypath(mf.filename) for mf in included_files] +end diff --git a/packages/Revise/src/logging.jl b/packages/Revise/src/logging.jl new file mode 100644 index 0000000..4b545af --- /dev/null +++ b/packages/Revise/src/logging.jl @@ -0,0 +1,145 @@ +using Base.CoreLogging +using Base.CoreLogging: Info, Debug + +struct LogRecord + level + message + group + id + file + line + kwargs +end +LogRecord(args...; kwargs...) = LogRecord(args..., kwargs) + +mutable struct ReviseLogger <: AbstractLogger + logs::Vector{LogRecord} + min_level::LogLevel +end + +ReviseLogger(; min_level=Info) = ReviseLogger(LogRecord[], min_level) + +CoreLogging.min_enabled_level(logger::ReviseLogger) = logger.min_level + +CoreLogging.shouldlog(logger::ReviseLogger, level, _module, group, id) = _module == Revise + +function CoreLogging.handle_message(logger::ReviseLogger, level, msg, _module, + group, id, file, line; kwargs...) + rec = LogRecord(level, msg, group, id, file, line, kwargs) + push!(logger.logs, rec) + if level >= Info + if group == "lowered" && haskey(kwargs, :mod) && haskey(kwargs, :ex) && haskey(kwargs, :exception) + ex, bt = kwargs[:exception] + printstyled(stderr, msg; color=:red) + print(stderr, "\n ") + showerror(stderr, ex, bt; backtrace = bt!==nothing) + println(stderr, "\nwhile evaluating\n", kwargs[:ex], "\nin module ", kwargs[:mod]) + else + show(stderr, rec) + end + end +end + +CoreLogging.catch_exceptions(::ReviseLogger) = false + +function Base.show(io::IO, l::LogRecord; verbose::Bool=true) + if verbose + print(io, LogRecord) + print(io, '(', l.level, ", ", l.message, ", ", l.group, ", ", l.id, ", \"", l.file, "\", ", l.line) + else + printstyled(io, "Revise ", l.message, '\n'; color=Base.error_color()) + end + exc = nothing + if !isempty(l.kwargs) + verbose && print(io, ", (") + prefix = "" + for (kw, val) in l.kwargs + kw === :exception && (exc = val; continue) + verbose && print(io, prefix, kw, "=", val) + prefix = ", " + end + verbose && print(io, ')') + end + if exc !== nothing + ex, bt = exc + showerror(io, ex, bt; backtrace = bt!==nothing) + verbose || println(io) + end + verbose && println(io, ')') +end + +const _debug_logger = ReviseLogger() + +""" + logger = Revise.debug_logger(; min_level=Debug) + +Turn on [debug logging](https://docs.julialang.org/en/v1/stdlib/Logging/) +(if `min_level` is set to `Debug` or better) and return the logger object. +`logger.logs` contains a list of the logged events. The items in this list are of type `Revise.LogRecord`, +with the following relevant fields: + +- `group`: the event category. Revise currently uses the following groups: + + "Action": a change was implemented, of type described in the `message` field. + + "Parsing": a "significant" event in parsing. For these, examine the `message` field + for more information. + + "Watching": an indication that Revise determined that a particular file needed to be + examined for possible code changes. This is typically done on the basis of `mtime`, + the modification time of the file, and does not necessarily indicate that there were + any changes. +- `message`: a string containing more information. Some examples: + + For entries in the "Action" group, `message` can be `"Eval"` when modifying + old methods or defining new ones, "DeleteMethod" when deleting a method, + and "LineOffset" to indicate that the line offset for a method + was updated (the last only affects the printing of stacktraces upon error, + it does not change how code runs) + + Items with group "Parsing" and message "Diff" contain sets `:newexprs` and `:oldexprs` + that contain the expression unique to post- or pre-revision, respectively. +- `kwargs`: a pairs list of any other data. This is usually specific to particular `group`/`message` + combinations. + +See also [`Revise.actions`](@ref) and [`Revise.diffs`](@ref). +""" +function debug_logger(; min_level=Debug) + _debug_logger.min_level = min_level + return _debug_logger +end + +""" + actions(logger; line=false) + +Return a vector of all log events in the "Action" group. "LineOffset" events are returned +only if `line=true`; by default the returned items are the events that modified +methods in your session. +""" +function actions(logger::ReviseLogger; line=false) + filter(logger.logs) do r + r.group=="Action" && (line || r.message!="LineOffset") + end +end + +""" + diffs(logger) + +Return a vector of all log events that encode a (non-empty) diff between two versions of a file. +""" +function diffs(logger::ReviseLogger) + filter(logger.logs) do r + r.message=="Diff" && r.group=="Parsing" && (!isempty(r.kwargs[:newexprs]) || !isempty(r.kwargs[:oldexprs])) + end +end + +## Make the logs portable + +""" + MethodSummary(method) + +Create a portable summary of a method. In particular, a MethodSummary can be saved to a JLD2 file. +""" +struct MethodSummary + name::Symbol + modulename::Symbol + file::Symbol + line::Int32 + sig::Type +end +MethodSummary(m::Method) = MethodSummary(m.name, nameof(m.module), m.file, m.line, m.sig) diff --git a/packages/Revise/src/lowered.jl b/packages/Revise/src/lowered.jl new file mode 100644 index 0000000..bcf0df7 --- /dev/null +++ b/packages/Revise/src/lowered.jl @@ -0,0 +1,460 @@ +## Analyzing lowered code + +function add_docexpr!(docexprs::AbstractDict{Module,V}, mod::Module, ex) where V + docexs = get(docexprs, mod, nothing) + if docexs === nothing + docexs = docexprs[mod] = V() + end + push!(docexs, ex) + return docexprs +end + +function assign_this!(frame, value) + frame.framedata.ssavalues[frame.pc] = value +end + +# This defines the API needed to store signatures using methods_by_execution! +# This default version is simple and only used for testing purposes. +# The "real" one is CodeTrackingMethodInfo in Revise.jl. +const MethodInfo = IdDict{Type,LineNumberNode} +add_signature!(methodinfo::MethodInfo, @nospecialize(sig), ln) = push!(methodinfo, sig=>ln) +push_expr!(methodinfo::MethodInfo, mod::Module, ex::Expr) = methodinfo +pop_expr!(methodinfo::MethodInfo) = methodinfo +add_dependencies!(methodinfo::MethodInfo, be::CodeEdges, src, isrequired) = methodinfo +add_includes!(methodinfo::MethodInfo, mod::Module, filename) = methodinfo + +# This is not generally used, see `is_method_or_eval` instead +function hastrackedexpr(stmt; heads=LoweredCodeUtils.trackedheads) + haseval = false + if isa(stmt, Expr) + haseval = matches_eval(stmt) + if stmt.head === :call + f = stmt.args[1] + callee_matches(f, Core, :_typebody!) && return true, haseval + callee_matches(f, Core, :_setsuper!) && return true, haseval + f === :include && return true, haseval + elseif stmt.head === :thunk + any(s->any(hastrackedexpr(s; heads=heads)), (stmt.args[1]::Core.CodeInfo).code) && return true, haseval + elseif stmt.head ∈ heads + return true, haseval + end + end + return false, haseval +end + +function matches_eval(stmt::Expr) + stmt.head === :call || return false + f = stmt.args[1] + return f === :eval || + (callee_matches(f, Base, :getproperty) && is_quotenode_egal(stmt.args[end], :eval)) || + (isa(f, GlobalRef) && f.name === :eval) || is_quotenode_egal(f, Core.eval) +end + +function categorize_stmt(@nospecialize(stmt)) + ismeth, haseval, isinclude, isnamespace, istoplevel = false, false, false, false, false + if isa(stmt, Expr) + haseval = matches_eval(stmt) + ismeth = stmt.head === :method + istoplevel = stmt.head === :toplevel + isnamespace = stmt.head === :export || stmt.head === :import || stmt.head === :using + isinclude = stmt.head === :call && stmt.args[1] === :include + end + return ismeth, haseval, isinclude, isnamespace, istoplevel +end + +""" + isrequired, evalassign = minimal_evaluation!([predicate,] methodinfo, src::Core.CodeInfo, mode::Symbol) + +Mark required statements in `src`: `isrequired[i]` is `true` if `src.code[i]` should be evaluated. +Statements are analyzed by `isreq, haseval = predicate(stmt)`, and `predicate` defaults +to `Revise.is_method_or_eval`. +`haseval` is true if the statement came from `@eval` or `eval(...)` call. +Since the contents of such expression are difficult to analyze, it is generally +safest to execute all such evals. +""" +function minimal_evaluation!(@nospecialize(predicate), methodinfo, src::Core.CodeInfo, mode::Symbol) + edges = CodeEdges(src) + # LoweredCodeUtils.print_with_code(stdout, src, edges) + isrequired = fill(false, length(src.code)) + evalassign = false + for (i, stmt) in enumerate(src.code) + if !isrequired[i] + isrequired[i], haseval = predicate(stmt)::Tuple{Bool,Bool} + if haseval # line `i` may be the equivalent of `f = Core.eval`, so... + isrequired[edges.succs[i]] .= true # ...require each stmt that calls `eval` via `f(expr)` + isrequired[i] = true + end + end + if mode === :evalassign && isexpr(stmt, :(=)) + evalassign = isrequired[i] = true + lhs = (stmt::Expr).args[1] + if isa(lhs, Symbol) + isrequired[edges.byname[lhs].succs] .= true # mark any `const` statements or other "uses" in this block + end + end + end + # Check for docstrings + if length(src.code) > 1 && mode !== :sigs + stmt = src.code[end-1] + if isexpr(stmt, :call) && (stmt::Expr).args[1] === Base.Docs.doc! + isrequired[end-1] = true + end + end + # All tracked expressions are marked. Now add their dependencies. + # LoweredCodeUtils.print_with_code(stdout, src, isrequired) + lines_required!(isrequired, src, edges;) + # norequire=mode===:sigs ? LoweredCodeUtils.exclude_named_typedefs(src, edges) : ()) + # LoweredCodeUtils.print_with_code(stdout, src, isrequired) + add_dependencies!(methodinfo, edges, src, isrequired) + return isrequired, evalassign +end +minimal_evaluation!(@nospecialize(predicate), methodinfo, frame::JuliaInterpreter.Frame, mode::Symbol) = + minimal_evaluation!(predicate, methodinfo, frame.framecode.src, mode) + +function minimal_evaluation!(methodinfo, frame, mode::Symbol) + minimal_evaluation!(methodinfo, frame, mode) do @nospecialize(stmt) + ismeth, haseval, isinclude, isnamespace, istoplevel = categorize_stmt(stmt) + isreq = ismeth | isinclude | istoplevel + return mode === :sigs ? (isreq, haseval) : (isreq | isnamespace, haseval) + end +end + +function methods_by_execution(mod::Module, ex::Expr; kwargs...) + methodinfo = MethodInfo() + docexprs = DocExprs() + value, frame = methods_by_execution!(JuliaInterpreter.Compiled(), methodinfo, docexprs, mod, ex; kwargs...) + return methodinfo, docexprs, frame +end + +""" + methods_by_execution!(recurse=JuliaInterpreter.Compiled(), methodinfo, docexprs, mod::Module, ex::Expr; + mode=:eval, disablebp=true, skip_include=mode!==:eval, always_rethrow=false) + +Evaluate or analyze `ex` in the context of `mod`. +Depending on the setting of `mode` (see the Extended help), it supports full evaluation or just the minimal +evaluation needed to extract method signatures. +`recurse` controls JuliaInterpreter's evaluation of any non-intercepted statement; +likely choices are `JuliaInterpreter.Compiled()` or `JuliaInterpreter.finish_and_return!`. +`methodinfo` is a cache for storing information about any method definitions (see [`CodeTrackingMethodInfo`](@ref)). +`docexprs` is a cache for storing documentation expressions; obtain an empty one with `Revise.DocExprs()`. + +# Extended help + +The action depends on `mode`: + +- `:eval` evaluates the expression in `mod`, similar to `Core.eval(mod, ex)` except that `methodinfo` and `docexprs` + will be populated with information about any signatures or docstrings. This mode is used to implement `includet`. +- `:sigs` analyzes `ex` and extracts signatures of methods and docstrings (specifically, statements flagged by + [`Revise.minimal_evaluation!`](@ref)), but does not evaluate `ex` in the traditional sense. + It will selectively execute statements needed to form the signatures of defined methods. + It will also expand any `@eval`ed expressions, since these might contain method definitions. +- `:evalmeth` analyzes `ex` and extracts signatures and docstrings like `:sigs`, but takes the additional step of + evaluating any `:method` statements. +- `:evalassign` acts similarly to `:evalmeth`, and also evaluates assignment statements. + +When selectively evaluating an expression, Revise will incorporate required dependencies, even for +minimal-evaluation modes like `:sigs`. For example, the method definition + + max_values(T::Union{map(X -> Type{X}, Base.BitIntegerSmall_types)...}) = 1 << (8*sizeof(T)) + +found in `base/abstractset.jl` requires that it create the anonymous function in order to compute the +signature. + +The other keyword arguments are more straightforward: + +- `disablebp` controls whether JuliaInterpreter's breakpoints are disabled before stepping through the code. + They are restored on exit. +- `skip_include` prevents execution of `include` statements, instead inserting them into `methodinfo`'s + cache. This defaults to `true` unless `mode` is `:eval`. +- `always_rethrow`, if true, causes an error to be thrown if evaluating `ex` triggered an error. + If false, the error is logged with `@error`. `InterruptException`s are always rethrown. + This is primarily useful for debugging. +""" +function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, mod::Module, ex::Expr; + mode::Symbol=:eval, disablebp::Bool=true, always_rethrow::Bool=false, kwargs...) + mode ∈ (:sigs, :eval, :evalmeth, :evalassign) || error("unsupported mode ", mode) + lwr = Meta.lower(mod, ex) + isa(lwr, Expr) || return nothing, nothing + if lwr.head === :error || lwr.head === :incomplete + error("lowering returned an error, ", lwr) + end + if lwr.head !== :thunk + mode === :sigs && return nothing, nothing + return Core.eval(mod, lwr), nothing + end + frame = JuliaInterpreter.Frame(mod, lwr.args[1]::CodeInfo) + mode === :eval || LoweredCodeUtils.rename_framemethods!(recurse, frame) + # Determine whether we need interpreted mode + isrequired, evalassign = minimal_evaluation!(methodinfo, frame, mode) + # LoweredCodeUtils.print_with_code(stdout, frame.framecode.src, isrequired) + if !any(isrequired) && (mode===:eval || !evalassign) + # We can evaluate the entire expression in compiled mode + if mode===:eval + ret = try + Core.eval(mod, ex) + catch err + (always_rethrow || isa(err, InterruptException)) && rethrow(err) + loc = location_string(whereis(frame)...) + bt = trim_toplevel!(catch_backtrace()) + throw(ReviseEvalException(loc, err, Any[(sf, 1) for sf in stacktrace(bt)])) + end + else + ret = nothing + end + else + # Use the interpreter + local active_bp_refs + if disablebp + # We have to turn off all active breakpoints, https://github.com/timholy/CodeTracking.jl/issues/27 + bp_refs = JuliaInterpreter.BreakpointRef[] + for bp in JuliaInterpreter.breakpoints() + append!(bp_refs, bp.instances) + end + active_bp_refs = filter(bp->bp[].isactive, bp_refs) + foreach(disable, active_bp_refs) + end + ret = try + methods_by_execution!(recurse, methodinfo, docexprs, frame, isrequired; mode=mode, kwargs...) + catch err + (always_rethrow || isa(err, InterruptException)) && (disablebp && foreach(enable, active_bp_refs); rethrow(err)) + loc = location_string(whereis(frame)...) + sfs = [] # crafted for interaction with Base.show_backtrace + frame = JuliaInterpreter.leaf(frame) + while frame !== nothing + push!(sfs, (Base.StackTraces.StackFrame(frame), 1)) + frame = frame.caller + end + throw(ReviseEvalException(loc, err, sfs)) + end + if disablebp + foreach(enable, active_bp_refs) + end + end + return ret, lwr +end +methods_by_execution!(methodinfo, docexprs, mod::Module, ex::Expr; kwargs...) = + methods_by_execution!(JuliaInterpreter.Compiled(), methodinfo, docexprs, mod, ex; kwargs...) + +function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, frame::Frame, isrequired::AbstractVector{Bool}; mode::Symbol=:eval, skip_include::Bool=true) + isok(lnn::LineTypes) = !iszero(lnn.line) || lnn.file !== :none # might fail either one, but accept anything + + mod = moduleof(frame) + # Hoist this lookup for performance. Don't throw even when `mod` is a baremodule: + modinclude = isdefined(mod, :include) ? getfield(mod, :include) : nothing + signatures = [] # temporary for method signature storage + pc = frame.pc + while true + JuliaInterpreter.is_leaf(frame) || (@warn("not a leaf"); break) + stmt = pc_expr(frame, pc) + if !isrequired[pc] && mode !== :eval && !(mode === :evalassign && isexpr(stmt, :(=))) + pc = next_or_nothing!(frame) + pc === nothing && break + continue + end + if isa(stmt, Expr) + head = stmt.head + if head === :toplevel + local value + for ex in stmt.args + ex isa Expr || continue + value = methods_by_execution!(recurse, methodinfo, docexprs, mod, ex; mode=mode, disablebp=false, skip_include=skip_include) + end + isassign(frame, pc) && assign_this!(frame, value) + pc = next_or_nothing!(frame) + # elseif head === :thunk && isanonymous_typedef(stmt.args[1]) + # # Anonymous functions should just be defined anew, since there does not seem to be a practical + # # way to find them within the already-defined module. + # # They may be needed to define later signatures. + # # Note that named inner methods don't require special treatment. + # pc = step_expr!(recurse, frame, stmt, true) + elseif head === :method + empty!(signatures) + ret = methoddef!(recurse, signatures, frame, stmt, pc; define=mode!==:sigs) + if ret === nothing + # This was just `function foo end` or similar. + # However, it might have been followed by a thunk that defined a + # method (issue #435), so we still need to check for additions. + if !isempty(signatures) + file, line = whereis(frame.framecode, pc) + lnn = LineNumberNode(Int(line), Symbol(file)) + for sig in signatures + add_signature!(methodinfo, sig, lnn) + end + end + pc = ret + else + pc, pc3 = ret + # Get the line number from the body + stmt3 = pc_expr(frame, pc3)::Expr + lnn = nothing + if line_is_decl + sigcode = @lookup(frame, stmt3.args[2])::Core.SimpleVector + lnn = sigcode[end] + if !isa(lnn, LineNumberNode) + lnn = nothing + end + end + if lnn === nothing + bodycode = stmt3.args[end] + if !isa(bodycode, CodeInfo) + bodycode = @lookup(frame, bodycode) + end + if isa(bodycode, CodeInfo) + lnn = linetable(bodycode, 1) + if !isok(lnn) + lnn = nothing + if length(bodycode.code) > 1 + # This may be a kwarg method. Mimic LoweredCodeUtils.bodymethod, + # except without having a method + stmt = bodycode.code[end-1] + if isa(stmt, Expr) && length(stmt.args) > 1 + stmt = stmt::Expr + a = stmt.args[1] + nargs = length(stmt.args) + hasself = let stmt = stmt, slotnames::Vector{Symbol} = bodycode.slotnames + any(i->LoweredCodeUtils.is_self_call(stmt, slotnames, i), 2:nargs) + end + if isa(a, Core.SlotNumber) + a = bodycode.slotnames[a.id] + end + if hasself && (isa(a, Symbol) || isa(a, GlobalRef)) + thismod, thisname = isa(a, Symbol) ? (mod, a) : (a.mod, a.name) + if isdefined(thismod, thisname) + f = getfield(thismod, thisname) + mths = methods(f) + if length(mths) == 1 + mth = first(mths) + lnn = LineNumberNode(Int(mth.line), mth.file) + end + end + end + end + end + if lnn === nothing + # Just try to find *any* line number + for lnntmp in linetable(bodycode) + lnntmp = lnntmp::LineTypes + if isok(lnntmp) + lnn = lnntmp + break + end + end + end + end + elseif isexpr(bodycode, :lambda) + bodycode = bodycode::Expr + lnntmp = bodycode.args[end][1]::LineTypes + if isok(lnntmp) + lnn = lnntmp + end + end + end + if lnn === nothing + i = codelocs(frame, pc3) + while i > 0 + lnntmp = linetable(frame, i) + if isok(lnntmp) + lnn = lnntmp + break + end + i -= 1 + end + end + if lnn !== nothing && isok(lnn) + for sig in signatures + add_signature!(methodinfo, sig, lnn) + end + end + end + elseif head === :(=) + # If we're here, either isrequired[pc] is true, or the mode forces us to eval assignments + pc = step_expr!(recurse, frame, stmt, true) + elseif head === :call + f = @lookup(frame, stmt.args[1]) + if f === Core.eval + # an @eval or eval block: this may contain method definitions, so intercept it. + evalmod = @lookup(frame, stmt.args[2])::Module + evalex = @lookup(frame, stmt.args[3]) + value = nothing + for (newmod, newex) in ExprSplitter(evalmod, evalex) + if is_doc_expr(newex) + add_docexpr!(docexprs, newmod, newex) + newex = newex.args[4] + end + newex = unwrap(newex) + push_expr!(methodinfo, newmod, newex) + value = methods_by_execution!(recurse, methodinfo, docexprs, newmod, newex; mode=mode, skip_include=skip_include, disablebp=false) + pop_expr!(methodinfo) + end + assign_this!(frame, value) + pc = next_or_nothing!(frame) + elseif skip_include && (f === modinclude || f === Core.include) + # include calls need to be managed carefully from several standpoints, including + # path management and parsing new expressions + if length(stmt.args) == 2 + add_includes!(methodinfo, mod, @lookup(frame, stmt.args[2])) + else + error("include(mapexpr, path) is not supported") # TODO (issue #634) + end + assign_this!(frame, nothing) # FIXME: the file might return something different from `nothing` + pc = next_or_nothing!(frame) + elseif skip_include && f === Base.include + if length(stmt.args) == 2 + add_includes!(methodinfo, mod, @lookup(frame, stmt.args[2])) + else # either include(module, path) or include(mapexpr, path) + mod_or_mapexpr = @lookup(frame, stmt.args[2]) + if isa(mod_or_mapexpr, Module) + add_includes!(methodinfo, mod_or_mapexpr, @lookup(frame, stmt.args[3])) + else + error("include(mapexpr, path) is not supported") + end + end + assign_this!(frame, nothing) # FIXME: the file might return something different from `nothing` + pc = next_or_nothing!(frame) + elseif f === Base.Docs.doc! # && mode !== :eval + fargs = JuliaInterpreter.collect_args(recurse, frame, stmt) + popfirst!(fargs) + length(fargs) == 3 && push!(fargs, Union{}) # add the default sig + dmod::Module, b::Base.Docs.Binding, str::Base.Docs.DocStr, sig = fargs + if isdefined(b.mod, b.var) + tmpvar = getfield(b.mod, b.var) + if isa(tmpvar, Module) + dmod = tmpvar + end + end + # Workaround for julia#38819 on older Julia versions + if !isdefined(dmod, Base.Docs.META) + Base.Docs.initmeta(dmod) + end + m = get!(Base.Docs.meta(dmod), b, Base.Docs.MultiDoc())::Base.Docs.MultiDoc + if haskey(m.docs, sig) + currentstr = m.docs[sig]::Base.Docs.DocStr + redefine = currentstr.text != str.text + else + push!(m.order, sig) + redefine = true + end + # (Re)assign without the warning + if redefine + m.docs[sig] = str + str.data[:binding] = b + str.data[:typesig] = sig + end + assign_this!(frame, Base.Docs.doc(b, sig)) + pc = next_or_nothing!(frame) + else + # A :call Expr we don't want to intercept + pc = step_expr!(recurse, frame, stmt, true) + end + else + # An Expr we don't want to intercept + pc = step_expr!(recurse, frame, stmt, true) + end + else + # A statement we don't want to intercept + pc = step_expr!(recurse, frame, stmt, true) + end + pc === nothing && break + end + return isrequired[frame.pc] ? get_return(frame) : nothing +end diff --git a/packages/Revise/src/packagedef.jl b/packages/Revise/src/packagedef.jl new file mode 100644 index 0000000..4890992 --- /dev/null +++ b/packages/Revise/src/packagedef.jl @@ -0,0 +1,1389 @@ +@eval Base.Experimental.@optlevel 1 + +using FileWatching, REPL, Distributed, UUIDs, Pkg +import LibGit2 +using Base: PkgId +using Base.Meta: isexpr +using Core: CodeInfo + +export revise, includet, entr, MethodSummary + +""" + Revise.watching_files[] + +Returns `true` if we watch files rather than their containing directory. +FreeBSD and NFS-mounted systems should watch files, otherwise we prefer to watch +directories. +""" +const watching_files = Ref(Sys.KERNEL === :FreeBSD) + +""" + Revise.polling_files[] + +Returns `true` if we should poll the filesystem for changes to the files that define +loaded code. It is preferable to avoid polling, instead relying on operating system +notifications via `FileWatching.watch_file`. However, NFS-mounted +filesystems (and perhaps others) do not support file-watching, so for code stored +on such filesystems you should turn polling on. + +See the documentation for the `JULIA_REVISE_POLL` environment variable. +""" +const polling_files = Ref(false) +function wait_changed(file) + try + polling_files[] ? poll_file(file) : watch_file(file) + catch err + if Sys.islinux() && err isa Base.IOError && err.code == -28 # ENOSPC + @warn """Your operating system has run out of inotify capacity. + Check the current value with `cat /proc/sys/fs/inotify/max_user_watches`. + Set it to a higher level with, e.g., `echo 65536 | sudo tee -a /proc/sys/fs/inotify/max_user_watches`. + This requires having administrative privileges on your machine (or talk to your sysadmin). + See https://github.com/timholy/Revise.jl/issues/26 for more information.""" + end + rethrow(err) + end + return nothing +end + +""" + Revise.tracking_Main_includes[] + +Returns `true` if files directly included from the REPL should be tracked. +The default is `false`. See the documentation regarding the `JULIA_REVISE_INCLUDE` +environment variable to customize it. +""" +const tracking_Main_includes = Ref(false) + +include("relocatable_exprs.jl") +include("types.jl") +include("utils.jl") +include("parsing.jl") +include("lowered.jl") +include("pkgs.jl") +include("git.jl") +include("recipes.jl") +include("logging.jl") +include("callbacks.jl") + +### Globals to keep track of state + +""" + Revise.watched_files + +Global variable, `watched_files[dirname]` returns the collection of files in `dirname` +that we're monitoring for changes. The returned value has type [`Revise.WatchList`](@ref). + +This variable allows us to watch directories rather than files, reducing the burden on +the OS. +""" +const watched_files = Dict{String,WatchList}() + +""" + Revise.watched_manifests + +Global variable, a set of `Manifest.toml` files from the active projects used during this session. +""" +const watched_manifests = Set{String}() + +""" + Revise.revision_queue + +Global variable, `revision_queue` holds `(pkgdata,filename)` pairs that we need to revise, meaning +that these files have changed since we last processed a revision. +This list gets populated by callbacks that watch directories for updates. +""" +const revision_queue = Set{Tuple{PkgData,String}}() + +""" + Revise.queue_errors + +Global variable, maps `(pkgdata, filename)` pairs that errored upon last revision to +`(exception, backtrace)`. +""" +const queue_errors = Dict{Tuple{PkgData,String},Tuple{Exception, Any}}() + +""" + Revise.NOPACKAGE + +Global variable; default `PkgId` used for files which do not belong to any +package, but still have to be watched because user callbacks have been +registered for them. +""" +const NOPACKAGE = PkgId(nothing, "") + +""" + Revise.pkgdatas + +`pkgdatas` is the core information that tracks the relationship between source code +and julia objects, and allows re-evaluation of code in the proper module scope. +It is a dictionary indexed by PkgId: +`pkgdatas[id]` returns a value of type [`Revise.PkgData`](@ref). +""" +const pkgdatas = Dict{PkgId,PkgData}(NOPACKAGE => PkgData(NOPACKAGE)) + +const moduledeps = Dict{Module,DepDict}() +function get_depdict(mod::Module) + if !haskey(moduledeps, mod) + moduledeps[mod] = DepDict() + end + return moduledeps[mod] +end + +""" + Revise.included_files + +Global variable, `included_files` gets populated by callbacks we register with `include`. +It's used to track non-precompiled packages and, optionally, user scripts (see docs on +`JULIA_REVISE_INCLUDE`). +""" +const included_files = Tuple{Module,String}[] # (module, filename) + +""" + Revise.basesrccache + +Full path to the running Julia's cache of source code defining `Base`. +""" +const basesrccache = normpath(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "base.cache")) + +""" + Revise.basebuilddir + +Julia's top-level directory when Julia was built, as recorded by the entries in +`Base._included_files`. +""" +const basebuilddir = begin + sysimg = filter(x->endswith(x[2], "sysimg.jl"), Base._included_files)[1][2] + dirname(dirname(sysimg)) +end + +function fallback_juliadir() + candidate = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia") + if !isdir(joinpath(candidate, "base")) + while true + trydir = joinpath(candidate, "base") + isdir(trydir) && break + trydir = joinpath(candidate, "share", "julia", "base") + if isdir(trydir) + candidate = joinpath(candidate, "share", "julia") + break + end + next_candidate = dirname(candidate) + next_candidate == candidate && break + candidate = next_candidate + end + end + normpath(candidate) +end + +""" + Revise.juliadir + +Constant specifying full path to julia top-level source directory. +This should be reliable even for local builds, cross-builds, and binary installs. +""" +const juliadir = normpath( + if isdir(joinpath(basebuilddir, "base")) + basebuilddir + else + fallback_juliadir() # Binaries probably end up here. We fall back on Sys.BINDIR + end +) + +const cache_file_key = Dict{String,String}() # corrected=>uncorrected filenames +const src_file_key = Dict{String,String}() # uncorrected=>corrected filenames + +""" + Revise.dont_watch_pkgs + +Global variable, use `push!(Revise.dont_watch_pkgs, :MyPackage)` to prevent Revise +from tracking changes to `MyPackage`. You can do this from the REPL or from your +`.julia/config/startup.jl` file. + +See also [`Revise.silence`](@ref). +""" +const dont_watch_pkgs = Set{Symbol}() +const silence_pkgs = Set{Symbol}() +const depsdir = joinpath(dirname(@__DIR__), "deps") +const silencefile = Ref(joinpath(depsdir, "silence.txt")) # Ref so that tests don't clobber + +## +## The inputs are sets of expressions found in each file. +## Some of those expressions will generate methods which are identified via their signatures. +## From "old" expressions we know their corresponding signatures, but from "new" +## expressions we have not yet computed them. This makes old and new asymmetric. +## +## Strategy: +## - For every old expr not found in the new ones, +## + delete the corresponding methods (using the signatures we've previously computed) +## + remove the sig entries from CodeTracking.method_info (") +## Best to do all the deletion first (across all files and modules) in case a method is +## simply being moved from one file to another. +## - For every new expr found among the old ones, +## + update the location info in CodeTracking.method_info +## - For every new expr not found in the old ones, +## + eval the expr +## + extract signatures +## + add to the ModuleExprsSigs +## + add to CodeTracking.method_info +## +## Interestingly, the ex=>sigs link may not be the same as the sigs=>ex link. +## Consider a conditional block, +## if Sys.islinux() +## f() = 1 +## g() = 2 +## else +## g() = 3 +## end +## From the standpoint of Revise's diff-and-patch functionality, we should look for +## diffs in this entire block. (Really good backedge support---or a variant of `lower` that +## links back to the specific expression---might change this, but for +## now this is the right strategy.) From the standpoint of CodeTracking, we should +## link the signature to the actual method-defining expression (either :(f() = 1) or :(g() = 2)). + +get_method_from_match(mm::Core.MethodMatch) = mm.method + +function delete_missing!(exs_sigs_old::ExprsSigs, exs_sigs_new) + with_logger(_debug_logger) do + for (ex, sigs) in exs_sigs_old + haskey(exs_sigs_new, ex) && continue + # ex was deleted + sigs === nothing && continue + for sig in sigs + ret = Base._methods_by_ftype(sig, -1, typemax(UInt)) + success = false + if !isempty(ret) + m = get_method_from_match(ret[end]) # the last method returned is the least-specific that matches, and thus most likely to be type-equal + methsig = m.sig + if sig <: methsig && methsig <: sig + locdefs = get(CodeTracking.method_info, sig, nothing) + if isa(locdefs, Vector{Tuple{LineNumberNode,Expr}}) + if length(locdefs) > 1 + # Just delete this reference but keep the method + line = firstline(ex) + ld = map(pr->linediff(line, pr[1]), locdefs) + idx = argmin(ld) + @assert ld[idx] < typemax(eltype(ld)) + deleteat!(locdefs, idx) + continue + else + @assert length(locdefs) == 1 + end + end + @debug "DeleteMethod" _group="Action" time=time() deltainfo=(sig, MethodSummary(m)) + # Delete the corresponding methods + for p in workers() + try # guard against serialization errors if the type isn't defined on the worker + remotecall(Core.eval, p, Main, :(delete_method_by_sig($sig))) + catch + end + end + Base.delete_method(m) + # Remove the entries from CodeTracking data + delete!(CodeTracking.method_info, sig) + # Remove frame from JuliaInterpreter, if applicable. Otherwise debuggers + # may erroneously work with outdated code (265-like problems) + if haskey(JuliaInterpreter.framedict, m) + delete!(JuliaInterpreter.framedict, m) + end + if isdefined(m, :generator) + # defensively delete all generated functions + empty!(JuliaInterpreter.genframedict) + end + success = true + end + end + if !success + @debug "FailedDeletion" _group="Action" time=time() deltainfo=(sig,) + end + end + end + end + return exs_sigs_old +end + +const empty_exs_sigs = ExprsSigs() +function delete_missing!(mod_exs_sigs_old::ModuleExprsSigs, mod_exs_sigs_new) + for (mod, exs_sigs_old) in mod_exs_sigs_old + exs_sigs_new = get(mod_exs_sigs_new, mod, empty_exs_sigs) + delete_missing!(exs_sigs_old, exs_sigs_new) + end + return mod_exs_sigs_old +end + +function eval_rex(rex::RelocatableExpr, exs_sigs_old::ExprsSigs, mod::Module; mode::Symbol=:eval) + sigs, includes = nothing, nothing + with_logger(_debug_logger) do + rexo = getkey(exs_sigs_old, rex, nothing) + # extract the signatures and update the line info + if rexo === nothing + ex = rex.ex + # ex is not present in old + @debug "Eval" _group="Action" time=time() deltainfo=(mod, ex) + sigs, deps, includes, thunk = eval_with_signatures(mod, ex; mode=mode) # All signatures defined by `ex` + if !isexpr(thunk, :thunk) + thunk = ex + end + if myid() == 1 + for p in workers() + p == myid() && continue + try # don't error if `mod` isn't defined on the worker + remotecall(Core.eval, p, mod, thunk) + catch + end + end + end + storedeps(deps, rex, mod) + else + sigs = exs_sigs_old[rexo] + # Update location info + ln, lno = firstline(unwrap(rex)), firstline(unwrap(rexo)) + if sigs !== nothing && !isempty(sigs) && ln != lno + @debug "LineOffset" _group="Action" time=time() deltainfo=(sigs, lno=>ln) + for sig in sigs + locdefs = CodeTracking.method_info[sig] + ld = map(pr->linediff(lno, pr[1]), locdefs) + idx = argmin(ld) + if ld[idx] === typemax(eltype(ld)) + # println("Missing linediff for $lno and $(first.(locdefs)) with ", rex.ex) + idx = length(locdefs) + end + methloc, methdef = locdefs[idx] + locdefs[idx] = (newloc(methloc, ln, lno), methdef) + end + end + end + end + return sigs, includes +end + +# These are typically bypassed in favor of expression-by-expression evaluation to +# allow handling of new `include` statements. +function eval_new!(exs_sigs_new::ExprsSigs, exs_sigs_old, mod::Module; mode::Symbol=:eval) + includes = Vector{Pair{Module,String}}() + for rex in keys(exs_sigs_new) + sigs, _includes = eval_rex(rex, exs_sigs_old, mod; mode=mode) + if sigs !== nothing + exs_sigs_new[rex] = sigs + end + if _includes !== nothing + append!(includes, _includes) + end + end + return exs_sigs_new, includes +end + +function eval_new!(mod_exs_sigs_new::ModuleExprsSigs, mod_exs_sigs_old; mode::Symbol=:eval) + includes = Vector{Pair{Module,String}}() + for (mod, exs_sigs_new) in mod_exs_sigs_new + # Allow packages to override the supplied mode + if isdefined(mod, :__revise_mode__) + mode = getfield(mod, :__revise_mode__)::Symbol + end + exs_sigs_old = get(mod_exs_sigs_old, mod, empty_exs_sigs) + _, _includes = eval_new!(exs_sigs_new, exs_sigs_old, mod; mode=mode) + append!(includes, _includes) + end + return mod_exs_sigs_new, includes +end + +""" + CodeTrackingMethodInfo(ex::Expr) + +Create a cache for storing information about method definitions. +Adding signatures to such an object inserts them into `CodeTracking.method_info`, +which maps signature Tuple-types to `(lnn::LineNumberNode, ex::Expr)` pairs. +Because method signatures are unique within a module, this is the foundation for +identifying methods in a manner independent of source-code location. + +It also has the following fields: + +- `exprstack`: used when descending into `@eval` statements (via `push_expr` and `pop_expr!`) + `ex` (used in creating the `CodeTrackingMethodInfo` object) is the first entry in the stack. +- `allsigs`: a list of all method signatures defined by a given expression +- `deps`: list of top-level named objects (`Symbol`s and `GlobalRef`s) that method definitions + in this block depend on. For example, `if Sys.iswindows() f() = 1 else f() = 2 end` would + store `Sys.iswindows` here. +- `includes`: a list of `module=>filename` for any `include` statements encountered while the + expression was parsed. +""" +struct CodeTrackingMethodInfo + exprstack::Vector{Expr} + allsigs::Vector{Any} + deps::Set{Union{GlobalRef,Symbol}} + includes::Vector{Pair{Module,String}} +end +CodeTrackingMethodInfo(ex::Expr) = CodeTrackingMethodInfo([ex], Any[], Set{Union{GlobalRef,Symbol}}(), Pair{Module,String}[]) + +function add_signature!(methodinfo::CodeTrackingMethodInfo, @nospecialize(sig), ln) + locdefs = CodeTracking.invoked_get!(Vector{Tuple{LineNumberNode,Expr}}, CodeTracking.method_info, sig) + newdef = unwrap(methodinfo.exprstack[end]) + if newdef !== nothing + if !any(locdef->locdef[1] == ln && isequal(RelocatableExpr(locdef[2]), RelocatableExpr(newdef)), locdefs) + push!(locdefs, (fixpath(ln), newdef)) + end + push!(methodinfo.allsigs, sig) + end + return methodinfo +end +push_expr!(methodinfo::CodeTrackingMethodInfo, mod::Module, ex::Expr) = (push!(methodinfo.exprstack, ex); methodinfo) +pop_expr!(methodinfo::CodeTrackingMethodInfo) = (pop!(methodinfo.exprstack); methodinfo) +function add_dependencies!(methodinfo::CodeTrackingMethodInfo, edges::CodeEdges, src, musteval) + isempty(src.code) && return methodinfo + stmt1 = first(src.code) + if (isexpr(stmt1, :gotoifnot) && (dep = (stmt1::Expr).args[1]; isa(dep, Union{GlobalRef,Symbol}))) || + (is_GotoIfNot(stmt1) && (dep = stmt1.cond; isa(dep, Union{GlobalRef,Symbol}))) + # This is basically a hack to look for symbols that control definition of methods via a conditional. + # It is aimed at solving #249, but this will have to be generalized for anything real. + for (stmt, me) in zip(src.code, musteval) + me || continue + if hastrackedexpr(stmt)[1] + push!(methodinfo.deps, dep) + break + end + end + end + # for (dep, lines) in be.byname + # for ln in lines + # stmt = src.code[ln] + # if isexpr(stmt, :(=)) && stmt.args[1] == dep + # continue + # else + # push!(methodinfo.deps, dep) + # end + # end + # end + return methodinfo +end +function add_includes!(methodinfo::CodeTrackingMethodInfo, mod::Module, filename) + push!(methodinfo.includes, mod=>filename) + return methodinfo +end + +# Eval and insert into CodeTracking data +function eval_with_signatures(mod, ex::Expr; mode=:eval, kwargs...) + methodinfo = CodeTrackingMethodInfo(ex) + docexprs = DocExprs() + frame = methods_by_execution!(finish_and_return!, methodinfo, docexprs, mod, ex; mode=mode, kwargs...)[2] + return methodinfo.allsigs, methodinfo.deps, methodinfo.includes, frame +end + +function instantiate_sigs!(modexsigs::ModuleExprsSigs; mode=:sigs, kwargs...) + for (mod, exsigs) in modexsigs + for rex in keys(exsigs) + is_doc_expr(rex.ex) && continue + sigs, deps, _ = eval_with_signatures(mod, rex.ex; mode=mode, kwargs...) + exsigs[rex] = sigs + storedeps(deps, rex, mod) + end + end + return modexsigs +end + +function storedeps(deps, rex, mod) + for dep in deps + if isa(dep, GlobalRef) + haskey(moduledeps, dep.mod) || continue + ddict, sym = get_depdict(dep.mod), dep.name + else + ddict, sym = get_depdict(mod), dep + end + if !haskey(ddict, sym) + ddict[sym] = Set{DepDictVals}() + end + push!(ddict[sym], (mod, rex)) + end + return rex +end + +# This is intended for testing purposes, but not general use. The key problem is +# that it doesn't properly handle methods that move from one file to another; there is the +# risk you could end up deleting the method altogether depending on the order in which you +# process these. +# See `revise` for the proper approach. +function eval_revised(mod_exs_sigs_new, mod_exs_sigs_old) + delete_missing!(mod_exs_sigs_old, mod_exs_sigs_new) + eval_new!(mod_exs_sigs_new, mod_exs_sigs_old) # note: drops `includes` + instantiate_sigs!(mod_exs_sigs_new) +end + +""" + Revise.init_watching(files) + Revise.init_watching(pkgdata::PkgData, files) + +For every filename in `files`, monitor the filesystem for updates. When the file is +updated, either [`Revise.revise_dir_queued`](@ref) or [`Revise.revise_file_queued`](@ref) will +be called. + +Use the `pkgdata` version if the files are supplied using relative paths. +""" +function init_watching(pkgdata::PkgData, files=srcfiles(pkgdata)) + udirs = Set{String}() + for file in files + file = String(file)::String + dir, basename = splitdir(file) + dirfull = joinpath(basedir(pkgdata), dir) + already_watching_dir = haskey(watched_files, dirfull) + already_watching_dir || (watched_files[dirfull] = WatchList()) + watchlist = watched_files[dirfull] + current_id = get(watchlist.trackedfiles, basename, nothing) + new_id = pkgdata.info.id + if new_id != NOPACKAGE || current_id === nothing + # Allow the package id to be updated + push!(watchlist, basename=>pkgdata) + if watching_files[] + fwatcher = TaskThunk(revise_file_queued, (pkgdata, file)) + schedule(Task(fwatcher)) + else + already_watching_dir || push!(udirs, dir) + end + end + end + for dir in udirs + dirfull = joinpath(basedir(pkgdata), dir) + updatetime!(watched_files[dirfull]) + if !watching_files[] + dwatcher = TaskThunk(revise_dir_queued, (dirfull,)) + schedule(Task(dwatcher)) + end + end + return nothing +end +init_watching(files) = init_watching(pkgdatas[NOPACKAGE], files) + +""" + revise_dir_queued(dirname) + +Wait for one or more of the files registered in `Revise.watched_files[dirname]` to be +modified, and then queue the corresponding files on [`Revise.revision_queue`](@ref). +This is generally called via a [`Revise.TaskThunk`](@ref). +""" +@noinline function revise_dir_queued(dirname) + @assert isabspath(dirname) + if !isdir(dirname) + sleep(0.1) # in case git has done a delete/replace cycle + end + stillwatching = true + while stillwatching + if !isdir(dirname) + with_logger(SimpleLogger(stderr)) do + @warn "$dirname is not an existing directory, Revise is not watching" + end + break + end + + latestfiles, stillwatching = watch_files_via_dir(dirname) # will block here until file(s) change + for (file, id) in latestfiles + key = joinpath(dirname, file) + if key in keys(user_callbacks_by_file) + union!(user_callbacks_queue, user_callbacks_by_file[key]) + notify(revision_event) + end + if id != NOPACKAGE + pkgdata = pkgdatas[id] + if hasfile(pkgdata, key) # issue #228 + push!(revision_queue, (pkgdata, relpath(key, pkgdata))) + notify(revision_event) + end + end + end + end + return +end + +# See #66. +""" + revise_file_queued(pkgdata::PkgData, filename) + +Wait for modifications to `filename`, and then queue the corresponding files on [`Revise.revision_queue`](@ref). +This is generally called via a [`Revise.TaskThunk`](@ref). + +This is used only on platforms (like BSD) which cannot use [`Revise.revise_dir_queued`](@ref). +""" +function revise_file_queued(pkgdata::PkgData, file) + if !isabspath(file) + file = joinpath(basedir(pkgdata), file) + end + if !file_exists(file) + sleep(0.1) # in case git has done a delete/replace cycle + end + + dirfull, basename = splitdir(file) + stillwatching = true + while stillwatching + if !file_exists(file) && !isdir(file) + let file=file + with_logger(SimpleLogger(stderr)) do + @warn "$file is not an existing file, Revise is not watching" + end + end + notify(revision_event) + break + end + try + wait_changed(file) # will block here until the file changes + catch e + # issue #459 + (isa(e, InterruptException) && throwto_repl(e)) || throw(e) + end + + if file in keys(user_callbacks_by_file) + union!(user_callbacks_queue, user_callbacks_by_file[file]) + notify(revision_event) + end + + # Check to see if we're still watching this file + stillwatching = haskey(watched_files, dirfull) + PkgId(pkgdata) != NOPACKAGE && push!(revision_queue, (pkgdata, relpath(file, pkgdata))) + end + return +end + +# Because we delete first, we have to make sure we've parsed the file +function handle_deletions(pkgdata, file) + fi = maybe_parse_from_cache!(pkgdata, file) + maybe_extract_sigs!(fi) + mexsold = fi.modexsigs + idx = fileindex(pkgdata, file) + filep = pkgdata.info.files[idx] + if isa(filep, AbstractString) + filep = normpath(joinpath(basedir(pkgdata), file)) + end + topmod = first(keys(mexsold)) + fileok = file_exists(String(filep)::String) + mexsnew = fileok ? parse_source(filep, topmod) : ModuleExprsSigs(topmod) + if mexsnew !== nothing + delete_missing!(mexsold, mexsnew) + end + if !fileok + @warn("$filep no longer exists, deleted all methods") + deleteat!(pkgdata.fileinfos, idx) + deleteat!(pkgdata.info.files, idx) + wl = get(watched_files, basedir(pkgdata), nothing) + if isa(wl, WatchList) + delete!(wl.trackedfiles, file) + end + end + return mexsnew, mexsold +end + +""" + Revise.revise_file_now(pkgdata::PkgData, file) + +Process revisions to `file`. This parses `file` and computes an expression-level diff +between the current state of the file and its most recently evaluated state. +It then deletes any removed methods and re-evaluates any changed expressions. +Note that generally it is better to use [`revise`](@ref) as it properly handles methods +that move from one file to another. + +`id` must be a key in [`Revise.pkgdatas`](@ref), and `file` a key in +`Revise.pkgdatas[id].fileinfos`. +""" +function revise_file_now(pkgdata::PkgData, file) + # @assert !isabspath(file) + i = fileindex(pkgdata, file) + if i === nothing + println("Revise is currently tracking the following files in $(PkgId(pkgdata)): ", srcfiles(pkgdata)) + error(file, " is not currently being tracked.") + end + mexsnew, mexsold = handle_deletions(pkgdata, file) + if mexsnew != nothing + _, includes = eval_new!(mexsnew, mexsold) + fi = fileinfo(pkgdata, i) + pkgdata.fileinfos[i] = FileInfo(mexsnew, fi) + maybe_add_includes_to_pkgdata!(pkgdata, file, includes; eval_now=true) + end + nothing +end + +""" + Revise.errors() + +Report the errors represented in [`Revise.queue_errors`](@ref). +Errors are automatically reported the first time they are encountered, but this function +can be used to report errors again. +""" +function errors(revision_errors=keys(queue_errors)) + printed = Set{eltype(revision_errors)}() + for item in revision_errors + item in printed && continue + push!(printed, item) + pkgdata, file = item + (err, bt) = queue_errors[(pkgdata, file)] + fullpath = joinpath(basedir(pkgdata), file) + if err isa ReviseEvalException + @error "Failed to revise $fullpath" exception=err + else + @error "Failed to revise $fullpath" exception=(err, trim_toplevel!(bt)) + end + end +end + +""" + Revise.retry() + +Attempt to perform previously-failed revisions. This can be useful in cases of order-dependent errors. +""" +function retry() + for (k, v) in queue_errors + push!(revision_queue, k) + end + revise() +end + +""" + revise(; throw=false) + +`eval` any changes in the revision queue. See [`Revise.revision_queue`](@ref). +If `throw` is `true`, throw any errors that occur during revision or callback; +otherwise these are only logged. +""" +function revise(; throw=false) + sleep(0.01) # in case the file system isn't quite done writing out the new files + have_queue_errors = !isempty(queue_errors) + + # Do all the deletion first. This ensures that a method that moved from one file to another + # won't get redefined first and deleted second. + revision_errors = Tuple{PkgData,String}[] + queue = sort!(collect(revision_queue); lt=pkgfileless) + finished = eltype(revision_queue)[] + mexsnews = ModuleExprsSigs[] + interrupt = false + for (pkgdata, file) in queue + try + push!(mexsnews, handle_deletions(pkgdata, file)[1]) + push!(finished, (pkgdata, file)) + catch err + throw && Base.throw(err) + interrupt |= isa(err, InterruptException) + push!(revision_errors, (pkgdata, file)) + queue_errors[(pkgdata, file)] = (err, catch_backtrace()) + end + end + # Do the evaluation + for ((pkgdata, file), mexsnew) in zip(finished, mexsnews) + defaultmode = PkgId(pkgdata).name == "Main" ? :evalmeth : :eval + i = fileindex(pkgdata, file) + i === nothing && continue # file was deleted by `handle_deletions` + fi = fileinfo(pkgdata, i) + modsremaining = Set(keys(mexsnew)) + changed, err = true, nothing + while changed + changed = false + for (mod, exsnew) in mexsnew + mod ∈ modsremaining || continue + try + mode = defaultmode + # Allow packages to override the supplied mode + if isdefined(mod, :__revise_mode__) + mode = getfield(mod, :__revise_mode__)::Symbol + end + mode ∈ (:sigs, :eval, :evalmeth, :evalassign) || error("unsupported mode ", mode) + exsold = get(fi.modexsigs, mod, empty_exs_sigs) + for rex in keys(exsnew) + sigs, includes = eval_rex(rex, exsold, mod; mode=mode) + if sigs !== nothing + exsnew[rex] = sigs + end + if includes !== nothing + maybe_add_includes_to_pkgdata!(pkgdata, file, includes; eval_now=true) + end + end + delete!(modsremaining, mod) + changed = true + catch _err + err = _err + end + end + end + if isempty(modsremaining) + pkgdata.fileinfos[i] = FileInfo(mexsnew, fi) + delete!(queue_errors, (pkgdata, file)) + else + throw && Base.throw(err) + interrupt |= isa(err, InterruptException) + push!(revision_errors, (pkgdata, file)) + queue_errors[(pkgdata, file)] = (err, catch_backtrace()) + end + end + if interrupt + for pkgfile in finished + haskey(queue_errors, pkgfile) || delete!(revision_queue, pkgfile) + end + else + empty!(revision_queue) + end + errors(revision_errors) + if !isempty(queue_errors) + if !have_queue_errors # only print on the first time errors occur + io = IOBuffer() + println(io, "\n") # better here than in the triple-quoted literal, see https://github.com/JuliaLang/julia/issues/34105 + for (pkgdata, file) in keys(queue_errors) + println(io, " ", joinpath(basedir(pkgdata), file)) + end + str = String(take!(io)) + @warn """The running code does not match the saved version for the following files:$str + If the error was due to evaluation order, it can sometimes be resolved by calling `Revise.retry()`. + Use Revise.errors() to report errors again. Only the first error in each file is shown. + Your prompt color may be yellow until the errors are resolved.""" + maybe_set_prompt_color(:warn) + end + else + maybe_set_prompt_color(:ok) + end + tracking_Main_includes[] && queue_includes(Main) + + process_user_callbacks!(throw=throw) + + nothing +end +revise(backend::REPL.REPLBackend) = revise() + +""" + revise(mod::Module) + +Reevaluate every definition in `mod`, whether it was changed or not. This is useful +to propagate an updated macro definition, or to force recompiling generated functions. +""" +function revise(mod::Module) + mod == Main && error("cannot revise(Main)") + id = PkgId(mod) + pkgdata = pkgdatas[id] + for (i, file) in enumerate(srcfiles(pkgdata)) + fi = fileinfo(pkgdata, i) + for (mod, exsigs) in fi.modexsigs + for def in keys(exsigs) + ex = def.ex + exuw = unwrap(ex) + isexpr(exuw, :call) && exuw.args[1] === :include && continue + try + Core.eval(mod, ex) + catch err + @show mod + display(ex) + rethrow(err) + end + end + end + end + return true # fixme try/catch? +end + +""" + Revise.track(mod::Module, file::AbstractString) + Revise.track(file::AbstractString) + +Watch `file` for updates and [`revise`](@ref) loaded code with any +changes. `mod` is the module into which `file` is evaluated; if omitted, +it defaults to `Main`. + +If this produces many errors, check that you specified `mod` correctly. +""" +function track(mod::Module, file; mode=:sigs, kwargs...) + isfile(file) || error(file, " is not a file") + # Determine whether we're already tracking this file + id = Base.moduleroot(mod) == Main ? PkgId(mod, string(mod)) : PkgId(mod) # see #689 for `Main` + if haskey(pkgdatas, id) + pkgdata = pkgdatas[id] + relfile = relpath(abspath(file), pkgdata) + hasfile(pkgdata, relfile) && return nothing + # Use any "fixes" provided by relpath + file = joinpath(basedir(pkgdata), relfile) + else + # Check whether `track` was called via a @require. Ref issue #403 & #431. + st = stacktrace(backtrace()) + if any(sf->sf.func === :listenpkg && endswith(String(sf.file), "require.jl"), st) + nameof(mod) === :Plots || Base.depwarn("Revise@2.4 or higher automatically handles `include` statements in `@require` expressions.\nPlease do not call `Revise.track` from such blocks.", :track) + return nothing + end + file = abspath(file) + end + # Set up tracking + fm = parse_source(file, mod; mode=mode) + if fm !== nothing + if mode === :includet + mode = :sigs # we already handled evaluation in `parse_source` + end + instantiate_sigs!(fm; mode=mode, kwargs...) + if !haskey(pkgdatas, id) + # Wait a bit to see if `mod` gets initialized + sleep(0.1) + end + pkgdata = get(pkgdatas, id, nothing) + if pkgdata === nothing + pkgdata = PkgData(id, pathof(mod)) + end + if !haskey(CodeTracking._pkgfiles, id) + CodeTracking._pkgfiles[id] = pkgdata.info + end + push!(pkgdata, relpath(file, pkgdata)=>FileInfo(fm)) + init_watching(pkgdata, (String(file)::String,)) + pkgdatas[id] = pkgdata + end + return nothing +end + +function track(file; kwargs...) + startswith(file, juliadir) && error("use Revise.track(Base) or Revise.track()") + track(Main, file; kwargs...) +end + +""" + includet(filename) + +Load `filename` and track future changes. `includet` is intended for quick "user scripts"; larger or more +established projects are encouraged to put the code in one or more packages loaded with `using` +or `import` instead of using `includet`. See https://timholy.github.io/Revise.jl/stable/cookbook/ +for tips about setting up the package workflow. + +By default, `includet` only tracks modifications to *methods*, not *data*. See the extended help for details. +Note that this differs from packages, which evaluate all changes by default. +This default behavior can be overridden; see [Configuring the revise mode](@ref). + +# Extended help + +## Behavior and justification for the default revision mode (`:evalmeth`) + +`includet` uses a default `__revise_mode__ = :evalmeth`. The consequence is that if you change + +``` +a = [1] +f() = 1 +``` +to +``` +a = [2] +f() = 2 +``` +then Revise will update `f` but not `a`. + +This is the default choice for `includet` because such files typically mix method definitions and data-handling. +Data often has many untracked dependencies; later in the same file you might `push!(a, 22)`, but Revise cannot +determine whether you wish it to re-run that line after redefining `a`. +Consequently, the safest default choice is to leave the user in charge of data. + +## Workflow tips + +If you have a series of computations that you want to run when you redefine your methods, consider separating +your method definitions from your computations: + +- method definitions go in a package, or a file that you `includet` *once* +- the computations go in a separate file, that you re-`include` (no "t" at the end) each time you want to rerun + your computations. + +This can be automated using [`entr`](@ref). + +## Internals + +`includet` is essentially shorthand for + + Revise.track(Main, filename; mode=:eval, skip_include=false) + +Do *not* use `includet` for packages, as those should be handled by `using` or `import`. +If `using` and `import` aren't working, you may have packages in a non-standard location; +try fixing it with something like `push!(LOAD_PATH, "/path/to/my/private/repos")`. +(If you're working with code in Base or one of Julia's standard libraries, use +`Revise.track(mod)` instead, where `mod` is the module.) + +`includet` is deliberately non-recursive, so if `filename` loads any other files, +they will not be automatically tracked. +(See [`Revise.track`](@ref) to set it up manually.) +""" +function includet(mod::Module, file) + prev = Base.source_path(nothing) + file = if prev === nothing + abspath(file) + else + normpath(joinpath(dirname(prev), file)) + end + tls = task_local_storage() + tls[:SOURCE_PATH] = file + try + track(mod, file; mode=:includet, skip_include=true) + if prev === nothing + delete!(tls, :SOURCE_PATH) + else + tls[:SOURCE_PATH] = prev + end + catch err + if prev === nothing + delete!(tls, :SOURCE_PATH) + else + tls[:SOURCE_PATH] = prev + end + if isa(err, ReviseEvalException) + printstyled(stderr, "ERROR: "; color=Base.error_color()); + showerror(stderr, err; blame_revise=false) + println(stderr, "\nin expression starting at ", err.loc) + else + throw(err) + end + end + return nothing +end +includet(file) = includet(Main, file) + +""" + Revise.silence(pkg) + +Silence warnings about not tracking changes to package `pkg`. +""" +function silence(pkg::Symbol) + push!(silence_pkgs, pkg) + if !isdir(depsdir) + mkpath(depsdir) + end + open(silencefile[], "w") do io + for p in silence_pkgs + println(io, p) + end + end + nothing +end +silence(pkg::AbstractString) = silence(Symbol(pkg)) + +## Utilities + +""" + success = get_def(method::Method) + +As needed, load the source file necessary for extracting the code defining `method`. +The source-file defining `method` must be tracked. +If it is in Base, this will execute `track(Base)` if necessary. + +This is a callback function used by `CodeTracking.jl`'s `definition`. +""" +function get_def(method::Method; modified_files=revision_queue) + yield() # magic bug fix for the OSX test failures. TODO: figure out why this works (prob. Julia bug) + if method.file === :none && String(method.name)[1] == '#' + # This is likely to be a kwarg method, try to find something with location info + method = bodymethod(method) + end + filename = fixpath(String(method.file)) + if startswith(filename, "REPL[") + isdefined(Base, :active_repl) || return false + fi = add_definitions_from_repl(filename) + hassig = false + for (mod, exs) in fi.modexsigs + for sigs in values(exs) + hassig |= !isempty(sigs) + end + end + return hassig + end + id = get_tracked_id(method.module; modified_files=modified_files) + id === nothing && return false + pkgdata = pkgdatas[id] + filename = relpath(filename, pkgdata) + if hasfile(pkgdata, filename) + def = get_def(method, pkgdata, filename) + def !== nothing && return true + end + # Lookup can fail for macro-defined methods, see https://github.com/JuliaLang/julia/issues/31197 + # We need to find the right file. + if method.module == Base || method.module == Core || method.module == Core.Compiler + @warn "skipping $method to avoid parsing too much code" + CodeTracking.invoked_setindex!(CodeTracking.method_info, method.sig, missing) + return false + end + parentfile, included_files = modulefiles(method.module) + if parentfile !== nothing + def = get_def(method, pkgdata, relpath(parentfile, pkgdata)) + def !== nothing && return true + for modulefile in included_files + def = get_def(method, pkgdata, relpath(modulefile, pkgdata)) + def !== nothing && return true + end + end + # As a last resort, try every file in the package + for file in srcfiles(pkgdata) + def = get_def(method, pkgdata, file) + def !== nothing && return true + end + @warn "$(method.sig) was not found" + # So that we don't call it again, store missingness info in CodeTracking + CodeTracking.invoked_setindex!(CodeTracking.method_info, method.sig, missing) + return false +end + +function get_def(method, pkgdata, filename) + maybe_extract_sigs!(maybe_parse_from_cache!(pkgdata, filename)) + return get(CodeTracking.method_info, method.sig, nothing) +end + +function get_tracked_id(id::PkgId; modified_files=revision_queue) + # Methods from Base or the stdlibs may require that we start tracking + if !haskey(pkgdatas, id) + recipe = id.name === "Compiler" ? :Compiler : Symbol(id.name) + recipe === :Core && return nothing + _track(id, recipe; modified_files=modified_files) + @info "tracking $recipe" + if !haskey(pkgdatas, id) + @warn "despite tracking $recipe, $id was not found" + return nothing + end + end + return id +end +get_tracked_id(mod::Module; modified_files=revision_queue) = + get_tracked_id(PkgId(mod); modified_files=modified_files) + +function get_expressions(id::PkgId, filename) + get_tracked_id(id) + pkgdata = pkgdatas[id] + fi = maybe_parse_from_cache!(pkgdata, filename) + maybe_extract_sigs!(fi) + return fi.modexsigs +end + +function add_definitions_from_repl(filename::String) + hist_idx = parse(Int, filename[6:end-1]) + hp = (Base.active_repl::REPL.LineEditREPL).interface.modes[1].hist::REPL.REPLHistoryProvider + src = hp.history[hp.start_idx+hist_idx] + id = PkgId(nothing, "@REPL") + pkgdata = pkgdatas[id] + mexs = ModuleExprsSigs(Main::Module) + parse_source!(mexs, src, filename, Main::Module) + instantiate_sigs!(mexs) + fi = FileInfo(mexs) + push!(pkgdata, filename=>fi) + return fi +end +add_definitions_from_repl(filename::AbstractString) = add_definitions_from_repl(convert(String, filename)::String) + +function update_stacktrace_lineno!(trace) + local nrep + for i = 1:length(trace) + t = trace[i] + has_nrep = !isa(t, StackTraces.StackFrame) + if has_nrep + t, nrep = t + end + t = t::StackTraces.StackFrame + if t.linfo isa Core.MethodInstance + m = t.linfo.def + sigt = m.sig + # Why not just call `whereis`? Because that forces tracking. This is being + # clever by recognizing that these entries exist only if there have been updates. + updated = get(CodeTracking.method_info, sigt, nothing) + if updated !== nothing + lnn = updated[1][1] # choose the first entry by default + lineoffset = lnn.line - m.line + t = StackTraces.StackFrame(t.func, lnn.file, t.line+lineoffset, t.linfo, t.from_c, t.inlined, t.pointer) + trace[i] = has_nrep ? (t, nrep) : t + end + end + end + return trace +end + +function method_location(method::Method) + # Why not just call `whereis`? Because that forces tracking. This is being + # clever by recognizing that these entries exist only if there have been updates. + updated = get(CodeTracking.method_info, method.sig, nothing) + if updated !== nothing + lnn = updated[1][1] + return lnn.file, lnn.line + end + return method.file, method.line +end + +# Set the prompt color to indicate the presence of unhandled revision errors +const original_repl_prefix = Ref{Union{String,Function,Nothing}}(nothing) +function maybe_set_prompt_color(color) + if isdefined(Base, :active_repl) + repl = Base.active_repl + if isa(repl, REPL.LineEditREPL) + if color === :warn + # First save the original setting + if original_repl_prefix[] === nothing + original_repl_prefix[] = repl.mistate.current_mode.prompt_prefix + end + repl.mistate.current_mode.prompt_prefix = "\e[33m" # yellow + else + color = original_repl_prefix[] + color === nothing && return nothing + repl.mistate.current_mode.prompt_prefix = color + original_repl_prefix[] = nothing + end + end + end + return nothing +end + +# `revise_first` gets called by the REPL prior to executing the next command (by having been pushed +# onto the `ast_transform` list). +# This uses invokelatest not for reasons of world age but to ensure that the call is made at runtime. +# This allows `revise_first` to be compiled without compiling `revise` itself, and greatly +# reduces the overhead of using Revise. +function revise_first(ex) + # Special-case `exit()` (issue #562) + if isa(ex, Expr) + exu = unwrap(ex) + isa(exu, Expr) && exu.head === :call && length(exu.args) == 1 && exu.args[1] === :exit && return ex + end + # Check for queued revisions, and if so call `revise` first before executing the expression + return Expr(:toplevel, :(isempty($revision_queue) || Base.invokelatest($revise)), ex) +end + +steal_repl_backend(args...) = @warn "`steal_repl_backend` has been removed from Revise, please update your `~/.julia/config/startup.jl`.\nSee https://timholy.github.io/Revise.jl/stable/config/" +wait_steal_repl_backend() = steal_repl_backend() +async_steal_repl_backend() = steal_repl_backend() + +""" + Revise.init_worker(p) + +Define methods on worker `p` that Revise needs in order to perform revisions on `p`. +Revise itself does not need to be running on `p`. +""" +function init_worker(p) + remotecall(Core.eval, p, Main, quote + function whichtt(sig) + ret = Base._methods_by_ftype(sig, -1, typemax(UInt)) + isempty(ret) && return nothing + m = ret[end][3]::Method # the last method returned is the least-specific that matches, and thus most likely to be type-equal + methsig = m.sig + (sig <: methsig && methsig <: sig) || return nothing + return m + end + function delete_method_by_sig(sig) + m = whichtt(sig) + isa(m, Method) && Base.delete_method(m) + end + end) +end + +function __init__() + run_on_worker = get(ENV, "JULIA_REVISE_WORKER_ONLY", "0") + if !(myid() == 1 || run_on_worker == "1") + return nothing + end + # Check Julia paths (issue #601) + if !isdir(juliadir) + major, minor = Base.VERSION.major, Base.VERSION.minor + @warn """Expected non-existent $juliadir to be your Julia directory. + Certain functionality will be disabled. + To fix this, try deleting Revise's cache files in ~/.julia/compiled/v$major.$minor/Revise, then restart Julia and load Revise. + If this doesn't fix the problem, please report an issue at https://github.com/timholy/Revise.jl/issues.""" + end + if isfile(silencefile[]) + pkgs = readlines(silencefile[]) + for pkg in pkgs + push!(silence_pkgs, Symbol(pkg)) + end + end + polling = get(ENV, "JULIA_REVISE_POLL", "0") + if polling == "1" + polling_files[] = watching_files[] = true + end + rev_include = get(ENV, "JULIA_REVISE_INCLUDE", "0") + if rev_include == "1" + tracking_Main_includes[] = true + end + # Correct line numbers for code moving around + Base.update_stackframes_callback[] = update_stacktrace_lineno! + if isdefined(Base, :methodloc_callback) + Base.methodloc_callback[] = method_location + end + # Add `includet` to the compiled_modules (fixes #302) + for m in methods(includet) + push!(JuliaInterpreter.compiled_methods, m) + end + # Set up a repository for methods defined at the REPL + id = PkgId(nothing, "@REPL") + pkgdatas[id] = pkgdata = PkgData(id, nothing) + # Set the lookup callbacks + CodeTracking.method_lookup_callback[] = get_def + CodeTracking.expressions_callback[] = get_expressions + + # Register the active-project watcher + if isdefined(Pkg.Types, :active_project_watcher_thunks) + push!(Pkg.Types.active_project_watcher_thunks, active_project_watcher) + end + + # Watch the manifest file for changes + mfile = manifest_file() + if mfile !== nothing + push!(watched_manifests, mfile) + wmthunk = TaskThunk(watch_manifest, (mfile,)) + schedule(Task(wmthunk)) + end + push!(Base.include_callbacks, watch_includes) + push!(Base.package_callbacks, watch_package_callback) + + mode = get(ENV, "JULIA_REVISE", "auto") + if mode == "auto" + if isdefined(Main, :IJulia) + Main.IJulia.push_preexecute_hook(revise) + else + pushfirst!(REPL.repl_ast_transforms, revise_first) + # #664: once a REPL is started, it no longer interacts with REPL.repl_ast_transforms + if isdefined(Base, :active_repl_backend) + push!(Base.active_repl_backend.ast_transforms, revise_first) + else + # wait for active_repl_backend to exist + # #719: do this async in case Revise is being loaded from startup.jl + t = @async begin + iter = 0 + while !isdefined(Base, :active_repl_backend) && iter < 20 + sleep(0.05) + iter += 1 + end + if isdefined(Base, :active_repl_backend) + push!(Base.active_repl_backend.ast_transforms, revise_first) + end + end + isdefined(Base, :errormonitor) && Base.errormonitor(t) + end + end + if isdefined(Main, :Atom) + Atom = getfield(Main, :Atom) + if Atom isa Module && isdefined(Atom, :handlers) + setup_atom(Atom) + end + end + end + return nothing +end + +const REVISE_ID = Base.PkgId(Base.UUID("295af30f-e4ad-537b-8983-00126c2a3abe"), "Revise") +function watch_package_callback(id::PkgId) + # `Base.package_callbacks` fire immediately after module initialization, and + # would fire on Revise itself. This is not necessary for most users, and has + # the downside that the user doesn't get to the REPL prompt until + # `watch_package` finishes compiling. To prevent this, Revise hides the + # actual `watch_package` method behind an `invokelatest`. This delays + # compilation of everything that `watch_package` requires, leading to faster + # perceived startup times. + if id != REVISE_ID + Base.invokelatest(watch_package, id) + end + return +end + +function setup_atom(atommod::Module)::Nothing + handlers = getfield(atommod, :handlers) + for x in ["eval", "evalall", "evalshow", "evalrepl"] + if haskey(handlers, x) + old = handlers[x] + Main.Atom.handle(x) do data + revise() + old(data) + end + end + end + return nothing +end + +function add_revise_deps() + # Populate CodeTracking data for dependencies and initialize watching on code that Revise depends on + for mod in (CodeTracking, OrderedCollections, JuliaInterpreter, LoweredCodeUtils, Revise) + id = PkgId(mod) + pkgdata = parse_pkg_files(id) + init_watching(pkgdata, srcfiles(pkgdata)) + pkgdatas[id] = pkgdata + end + return nothing +end + +include("precompile.jl") +_precompile_() diff --git a/packages/Revise/src/parsing.jl b/packages/Revise/src/parsing.jl new file mode 100644 index 0000000..68089f0 --- /dev/null +++ b/packages/Revise/src/parsing.jl @@ -0,0 +1,91 @@ +""" + mexs = parse_source(filename::AbstractString, mod::Module) + +Parse the source `filename`, returning a [`ModuleExprsSigs`](@ref) `mexs`. +`mod` is the "parent" module for the file (i.e., the one that `include`d the file); +if `filename` defines more module(s) then these will all have separate entries in `mexs`. + +If parsing `filename` fails, `nothing` is returned. +""" +parse_source(filename, mod::Module; kwargs...) = + parse_source!(ModuleExprsSigs(mod), filename, mod; kwargs...) + +""" + parse_source!(mexs::ModuleExprsSigs, filename, mod::Module) + +Top-level parsing of `filename` as included into module +`mod`. Successfully-parsed expressions will be added to `mexs`. Returns +`mexs` if parsing finished successfully, otherwise `nothing` is returned. + +See also [`Revise.parse_source`](@ref). +""" +function parse_source!(mod_exprs_sigs::ModuleExprsSigs, filename::AbstractString, mod::Module; kwargs...) + if !isfile(filename) + @warn "$filename is not a file, omitting from revision tracking" + return nothing + end + parse_source!(mod_exprs_sigs, read(filename, String), filename, mod; kwargs...) +end + +""" + success = parse_source!(mod_exprs_sigs::ModuleExprsSigs, src::AbstractString, filename::AbstractString, mod::Module) + +Parse a string `src` obtained by reading `file` as a single +string. `pos` is the 1-based byte offset from which to begin parsing `src`. + +See also [`Revise.parse_source`](@ref). +""" +function parse_source!(mod_exprs_sigs::ModuleExprsSigs, src::AbstractString, filename::AbstractString, mod::Module; kwargs...) + startswith(src, "# REVISE: DO NOT PARSE") && return nothing + ex = Base.parse_input_line(src; filename=filename) + ex === nothing && return mod_exprs_sigs + if isexpr(ex, :error) || isexpr(ex, :incomplete) + prevex, pos = first_bad_position(src) + ln = count(isequal('\n'), SubString(src, 1, min(pos, length(src)))) + 1 + throw(LoadError(filename, ln, ex.args[1])) + end + return process_source!(mod_exprs_sigs, ex, filename, mod; kwargs...) +end + +function process_source!(mod_exprs_sigs::ModuleExprsSigs, ex, filename, mod::Module; mode::Symbol=:sigs) + for (mod, ex) in ExprSplitter(mod, ex) + if mode === :includet + try + Core.eval(mod, ex) + catch err + bt = trim_toplevel!(catch_backtrace()) + lnn = firstline(ex) + loc = location_string(lnn.file, lnn.line) + throw(ReviseEvalException(loc, err, Any[(sf, 1) for sf in stacktrace(bt)])) + end + end + exprs_sigs = get(mod_exprs_sigs, mod, nothing) + if exprs_sigs === nothing + mod_exprs_sigs[mod] = exprs_sigs = ExprsSigs() + end + if ex.head === :toplevel + lnn = nothing + for a in ex.args + if isa(a, LineNumberNode) + lnn = a + else + pushex!(exprs_sigs, Expr(:block, lnn, a)) + end + end + else + pushex!(exprs_sigs, ex) + end + end + return mod_exprs_sigs +end + +function first_bad_position(str) + ex, pos, n = nothing, 1, length(str) + while pos < n + ex, pos = Meta.parse(str, pos; greedy=true, raise=false) + if isexpr(ex, :error) || isexpr(ex, :incomplete) + return ex, pos + end + end + error("expected an error, finished without one") +end diff --git a/packages/Revise/src/pkgs.jl b/packages/Revise/src/pkgs.jl new file mode 100644 index 0000000..74390d2 --- /dev/null +++ b/packages/Revise/src/pkgs.jl @@ -0,0 +1,490 @@ +using Base: PkgId + +include("loading.jl") + +""" + parse_pkg_files(id::PkgId) + +This function gets called by `watch_package` and runs when a package is first loaded. +Its job is to organize the files and expressions defining the module so that later we can +detect and process revisions. +""" +parse_pkg_files(id::PkgId) + +""" + parentfile, included_files = modulefiles(mod::Module) + +Return the `parentfile` in which `mod` was defined, as well as a list of any +other files that were `include`d to define `mod`. If this operation is unsuccessful, +`(nothing, nothing)` is returned. + +All files are returned as absolute paths. +""" +modulefiles(mod::Module) + +# This is primarily used to parse non-precompilable packages. +# These lack a cache header that lists the files that constitute the package; +# they also lack the source cache, and so have to parsed immediately or +# we won't be able to compute a diff when a file is modified (we don't have a record +# of what the source was before the modification). +# +# The main trick here is that since `using` is recursive, `included_files` +# might contain files associated with many different packages. We have to figure +# out which correspond to a particular module `mod`, which we do by: +# - checking the module in which each file is evaluated. This suffices to +# detect "supporting" files, i.e., those `included` within the module +# definition. +# - checking the filename. Since the "top level" file is evaluated into Main, +# we can't use the module-of-evaluation to find it. Here we hope that the +# top-level filename follows convention and matches the module. TODO?: it's +# possible that this needs to be supplemented with parsing. +function queue_includes!(pkgdata::PkgData, id::PkgId) + modstring = id.name + delids = Int[] + for i = 1:length(included_files) + mod, fname = included_files[i] + if mod == Base.__toplevel__ + mod = Main + end + modname = String(Symbol(mod)) + if startswith(modname, modstring) || endswith(fname, modstring*".jl") + modexsigs = parse_source(fname, mod) + if modexsigs !== nothing + fname = relpath(fname, pkgdata) + push!(pkgdata, fname=>FileInfo(modexsigs)) + end + push!(delids, i) + end + end + deleteat!(included_files, delids) + CodeTracking._pkgfiles[id] = pkgdata.info + return pkgdata +end + +function queue_includes(mod::Module) + id = PkgId(mod) + pkgdata = get(pkgdatas, id, nothing) + if pkgdata === nothing + pkgdata = PkgData(id) + end + queue_includes!(pkgdata, id) + if has_writable_paths(pkgdata) + init_watching(pkgdata) + end + pkgdatas[id] = pkgdata + return pkgdata +end + +# A near-duplicate of some of the functionality of queue_includes! +# This gets called for silenced packages, to make sure they don't "contaminate" +# included_files +function remove_from_included_files(modsym::Symbol) + i = 1 + modstring = string(modsym) + while i <= length(included_files) + mod, fname = included_files[i] + modname = String(Symbol(mod)) + if startswith(modname, modstring) || endswith(fname, modstring*".jl") + deleteat!(included_files, i) + else + i += 1 + end + end +end + +function read_from_cache(pkgdata::PkgData, file::AbstractString) + fi = fileinfo(pkgdata, file) + filep = joinpath(basedir(pkgdata), file) + if fi.cachefile == basesrccache + # Get the original path + filec = get(cache_file_key, filep, filep) + return open(basesrccache) do io + Base._read_dependency_src(io, filec) + end + end + Base.read_dependency_src(fi.cachefile, filep) +end + +function maybe_parse_from_cache!(pkgdata::PkgData, file::AbstractString) + if startswith(file, "REPL[") + return add_definitions_from_repl(file) + end + fi = fileinfo(pkgdata, file) + if isempty(fi.modexsigs) && (!isempty(fi.cachefile) || !isempty(fi.cacheexprs)) + # Source was never parsed, get it from the precompile cache + src = read_from_cache(pkgdata, file) + filep = joinpath(basedir(pkgdata), file) + filec = get(cache_file_key, filep, filep) + topmod = first(keys(fi.modexsigs)) + if parse_source!(fi.modexsigs, src, filec, topmod) === nothing + @error "failed to parse cache file source text for $file" + end + add_modexs!(fi, fi.cacheexprs) + empty!(fi.cacheexprs) + end + return fi +end + +function add_modexs!(fi::FileInfo, modexs) + for (mod, rex) in modexs + exsigs = get(fi.modexsigs, mod, nothing) + if exsigs === nothing + fi.modexsigs[mod] = exsigs = ExprsSigs() + end + pushex!(exsigs, rex) + end + return fi +end + +function maybe_extract_sigs!(fi::FileInfo) + if !fi.extracted[] + instantiate_sigs!(fi.modexsigs) + fi.extracted[] = true + end + return fi +end +maybe_extract_sigs!(pkgdata::PkgData, file::AbstractString) = maybe_extract_sigs!(fileinfo(pkgdata, file)) + +function maybe_add_includes_to_pkgdata!(pkgdata::PkgData, file::AbstractString, includes; eval_now::Bool=false) + for (mod, inc) in includes + inc = joinpath(splitdir(file)[1], inc) + incrp = relpath(inc, pkgdata) + hasfile = false + for srcfile in srcfiles(pkgdata) + if srcfile == incrp + hasfile = true + break + end + end + if !hasfile + # Add the file to pkgdata + push!(pkgdata.info.files, incrp) + fi = FileInfo(mod) + push!(pkgdata.fileinfos, fi) + # Parse the source of the new file + fullfile = joinpath(basedir(pkgdata), incrp) + if isfile(fullfile) + parse_source!(fi.modexsigs, fullfile, mod) + if eval_now + # Use runtime dispatch to reduce latency + Base.invokelatest(instantiate_sigs!, fi.modexsigs; mode=:eval) + end + end + # Add to watchlist + init_watching(pkgdata, (incrp,)) + yield() + end + end +end + +# Use locking to prevent races between inner and outer @require blocks +const requires_lock = ReentrantLock() + +function add_require(sourcefile::String, modcaller::Module, idmod::String, modname::String, expr::Expr) + id = PkgId(modcaller) + # If this fires when the module is first being loaded (because the dependency + # was already loaded), Revise may not yet have the pkgdata for this package. + if !haskey(pkgdatas, id) + watch_package(id) + end + + lock(requires_lock) + try + # Get/create the FileInfo specifically for tracking @require blocks + pkgdata = pkgdatas[id] + filekey = relpath(sourcefile, pkgdata) * "__@require__" + fileidx = fileindex(pkgdata, filekey) + if fileidx === nothing + files = srcfiles(pkgdata) + fileidx = length(files) + 1 + push!(files, filekey) + push!(pkgdata.fileinfos, FileInfo(modcaller)) + end + fi = pkgdata.fileinfos[fileidx] + # Tag the expr to ensure it is unique + expr = Expr(:block, copy(expr)) + push!(expr.args, :(__pkguuid__ = $idmod)) + # Add the expression to the fileinfo + complex = true # is this too complex to delay? + if !fi.extracted[] + # If we haven't yet extracted signatures, do our best to avoid it now in case the + # signature-extraction code has not yet been compiled (latency reduction) + includes, complex = deferrable_require(expr) + if !complex + # [(modcaller, inc) for inc in includes] but without precompiling a Generator + modincludes = Tuple{Module,String}[] + for inc in includes + push!(modincludes, (modcaller, inc)) + end + maybe_add_includes_to_pkgdata!(pkgdata, filekey, modincludes) + if isempty(fi.modexsigs) + # Source has not even been parsed + push!(fi.cacheexprs, (modcaller, expr)) + else + add_modexs!(fi, [(modcaller, expr)]) + end + end + end + if complex + Base.invokelatest(eval_require_now, pkgdata, fileidx, filekey, sourcefile, modcaller, expr) + end + finally + unlock(requires_lock) + end +end + +function deferrable_require(expr) + includes = String[] + complex = deferrable_require!(includes, expr) + return includes, complex +end +function deferrable_require!(includes, expr::Expr) + if expr.head === :call + callee = expr.args[1] + if callee === :include + if isa(expr.args[2], AbstractString) + push!(includes, expr.args[2]) + else + return true + end + elseif callee === :eval || (isa(callee, Expr) && callee.head === :. && is_quotenode_egal(callee.args[2], :eval)) + # Any eval statement is suspicious and requires immediate action + return false + end + end + expr.head === :macrocall && expr.args[1] === Symbol("@eval") && return true + for a in expr.args + a isa Expr || continue + deferrable_require!(includes, a) && return true + end + return false +end + +function eval_require_now(pkgdata::PkgData, fileidx::Int, filekey::String, sourcefile::String, modcaller::Module, expr::Expr) + fi = pkgdata.fileinfos[fileidx] + exsnew = ExprsSigs() + exsnew[RelocatableExpr(expr)] = nothing + mexsnew = ModuleExprsSigs(modcaller=>exsnew) + # Before executing the expression we need to set the load path appropriately + prev = Base.source_path(nothing) + tls = task_local_storage() + tls[:SOURCE_PATH] = sourcefile + # Now execute the expression + mexsnew, includes = try + eval_new!(mexsnew, fi.modexsigs) + finally + if prev === nothing + delete!(tls, :SOURCE_PATH) + else + tls[:SOURCE_PATH] = prev + end + end + # Add any new methods or `include`d files to tracked objects + pkgdata.fileinfos[fileidx] = FileInfo(mexsnew, fi) + ret = maybe_add_includes_to_pkgdata!(pkgdata, filekey, includes; eval_now=true) + return ret +end + +function watch_files_via_dir(dirname) + try + wait_changed(dirname) # this will block until there is a modification + catch e + # issue #459 + (isa(e, InterruptException) && throwto_repl(e)) || throw(e) + end + latestfiles = Pair{String,PkgId}[] + # Check to see if we're still watching this directory + stillwatching = haskey(watched_files, dirname) + if stillwatching + wf = watched_files[dirname] + for (file, id) in wf.trackedfiles + fullpath = joinpath(dirname, file) + if isdir(fullpath) + # Detected a modification in a directory that we're watching in + # itself (not as a container for watched files) + push!(latestfiles, file=>id) + continue + elseif !file_exists(fullpath) + # File may have been deleted. But be very sure. + sleep(0.1) + if !file_exists(fullpath) + push!(latestfiles, file=>id) + continue + end + end + if newer(mtime(fullpath), wf.timestamp) + push!(latestfiles, file=>id) + end + end + isempty(latestfiles) || updatetime!(wf) # ref issue #341 + end + return latestfiles, stillwatching +end + +const wplock = ReentrantLock() + +""" + watch_package(id::Base.PkgId) + +Start watching a package for changes to the files that define it. +This function gets called via a callback registered with `Base.require`, at the completion +of module-loading by `using` or `import`. +""" +function watch_package(id::PkgId) + pkgdata = get(pkgdatas, id, nothing) + pkgdata !== nothing && return pkgdata + lock(wplock) + try + modsym = Symbol(id.name) + if modsym ∈ dont_watch_pkgs + if modsym ∉ silence_pkgs + @warn "$modsym is excluded from watching by Revise. Use Revise.silence(\"$modsym\") to quiet this warning." + end + remove_from_included_files(modsym) + return nothing + end + pkgdata = parse_pkg_files(id) + if has_writable_paths(pkgdata) + init_watching(pkgdata, srcfiles(pkgdata)) + end + pkgdatas[id] = pkgdata + finally + unlock(wplock) + end + return pkgdata +end + +function has_writable_paths(pkgdata::PkgData) + dir = basedir(pkgdata) + isdir(dir) || return true + haswritable = false + cd(dir) do + for file in srcfiles(pkgdata) + haswritable |= iswritable(file) + end + end + return haswritable +end + +function watch_includes(mod::Module, fn::AbstractString) + push!(included_files, (mod, normpath(abspath(fn)))) +end + +## Working with Pkg and code-loading + +# Much of this is adapted from base/loading.jl + +function manifest_file(project_file) + if project_file isa String && isfile(project_file) + mfile = Base.project_file_manifest_path(project_file) + if mfile isa String + return mfile + end + end + return nothing +end +manifest_file() = manifest_file(Base.active_project()) + +function manifest_paths!(pkgpaths::Dict, manifest_file::String) + d = if isdefined(Base, :get_deps) # `get_deps` is present in versions that support new manifest formats + Base.get_deps(Base.parsed_toml(manifest_file)) + else + Base.parsed_toml(manifest_file) + end + for (name, entries) in d + entries::Vector{Any} + for entry in entries + id = PkgId(UUID(entry["uuid"]::String), name) + path = Base.explicit_manifest_entry_path(manifest_file, id, entry) + if path !== nothing + pkgpaths[id] = path + end + end + end + return pkgpaths +end + +manifest_paths(manifest_file::String) = + manifest_paths!(Dict{PkgId,String}(), manifest_file) + +function watch_manifest(mfile) + while true + try + wait_changed(mfile) + catch e + # issue #459 + (isa(e, InterruptException) && throwto_repl(e)) || throw(e) + end + manifest_file() == mfile || continue # process revisions only if this is the active manifest + try + with_logger(_debug_logger) do + @debug "Pkg" _group="manifest_update" manifest_file=mfile + isfile(mfile) || return nothing + pkgdirs = manifest_paths(mfile) + for (id, pkgdir) in pkgdirs + if haskey(pkgdatas, id) + pkgdata = pkgdatas[id] + if pkgdir != basedir(pkgdata) + ## The package directory has changed + @debug "Pkg" _group="pathswitch" oldpath=basedir(pkgdata) newpath=pkgdir + # Stop all associated watching tasks + for dir in unique_dirs(srcfiles(pkgdata)) + @debug "Pkg" _group="unwatch" dir=dir + delete!(watched_files, joinpath(basedir(pkgdata), dir)) + # Note: if the file is revised, the task(s) will run one more time. + # However, because we've removed the directory from the watch list this will be a no-op, + # and then the tasks will be dropped. + end + # Revise code as needed + files = String[] + mustnotify = false + for file in srcfiles(pkgdata) + fi = try + maybe_parse_from_cache!(pkgdata, file) + catch err + # https://github.com/JuliaLang/julia/issues/42404 + # Get the source-text from the package source instead + fi = fileinfo(pkgdata, file) + if isempty(fi.modexsigs) && (!isempty(fi.cachefile) || !isempty(fi.cacheexprs)) + filep = joinpath(basedir(pkgdata), file) + src = read(filep, String) + topmod = first(keys(fi.modexsigs)) + if parse_source!(fi.modexsigs, src, filep, topmod) === nothing + @error "failed to parse source text for $filep" + end + add_modexs!(fi, fi.cacheexprs) + empty!(fi.cacheexprs) + end + fi + end + maybe_extract_sigs!(fi) + push!(revision_queue, (pkgdata, file)) + push!(files, file) + mustnotify = true + end + mustnotify && notify(revision_event) + # Update the directory + pkgdata.info.basedir = pkgdir + # Restart watching, if applicable + if has_writable_paths(pkgdata) + init_watching(pkgdata, files) + end + end + end + end + end + catch err + @error "Error watching manifest" exception=(err, trim_toplevel!(catch_backtrace())) + end + end +end + +function active_project_watcher() + mfile = manifest_file() + if mfile ∉ watched_manifests + push!(watched_manifests, mfile) + wmthunk = TaskThunk(watch_manifest, (mfile,)) + schedule(Task(wmthunk)) + end + return +end diff --git a/packages/Revise/src/precompile.jl b/packages/Revise/src/precompile.jl new file mode 100644 index 0000000..3ebcf27 --- /dev/null +++ b/packages/Revise/src/precompile.jl @@ -0,0 +1,92 @@ +# COV_EXCL_START +macro warnpcfail(ex::Expr) + modl = __module__ + file = __source__.file === nothing ? "?" : String(__source__.file) + line = __source__.line + quote + $(esc(ex)) || @warn """precompile directive + $($(Expr(:quote, ex))) + failed. Please report an issue in $($modl) (after checking for duplicates) or remove this directive.""" _file=$file _line=$line + end +end + +function _precompile_() + ccall(:jl_generating_output, Cint, ()) == 1 || return nothing + + @warnpcfail precompile(Tuple{TaskThunk}) + @warnpcfail precompile(Tuple{typeof(wait_changed), String}) + @warnpcfail precompile(Tuple{typeof(watch_package), PkgId}) + @warnpcfail precompile(Tuple{typeof(watch_includes), Module, String}) + @warnpcfail precompile(Tuple{typeof(watch_manifest), String}) + @warnpcfail precompile(Tuple{typeof(revise_dir_queued), String}) + @warnpcfail precompile(Tuple{typeof(revise_file_queued), PkgData, String}) + @warnpcfail precompile(Tuple{typeof(init_watching), PkgData, Vector{String}}) + @warnpcfail precompile(Tuple{typeof(add_revise_deps)}) + @warnpcfail precompile(Tuple{typeof(watch_package_callback), PkgId}) + + @warnpcfail precompile(Tuple{typeof(revise)}) + @warnpcfail precompile(Tuple{typeof(revise_first), Expr}) + @warnpcfail precompile(Tuple{typeof(includet), String}) + @warnpcfail precompile(Tuple{typeof(track), Module, String}) + # setindex! doesn't fully precompile, but it's still beneficial to do it + # (it shaves off a bit of the time) + # See https://github.com/JuliaLang/julia/pull/31466 + @warnpcfail precompile(Tuple{typeof(setindex!), ExprsSigs, Nothing, RelocatableExpr}) + @warnpcfail precompile(Tuple{typeof(setindex!), ExprsSigs, Vector{Any}, RelocatableExpr}) + @warnpcfail precompile(Tuple{typeof(setindex!), ModuleExprsSigs, ExprsSigs, Module}) + @warnpcfail precompile(Tuple{typeof(setindex!), Dict{PkgId,PkgData}, PkgData, PkgId}) + @warnpcfail precompile(Tuple{Type{WatchList}}) + @warnpcfail precompile(Tuple{typeof(setindex!), Dict{String,WatchList}, WatchList, String}) + + MI = CodeTrackingMethodInfo + @warnpcfail precompile(Tuple{typeof(minimal_evaluation!), MI, Core.CodeInfo, Symbol}) + @warnpcfail precompile(Tuple{typeof(minimal_evaluation!), Any, MI, Core.CodeInfo, Symbol}) + @warnpcfail precompile(Tuple{typeof(methods_by_execution!), Any, MI, DocExprs, Module, Expr}) + @warnpcfail precompile(Tuple{typeof(methods_by_execution!), Any, MI, DocExprs, JuliaInterpreter.Frame, Vector{Bool}}) + @warnpcfail precompile(Tuple{typeof(Core.kwfunc(methods_by_execution!)), + NamedTuple{(:mode,),Tuple{Symbol}}, + typeof(methods_by_execution!), Function, MI, DocExprs, Module, Expr}) + @warnpcfail precompile(Tuple{typeof(Core.kwfunc(methods_by_execution!)), + NamedTuple{(:skip_include,),Tuple{Bool}}, + typeof(methods_by_execution!), Function, MI, DocExprs, Module, Expr}) + @warnpcfail precompile(Tuple{typeof(Core.kwfunc(methods_by_execution!)), + NamedTuple{(:mode, :skip_include),Tuple{Symbol,Bool}}, + typeof(methods_by_execution!), Function, MI, DocExprs, Module, Expr}) + @warnpcfail precompile(Tuple{typeof(Core.kwfunc(methods_by_execution!)), + NamedTuple{(:mode,),Tuple{Symbol}}, + typeof(methods_by_execution!), Function, MI, DocExprs, Frame, Vector{Bool}}) + @warnpcfail precompile(Tuple{typeof(Core.kwfunc(methods_by_execution!)), + NamedTuple{(:mode, :skip_include),Tuple{Symbol,Bool}}, + typeof(methods_by_execution!), Function, MI, DocExprs, Frame, Vector{Bool}}) + + mex = which(methods_by_execution!, (Function, MI, DocExprs, Module, Expr)) + mbody = bodymethod(mex) + # use `typeof(pairs(NamedTuple()))` here since it actually differs between Julia versions + @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, typeof(pairs(NamedTuple())), typeof(methods_by_execution!), Any, MI, DocExprs, Module, Expr}) + @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, Iterators.Pairs{Symbol,Bool,Tuple{Symbol},NamedTuple{(:skip_include,),Tuple{Bool}}}, typeof(methods_by_execution!), Any, MI, DocExprs, Module, Expr}) + mfr = which(methods_by_execution!, (Function, MI, DocExprs, Frame, Vector{Bool})) + mbody = bodymethod(mfr) + @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, typeof(methods_by_execution!), Any, MI, DocExprs, Frame, Vector{Bool}}) + + @warnpcfail precompile(Tuple{typeof(hastrackedexpr), Expr}) + @warnpcfail precompile(Tuple{typeof(get_def), Method}) + @warnpcfail precompile(Tuple{typeof(parse_pkg_files), PkgId}) + if isdefined(Revise, :filter_valid_cachefiles) + @warnpcfail precompile(Tuple{typeof(filter_valid_cachefiles), String, Vector{String}}) + end + @warnpcfail precompile(Tuple{typeof(pkg_fileinfo), PkgId}) + @warnpcfail precompile(Tuple{typeof(push!), WatchList, Pair{String,PkgId}}) + @warnpcfail precompile(Tuple{typeof(pushex!), ExprsSigs, Expr}) + @warnpcfail precompile(Tuple{Type{ModuleExprsSigs}, Module}) + @warnpcfail precompile(Tuple{Type{FileInfo}, Module, String}) + @warnpcfail precompile(Tuple{Type{PkgData}, PkgId}) + @warnpcfail precompile(Tuple{typeof(Base._deleteat!), Vector{Tuple{Module,String,Float64}}, Vector{Int}}) + @warnpcfail precompile(Tuple{typeof(add_require), String, Module, String, String, Expr}) + @warnpcfail precompile(Tuple{Core.kwftype(typeof(maybe_add_includes_to_pkgdata!)),NamedTuple{(:eval_now,), Tuple{Bool}},typeof(maybe_add_includes_to_pkgdata!),PkgData,String,Vector{Pair{Module, String}}}) + + for TT in (Tuple{Module,Expr}, Tuple{DataType,MethodSummary}) + @warnpcfail precompile(Tuple{Core.kwftype(typeof(Base.CoreLogging.handle_message)),NamedTuple{(:time, :deltainfo), Tuple{Float64, TT}},typeof(Base.CoreLogging.handle_message),ReviseLogger,LogLevel,String,Module,String,Symbol,String,Int}) + end + return nothing +end +# COV_EXCL_STOP diff --git a/packages/Revise/src/recipes.jl b/packages/Revise/src/recipes.jl new file mode 100644 index 0000000..41bbab3 --- /dev/null +++ b/packages/Revise/src/recipes.jl @@ -0,0 +1,197 @@ +""" + Revise.track(Base) + Revise.track(Core.Compiler) + Revise.track(stdlib) + +Track updates to the code in Julia's `base` directory, `base/compiler`, or one of its +standard libraries. +""" +function track(mod::Module; modified_files=revision_queue) + id = PkgId(mod) + modname = nameof(mod) + return _track(id, modname; modified_files=modified_files) +end + +const vstring = "v$(VERSION.major).$(VERSION.minor)" + +function inpath(path, dirs) + spath = splitpath(path) + idx = findfirst(isequal(first(dirs)), spath) + idx === nothing && return false + for i = 2:length(dirs) + idx += 1 + idx <= length(spath) || return false + if spath[idx] == vstring + idx += 1 + end + spath[idx] == dirs[i] || return false + end + return true +end + +function _track(id, modname; modified_files=revision_queue) + haskey(pkgdatas, id) && return nothing # already tracked + isbase = modname === :Base + isstdlib = !isbase && modname ∈ stdlib_names + if isbase || isstdlib + # Test whether we know where to find the files + if isbase + srcdir = fixpath(joinpath(juliadir, "base")) + dirs = ["base"] + else + stdlibv = joinpath("stdlib", vstring, String(modname)) + srcdir = fixpath(joinpath(juliadir, stdlibv)) + if !isdir(srcdir) + srcdir = fixpath(joinpath(juliadir, "stdlib", String(modname))) + end + if !isdir(srcdir) + # This can happen for Pkg, since it's developed out-of-tree + srcdir = joinpath(juliadir, "usr", "share", "julia", stdlibv) # omit fixpath deliberately + end + dirs = ["stdlib", String(modname)] + end + if !isdir(srcdir) + @error "unable to find path containing source for $modname, tracking is not possible" + end + # Determine when the basesrccache was built + mtcache = mtime(basesrccache) + # Initialize expression-tracking for files, and + # note any modified since Base was built + pkgdata = get(pkgdatas, id, nothing) + if pkgdata === nothing + pkgdata = PkgData(id, srcdir) + end + for (submod, filename) in Iterators.drop(Base._included_files, 1) # stepping through sysimg.jl rebuilds Base, omit it + ffilename = fixpath(filename) + inpath(ffilename, dirs) || continue + keypath = ffilename[1:last(findfirst(dirs[end], ffilename))] + rpath = relpath(ffilename, keypath) + fullpath = joinpath(basedir(pkgdata), rpath) + if fullpath != filename + cache_file_key[fullpath] = filename + src_file_key[filename] = fullpath + end + push!(pkgdata, rpath=>FileInfo(submod, basesrccache)) + if mtime(ffilename) > mtcache + with_logger(_debug_logger) do + @debug "Recipe for Base/StdLib" _group="Watching" filename=filename mtime=mtime(filename) mtimeref=mtcache + end + push!(modified_files, (pkgdata, rpath)) + end + end + # Add files to CodeTracking pkgfiles + CodeTracking._pkgfiles[id] = pkgdata.info + # Add the files to the watch list + init_watching(pkgdata, srcfiles(pkgdata)) + # Save the result (unnecessary if already in pkgdatas, but doesn't hurt either) + pkgdatas[id] = pkgdata + elseif modname === :Compiler + compilerdir = normpath(joinpath(juliadir, "base", "compiler")) + pkgdata = get(pkgdatas, id, nothing) + if pkgdata === nothing + pkgdata = PkgData(id, compilerdir) + end + track_subdir_from_git!(pkgdata, compilerdir; modified_files=modified_files) + # insertion into pkgdatas is done by track_subdir_from_git! + else + error("no Revise.track recipe for module ", modname) + end + return nothing +end + +# Fix paths to files that define Julia (base and stdlibs) +function fixpath(filename::AbstractString; badpath=basebuilddir, goodpath=juliadir) + startswith(filename, badpath) || return normpath(filename) + filec = filename + relfilename = relpath(filename, badpath) + relfilename0 = relfilename + for strippath in (#joinpath("usr", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)"), + joinpath("usr", "share", "julia"),) + if startswith(relfilename, strippath) + relfilename = relpath(relfilename, strippath) + if occursin("stdlib", relfilename0) && !occursin("stdlib", relfilename) + relfilename = joinpath("stdlib", relfilename) + end + end + end + ffilename = normpath(joinpath(goodpath, relfilename)) + if (isfile(filename) & !isfile(ffilename)) + ffilename = normpath(filename) + end + return ffilename +end +_fixpath(lnn; kwargs...) = LineNumberNode(lnn.line, Symbol(fixpath(String(lnn.file); kwargs...))) +fixpath(lnn::LineNumberNode; kwargs...) = _fixpath(lnn; kwargs...) +fixpath(lnn::Core.LineInfoNode; kwargs...) = _fixpath(lnn; kwargs...) + +# For tracking subdirectories of Julia itself (base/compiler, stdlibs) +function track_subdir_from_git!(pkgdata::PkgData, subdir::AbstractString; commit=Base.GIT_VERSION_INFO.commit, modified_files=revision_queue) + # diff against files at the same commit used to build Julia + repo, repo_path = git_repo(subdir) + if repo == nothing + throw(GitRepoException(subdir)) + end + prefix = relpath(subdir, repo_path) # git-relative path of this subdir + tree = git_tree(repo, commit) + files = Iterators.filter(file->startswith(file, prefix) && endswith(file, ".jl"), keys(tree)) + ccall((:giterr_clear, :libgit2), Cvoid, ()) # necessary to avoid errors like "the global/xdg file 'attributes' doesn't exist: No such file or directory" + for file in files + fullpath = joinpath(repo_path, file) + rpath = relpath(fullpath, pkgdata) # this might undo the above, except for Core.Compiler + local src + try + src = git_source(file, tree) + catch err + if err isa KeyError + @warn "skipping $file, not found in repo" + continue + end + rethrow(err) + end + fmod = get(juliaf2m, fullpath, Core.Compiler) # Core.Compiler is not cached + if fmod === Core.Compiler + endswith(fullpath, "compiler.jl") && continue # defines the module, skip + @static if isdefined(Core.Compiler, :EscapeAnalysis) + # after https://github.com/JuliaLang/julia/pull/43800 + if contains(fullpath, "compiler/ssair/EscapeAnalysis") + fmod = Core.Compiler.EscapeAnalysis + end + end + end + if src != read(fullpath, String) + push!(modified_files, (pkgdata, rpath)) + end + fi = FileInfo(fmod) + if parse_source!(fi.modexsigs, src, file, fmod) === nothing + @warn "failed to parse Git source text for $file" + else + instantiate_sigs!(fi.modexsigs) + end + push!(pkgdata, rpath=>fi) + end + if !isempty(pkgdata.fileinfos) + id = PkgId(pkgdata) + CodeTracking._pkgfiles[id] = pkgdata.info + init_watching(pkgdata, srcfiles(pkgdata)) + pkgdatas[id] = pkgdata + end + return nothing +end + +# For tracking Julia's own stdlibs +const stdlib_names = Set([ + :Base64, :CRC32c, :Dates, :DelimitedFiles, :Distributed, + :FileWatching, :Future, :InteractiveUtils, :Libdl, + :LibGit2, :LinearAlgebra, :Logging, :Markdown, :Mmap, + :OldPkg, :Pkg, :Printf, :Profile, :Random, :REPL, + :Serialization, :SHA, :SharedArrays, :Sockets, :SparseArrays, + :Statistics, :SuiteSparse, :Test, :Unicode, :UUIDs, + :TOML, :Artifacts, :LibCURL_jll, :LibCURL, :MozillaCACerts_jll, + :Downloads, :Tar, :ArgTools, :NetworkOptions]) + +# This replacement is needed because the path written during compilation differs from +# the git source path +const stdlib_rep = joinpath("usr", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)") => "stdlib" + +const juliaf2m = Dict(normpath(replace(file, stdlib_rep))=>mod + for (mod,file) in Base._included_files) diff --git a/packages/Revise/src/relocatable_exprs.jl b/packages/Revise/src/relocatable_exprs.jl new file mode 100644 index 0000000..26d70f7 --- /dev/null +++ b/packages/Revise/src/relocatable_exprs.jl @@ -0,0 +1,141 @@ +# We will need to detect new function bodies, compare function bodies +# to see if they've changed, etc. This has to be done "blind" to the +# line numbers at which the functions are defined. +# +# Now, we could just discard line numbers from expressions, but that +# would have a very negative effect on the quality of backtraces. So +# we keep them, but introduce machinery to compare expressions without +# concern for line numbers. + +""" +A `RelocatableExpr` wraps an `Expr` to ensure that comparisons +between `RelocatableExpr`s ignore line numbering information. +This allows one to detect that two expressions are the same no matter +where they appear in a file. +""" +struct RelocatableExpr + ex::Expr +end + +const ExLike = Union{Expr,RelocatableExpr} + +Base.convert(::Type{Expr}, rex::RelocatableExpr) = rex.ex +Base.convert(::Type{RelocatableExpr}, ex::Expr) = RelocatableExpr(ex) +# Expr(rex::RelocatableExpr) = rex.ex # too costly (inference invalidation) + +Base.copy(rex::RelocatableExpr) = RelocatableExpr(copy(rex.ex)) + +# Implement the required comparison functions. `hash` is needed for Dicts. +function Base.:(==)(ra::RelocatableExpr, rb::RelocatableExpr) + a, b = ra.ex, rb.ex + if a.head == b.head + elseif a.head === :block + a = unwrap(a) + elseif b.head === :block + b = unwrap(b) + end + return a.head == b.head && isequal(LineSkippingIterator(a.args), LineSkippingIterator(b.args)) +end + +const hashrex_seed = UInt == UInt64 ? 0x7c4568b6e99c82d9 : 0xb9c82fd8 +Base.hash(x::RelocatableExpr, h::UInt) = hash(LineSkippingIterator(x.ex.args), + hash(x.ex.head, h + hashrex_seed)) + +function Base.show(io::IO, rex::RelocatableExpr) + show(io, striplines!(copy(rex.ex))) +end + +function striplines!(ex::Expr) + if ex.head === :macrocall + # for macros, the show method in Base assumes the line number is there, + # so don't strip it + args3 = [a isa ExLike ? striplines!(a) : a for a in ex.args[3:end]] + return Expr(ex.head, ex.args[1], nothing, args3...) + end + args = [a isa ExLike ? striplines!(a) : a for a in ex.args] + fargs = collect(LineSkippingIterator(args)) + return Expr(ex.head, fargs...) +end +striplines!(rex::RelocatableExpr) = RelocatableExpr(striplines!(rex.ex)) + +# We could just collect all the non-line statements to a Vector, but +# doing things in-place will be more efficient. + +struct LineSkippingIterator + args::Vector{Any} +end + +Base.IteratorSize(::Type{LineSkippingIterator}) = Base.SizeUnknown() + +function Base.iterate(iter::LineSkippingIterator, i=0) + i = skip_to_nonline(iter.args, i+1) + i > length(iter.args) && return nothing + return (iter.args[i], i) +end + +function skip_to_nonline(args, i) + while true + i > length(args) && return i + ex = args[i] + if isa(ex, Expr) && ex.head === :line + i += 1 + elseif isa(ex, LineNumberNode) + i += 1 + elseif isa(ex, Pair) && (ex::Pair).first === :linenumber # used in the doc system + i += 1 + elseif isa(ex, Base.RefValue) && !isdefined(ex, :x) # also in the doc system + i += 1 + else + return i + end + end +end + +function Base.isequal(itera::LineSkippingIterator, iterb::LineSkippingIterator) + # We could use `zip` here except that we want to insist that the + # iterators also have the same length. + reta, retb = iterate(itera), iterate(iterb) + while true + reta === nothing && retb === nothing && return true + (reta === nothing || retb === nothing) && return false + vala, ia = reta::Tuple{Any,Int} + valb, ib = retb::Tuple{Any,Int} + if isa(vala, Expr) && isa(valb, Expr) + vala, valb = vala::Expr, valb::Expr + vala.head == valb.head || return false + isequal(LineSkippingIterator(vala.args), LineSkippingIterator(valb.args)) || return false + elseif isa(vala, Symbol) && isa(valb, Symbol) + vala, valb = vala::Symbol, valb::Symbol + # two gensymed symbols do not need to match + sa, sb = String(vala), String(valb) + (startswith(sa, '#') && startswith(sb, '#')) || isequal(vala, valb) || return false + elseif isa(vala, Number) && isa(valb, Number) + vala === valb || return false # issue #233 + else + isequal(vala, valb) || return false + end + reta, retb = iterate(itera, ia), iterate(iterb, ib) + end +end + +const hashlsi_seed = UInt === UInt64 ? 0x533cb920dedccdae : 0x2667c89b +function Base.hash(iter::LineSkippingIterator, h::UInt) + h += hashlsi_seed + for x in iter + if x isa Expr + h += hash(LineSkippingIterator(x.args), hash(x.head, h + hashrex_seed)) + elseif x isa Symbol + xs = String(x) + if startswith(xs, '#') # all gensymmed symbols are treated as identical + h += hash("gensym", h) + else + h += hash(x, h) + end + elseif x isa Number + h += hash(typeof(x), hash(x, h))::UInt + else + h += hash(x, h)::UInt + end + end + h +end diff --git a/packages/Revise/src/types.jl b/packages/Revise/src/types.jl new file mode 100644 index 0000000..1b3902b --- /dev/null +++ b/packages/Revise/src/types.jl @@ -0,0 +1,268 @@ +""" + Revise.WatchList + +A struct for holding files that live inside a directory. +Some platforms (OSX) have trouble watching too many files. So we +watch parent directories, and keep track of which files in them +should be tracked. + +Fields: +- `timestamp`: mtime of last update +- `trackedfiles`: Set of filenames, generally expressed as a relative path +""" +mutable struct WatchList + timestamp::Float64 # unix time of last revision + trackedfiles::Dict{String,PkgId} +end + +const DocExprs = Dict{Module,Vector{Expr}} +const ExprsSigs = OrderedDict{RelocatableExpr,Union{Nothing,Vector{Any}}} +const DepDictVals = Tuple{Module,RelocatableExpr} +const DepDict = Dict{Symbol,Set{DepDictVals}} + +function Base.show(io::IO, exsigs::ExprsSigs) + compact = get(io, :compact, false) + if compact + n = 0 + for (rex, sigs) in exsigs + sigs === nothing && continue + n += length(sigs) + end + print(io, "ExprsSigs(<$(length(exsigs)) expressions>, <$n signatures>)") + else + print(io, "ExprsSigs with the following expressions: ") + for def in keys(exsigs) + print(io, "\n ") + Base.show_unquoted(io, RelocatableExpr(unwrap(def)), 2) + end + end +end + +""" + ModuleExprsSigs + +For a particular source file, the corresponding `ModuleExprsSigs` is a mapping +`mod=>exprs=>sigs` of the expressions `exprs` found in `mod` and the signatures `sigs` +that arise from them. Specifically, if `mes` is a `ModuleExprsSigs`, then `mes[mod][ex]` +is a list of signatures that result from evaluating `ex` in `mod`. It is possible that +this returns `nothing`, which can mean either that `ex` does not define any methods +or that the signatures have not yet been cached. + +The first `mod` key is guaranteed to be the module into which this file was `include`d. + +To create a `ModuleExprsSigs` from a source file, see [`Revise.parse_source`](@ref). +""" +const ModuleExprsSigs = OrderedDict{Module,ExprsSigs} + +function Base.typeinfo_prefix(io::IO, mexs::ModuleExprsSigs) + tn = typeof(mexs).name + return string(tn.module, '.', tn.name), true +end + +""" + fm = ModuleExprsSigs(mod::Module) + +Initialize an empty `ModuleExprsSigs` for a file that is `include`d into `mod`. +""" +ModuleExprsSigs(mod::Module) = ModuleExprsSigs(mod=>ExprsSigs()) + +Base.isempty(fm::ModuleExprsSigs) = length(fm) == 1 && isempty(first(values(fm))) + +""" + FileInfo(mexs::ModuleExprsSigs, cachefile="") + +Structure to hold the per-module expressions found when parsing a +single file. +`mexs` holds the [`Revise.ModuleExprsSigs`](@ref) for the file. + +Optionally, a `FileInfo` can also record the path to a cache file holding the original source code. +This is applicable only for precompiled modules and `Base`. +(This cache file is distinct from the original source file that might be edited by the +developer, and it will always hold the state +of the code when the package was precompiled or Julia's `Base` was built.) +When a cache is available, `mexs` will be empty until the file gets edited: +the original source code gets parsed only when a revision needs to be made. + +Source cache files greatly reduce the overhead of using Revise. +""" +struct FileInfo + modexsigs::ModuleExprsSigs + cachefile::String + cacheexprs::Vector{Tuple{Module,Expr}} # "unprocessed" exprs, used to support @require + extracted::Base.RefValue{Bool} # true if signatures have been processed from modexsigs +end +FileInfo(fm::ModuleExprsSigs, cachefile="") = FileInfo(fm, cachefile, Tuple{Module,Expr}[], Ref(false)) + +""" + FileInfo(mod::Module, cachefile="") + +Initialze an empty FileInfo for a file that is `include`d into `mod`. +""" +FileInfo(mod::Module, cachefile::AbstractString="") = FileInfo(ModuleExprsSigs(mod), cachefile) + +FileInfo(fm::ModuleExprsSigs, fi::FileInfo) = FileInfo(fm, fi.cachefile, copy(fi.cacheexprs), Ref(fi.extracted[])) + +function Base.show(io::IO, fi::FileInfo) + print(io, "FileInfo(") + for (mod, exsigs) in fi.modexsigs + show(io, mod) + print(io, "=>") + show(io, exsigs) + print(io, ", ") + end + if !isempty(fi.cachefile) + print(io, "with cachefile ", fi.cachefile) + end + print(io, ')') +end + +""" + PkgData(id, path, fileinfos::Dict{String,FileInfo}) + +A structure holding the data required to handle a particular package. +`path` is the top-level directory defining the package, +and `fileinfos` holds the [`Revise.FileInfo`](@ref) for each file defining the package. + +For the `PkgData` associated with `Main` (e.g., for files loaded with [`includet`](@ref)), +the corresponding `path` entry will be empty. +""" +mutable struct PkgData + info::PkgFiles + fileinfos::Vector{FileInfo} + requirements::Vector{PkgId} +end + +PkgData(id::PkgId, path) = PkgData(PkgFiles(id, path), FileInfo[], PkgId[]) +PkgData(id::PkgId, ::Nothing) = PkgData(id, "") +function PkgData(id::PkgId) + bp = basepath(id) + if !isempty(bp) + bp = normpath(bp) + end + PkgData(id, bp) +end + +# Abstraction interface for PkgData +Base.PkgId(pkgdata::PkgData) = PkgId(pkgdata.info) +CodeTracking.basedir(pkgdata::PkgData) = basedir(pkgdata.info) +CodeTracking.srcfiles(pkgdata::PkgData) = srcfiles(pkgdata.info) + +is_same_file(a, b) = String(a) == String(b) + +function fileindex(info, file) + for (i, f) in enumerate(srcfiles(info)) + is_same_file(f, file) && return i + end + return nothing +end + +function hasfile(info, file) + if isabspath(file) + file = relpath(file, info) + end + fileindex(info, file) !== nothing +end + +function fileinfo(pkgdata::PkgData, file::String) + i = fileindex(pkgdata, file) + i === nothing && error("file ", file, " not found") + return pkgdata.fileinfos[i] +end +fileinfo(pkgdata::PkgData, i::Int) = pkgdata.fileinfos[i] + +function Base.push!(pkgdata::PkgData, pr::Pair{<:Any,FileInfo}) + push!(srcfiles(pkgdata), pr.first) + push!(pkgdata.fileinfos, pr.second) + return pkgdata +end + +function Base.show(io::IO, pkgdata::PkgData) + compact = get(io, :compact, false) + print(io, "PkgData(") + if compact + print(io, '"', pkgdata.info.basedir, "\", ") + nexs, nsigs, nparsed = 0, 0, 0 + for fi in pkgdata.fileinfos + thisnexs, thisnsigs = 0, 0 + for (mod, exsigs) in fi.modexsigs + for (rex, sigs) in exsigs + thisnexs += 1 + sigs === nothing && continue + thisnsigs += length(sigs) + end + end + nexs += thisnexs + nsigs += thisnsigs + if thisnexs > 0 + nparsed += 1 + end + end + print(io, nparsed, '/', length(pkgdata.fileinfos), " parsed files, ", nexs, " expressions, ", nsigs, " signatures)") + else + show(io, pkgdata.info.id) + println(io, ", basedir \"", pkgdata.info.basedir, "\":") + for (f, fi) in zip(pkgdata.info.files, pkgdata.fileinfos) + print(io, " \"", f, "\": ") + show(IOContext(io, :compact=>true), fi) + print(io, '\n') + end + end +end + +function pkgfileless((pkgdata1,file1)::Tuple{PkgData,String}, (pkgdata2,file2)::Tuple{PkgData,String}) + # implements a partial order + PkgId(pkgdata1) ∈ pkgdata2.requirements && return true + PkgId(pkgdata1) == PkgId(pkgdata2) && return fileindex(pkgdata1, file1) < fileindex(pkgdata2, file2) + return false +end + +""" + ReviseEvalException(loc::String, exc::Exception, stacktrace=nothing) + +Provide additional location information about `exc`. + +When running via the interpreter, the backtraces point to interpreter code rather than the original +culprit. This makes it possible to use `loc` to provide information about the frame backtrace, +and even to supply a fake backtrace. + +If `stacktrace` is supplied it must be a `Vector{Any}` containing `(::StackFrame, n)` pairs where `n` +is the recursion count (typically 1). +""" +struct ReviseEvalException <: Exception + loc::String + exc::Exception + stacktrace::Union{Nothing,Vector{Any}} +end +ReviseEvalException(loc::AbstractString, exc::Exception) = ReviseEvalException(loc, exc, nothing) + +function Base.showerror(io::IO, ex::ReviseEvalException; blame_revise::Bool=true) + showerror(io, ex.exc) + st = ex.stacktrace + if st !== nothing + Base.show_backtrace(io, st) + end + if blame_revise + println(io, "\nRevise evaluation error at ", ex.loc) + end +end + +struct GitRepoException <: Exception + filename::String +end + +function Base.showerror(io::IO, ex::GitRepoException) + print(io, "no repository at ", ex.filename, " to track stdlibs you must build Julia from source") +end + +""" + thunk = TaskThunk(f, args) + +To facilitate precompilation and reduce latency, we avoid creation of anonymous thunks. +`thunk` can be used as an argument in `schedule(Task(thunk))`. +""" +struct TaskThunk + f # deliberately untyped + args # deliberately untyped +end + +@noinline (thunk::TaskThunk)() = thunk.f(thunk.args...) diff --git a/packages/Revise/src/utils.jl b/packages/Revise/src/utils.jl new file mode 100644 index 0000000..626474e --- /dev/null +++ b/packages/Revise/src/utils.jl @@ -0,0 +1,205 @@ +relpath_safe(path, startpath) = isempty(startpath) ? path : relpath(path, startpath) + +function Base.relpath(filename, pkgdata::PkgData) + if isabspath(filename) + # `Base.locate_package`, which is how `pkgdata` gets initialized, might strip pieces of the path. + # For example, on Travis macOS the paths returned by `abspath` + # can be preceded by "/private" which is not present in the value returned by `Base.locate_package`. + idx = findfirst(basedir(pkgdata), filename) + if idx !== nothing + idx = first(idx) + if idx > 1 + filename = filename[idx:end] + end + filename = relpath_safe(filename, basedir(pkgdata)) + end + elseif startswith(filename, "compiler") + # Core.Compiler's pkgid includes "compiler/" in the path + filename = relpath(filename, "compiler") + end + return filename +end + +function iswritable(file::AbstractString) # note this trashes the Base definition, but we don't need it + return uperm(stat(file)) & 0x02 != 0x00 +end + +function unique_dirs(iter) + udirs = Set{String}() + for file in iter + dir, basename = splitdir(file) + push!(udirs, dir) + end + return udirs +end + +function file_exists(filename) + filename = normpath(filename) + isfile(filename) && return true + alt = get(cache_file_key, filename, nothing) + alt === nothing && return false + return isfile(alt) +end + +function use_compiled_modules() + return Base.JLOptions().use_compiled_modules != 0 +end + +function firstline(ex::Expr) + for a in ex.args + isa(a, LineNumberNode) && return a + if isa(a, Expr) + line = firstline(a) + isa(line, LineNumberNode) && return line + end + end + return nothing +end +firstline(rex::RelocatableExpr) = firstline(rex.ex) + +newloc(methloc::LineNumberNode, ln, lno) = fixpath(ln) + +location_string(file::AbstractString, line) = abspath(file)*':'*string(line) +location_string(file::Symbol, line) = location_string(string(file), line) + +function linediff(la::LineNumberNode, lb::LineNumberNode) + (isa(la.file, Symbol) && isa(lb.file, Symbol) && (la.file::Symbol === lb.file::Symbol)) || return typemax(Int) + return abs(la.line - lb.line) +end + +# Return the only non-trivial expression in ex, or ex itself +function unwrap(ex::Expr) + if ex.head === :block || ex.head === :toplevel + for (i, a) in enumerate(ex.args) + if isa(a, Expr) + for j = i+1:length(ex.args) + istrivial(ex.args[j]) || return ex + end + return unwrap(a) + elseif !istrivial(a) + return ex + end + end + return nothing + end + return ex +end +unwrap(rex::RelocatableExpr) = RelocatableExpr(unwrap(rex.ex)) + +istrivial(a) = a === nothing || isa(a, LineNumberNode) + +isgoto(stmt) = isa(stmt, Core.GotoNode) | isexpr(stmt, :gotoifnot) + +function pushex!(exsigs::ExprsSigs, ex::Expr) + uex = unwrap(ex) + if is_doc_expr(uex) + body = uex.args[4] + if isa(body, Expr) && body.head !== :call # don't trigger for docexprs like `"docstr" f(x::Int)` + exsigs[RelocatableExpr(body)] = nothing + end + if length(uex.args) < 5 + push!(uex.args, false) + else + uex.args[5] = false + end + end + exsigs[RelocatableExpr(ex)] = nothing + return exsigs +end + +## WatchList utilities +function updatetime!(wl::WatchList) + wl.timestamp = time() +end +Base.push!(wl::WatchList, filenameid::Pair{<:AbstractString,PkgId}) = + push!(wl.trackedfiles, filenameid) +Base.push!(wl::WatchList, filenameid::Pair{<:AbstractString,PkgFiles}) = + push!(wl, filenameid.first=>filenameid.second.id) +Base.push!(wl::WatchList, filenameid::Pair{<:AbstractString,PkgData}) = + push!(wl, filenameid.first=>filenameid.second.info) +WatchList() = WatchList(time(), Dict{String,PkgId}()) +Base.in(file, wl::WatchList) = haskey(wl.trackedfiles, file) + +@static if Sys.isapple() + # HFS+ rounds time to seconds, see #22 + # https://developer.apple.com/library/archive/technotes/tn/tn1150.html#HFSPlusDates + newer(mtime, timestamp) = ceil(mtime) >= floor(timestamp) +else + newer(mtime, timestamp) = mtime >= timestamp +end + +""" + success = throwto_repl(e::Exception) + +Try throwing `e` from the REPL's backend task. Returns `true` if the necessary conditions +were met and the throw can be expected to succeed. The throw is generated from another +task, so a `yield` will need to occur before it happens. +""" +function throwto_repl(e::Exception) + if isdefined(Base, :active_repl_backend) && + Base.active_repl_backend.backend_task.state === :runnable && + isempty(Base.Workqueue) && + Base.active_repl_backend.in_eval + @async Base.throwto(Base.active_repl_backend.backend_task, e) + return true + end + return false +end + +function printf_maxsize(f::Function, io::IO, args...; maxchars::Integer=500, maxlines::Integer=20) + # This is dumb but certain to work + iotmp = IOBuffer() + for a in args + print(iotmp, a) + end + print(iotmp, '\n') + seek(iotmp, 0) + str = read(iotmp, String) + if length(str) > maxchars + str = first(str, (maxchars+1)÷2) * "…" * last(str, maxchars - (maxchars+1)÷2) + end + lines = split(str, '\n') + if length(lines) <= maxlines + for line in lines + f(io, line) + end + return + end + half = (maxlines+1) ÷ 2 + for i = 1:half + f(io, lines[i]) + end + maxlines > 1 && f(io, ⋮) + for i = length(lines) - (maxlines-half) + 1:length(lines) + f(io, lines[i]) + end +end +println_maxsize(args...; kwargs...) = println_maxsize(stdout, args...; kwargs...) +println_maxsize(io::IO, args...; kwargs...) = printf_maxsize(println, io, args...; kwargs...) + +""" + trim_toplevel!(bt) + +Truncate a list of instruction pointers, as obtained from `backtrace()` or `catch_backtrace()`, +at the first "top-level" call (e.g., as executed from the REPL prompt) or the +first entry corresponding to a method in Revise or its dependencies. + +This is used to make stacktraces obtained with Revise more similar to those obtained +without Revise, while retaining one entry to reveal Revise's involvement. +""" +function trim_toplevel!(bt) + # return bt # uncomment this line if you're debugging Revise itself + n = itoplevel = length(bt) + for (i, t) in enumerate(bt) + sfs = StackTraces.lookup(t) + for sf in sfs + if sf.func === Symbol("top-level scope") || (isa(sf.linfo, Core.MethodInstance) && isa(sf.linfo.def, Method) && ((sf.linfo::Core.MethodInstance).def::Method).module ∈ (JuliaInterpreter, LoweredCodeUtils, Revise)) + itoplevel = i + break + end + end + itoplevel < n && break + end + deleteat!(bt, itoplevel+1:length(bt)) + return bt +end diff --git a/packages/Revise/test/backedges.jl b/packages/Revise/test/backedges.jl new file mode 100644 index 0000000..3fb0dd0 --- /dev/null +++ b/packages/Revise/test/backedges.jl @@ -0,0 +1,83 @@ +using Revise, Test +using Base.Meta: isexpr + +isdefined(@__MODULE__, :do_test) || include("common.jl") + +module BackEdgesTest +using Test +flag = false # this needs to be defined for the conditional part to work +end + +do_test("Backedges") && @testset "Backedges" begin + src = Meta.lower(Base, :(max_values(T::Union{map(X -> Type{X}, BitIntegerSmall_types)...}) = 1 << (8*sizeof(T)))).args[1] + # Find the inner struct def for the anonymous function + idtype = findall(stmt->isexpr(stmt, :thunk) && isa(stmt.args[1], Core.CodeInfo), src.code)[end] + src2 = src.code[idtype].args[1] + methodinfo = Revise.MethodInfo() + isrequired = Revise.minimal_evaluation!(methodinfo, src, :sigs)[1] + @test sum(isrequired) == length(src.code)-2 # skips the `return` at the end + + src = """ + # issue #249 + flag = false + if flag + f() = 1 + else + f() = 2 + end + + # don't do work in the interpreter that isn't needed for function definitions + # inspired by #300 + const planetdiameters = Dict("Mercury" => 4_878) + planetdiameters["Venus"] = 12_104 + + function getdiameter(name) + return planetdiameters[name] + end + """ + mexs = Revise.parse_source!(Revise.ModuleExprsSigs(BackEdgesTest), src, "backedges_test.jl", BackEdgesTest) + Revise.moduledeps[BackEdgesTest] = Revise.DepDict() + Revise.instantiate_sigs!(mexs) + @test isempty(methods(BackEdgesTest.getdiameter)) + @test !isdefined(BackEdgesTest, :planetdiameters) + @test length(Revise.moduledeps[BackEdgesTest]) == 1 + @test Revise.moduledeps[BackEdgesTest][:flag] == Set([(BackEdgesTest, first(Iterators.drop(mexs[BackEdgesTest], 1))[1])]) + + # issue #399 + src = """ + for jy in ("j","y"), nu in (0,1) + jynu = Expr(:quote, Symbol(jy,nu)) + jynuf = Expr(:quote, Symbol(jy,nu,"f")) + bjynu = Symbol("bessel",jy,nu) + if jy == "y" + @eval begin + \$bjynu(x::Float64) = nan_dom_err(ccall((\$jynu,libm), Float64, (Float64,), x), x) + \$bjynu(x::Float32) = nan_dom_err(ccall((\$jynuf,libm), Float32, (Float32,), x), x) + \$bjynu(x::Float16) = Float16(\$bjynu(Float32(x))) + end + else + @eval begin + \$bjynu(x::Float64) = ccall((\$jynu,libm), Float64, (Float64,), x) + \$bjynu(x::Float32) = ccall((\$jynuf,libm), Float32, (Float32,), x) + \$bjynu(x::Float16) = Float16(\$bjynu(Float32(x))) + end + end + @eval begin + \$bjynu(x::Real) = \$bjynu(float(x)) + \$bjynu(x::Complex) = \$(Symbol("bessel",jy))(\$nu,x) + end + end + """ + ex = Meta.parse(src) + @test Revise.methods_by_execution(BackEdgesTest, ex) isa Tuple + + # Issue #428 + src = """ + @testset for i in (1, 2) + @test i == i + end + """ + ex = Meta.parse(src) + @test Revise.methods_by_execution(BackEdgesTest, ex) isa Tuple + +end diff --git a/packages/Revise/test/callee_error.jl b/packages/Revise/test/callee_error.jl new file mode 100644 index 0000000..9763062 --- /dev/null +++ b/packages/Revise/test/callee_error.jl @@ -0,0 +1,16 @@ +module CalleeError + +inner(A, i) = A[i] +function outer(A) + s = zero(eltype(A)) + for i = 1:length(A)+1 + s += inner(A, i) + end + return s +end + +s = outer([1,2,3]) + +foo(x::Float32) = 1 + +end diff --git a/packages/Revise/test/common.jl b/packages/Revise/test/common.jl new file mode 100644 index 0000000..c2c90b0 --- /dev/null +++ b/packages/Revise/test/common.jl @@ -0,0 +1,99 @@ +using Random +using Base.Meta: isexpr + +const rseed = Ref(Random.GLOBAL_RNG) # to get new random directories (see julia #24445) +if isempty(methods(Random.seed!, Tuple{typeof(rseed[])})) + # Julia 1.3-rc1 doesn't have this, fixed in https://github.com/JuliaLang/julia/pull/32961 + Random.seed!(rng::typeof(rseed[])) = Random.seed!(rng, nothing) +end +function randtmp() + Random.seed!(rseed[]) + dirname = joinpath(tempdir(), randstring(10)) + rseed[] = Random.GLOBAL_RNG + return dirname +end + +const to_remove = String[] + +function newtestdir() + testdir = randtmp() + mkdir(testdir) + push!(to_remove, testdir) + push!(LOAD_PATH, testdir) + return testdir +end + +@static if Sys.isapple() + const mtimedelay = 3.1 # so the defining files are old enough not to trigger mtime criterion +else + const mtimedelay = 0.1 +end + +yry() = (sleep(mtimedelay); revise(); sleep(mtimedelay)) + +function collectexprs(rex::Revise.RelocatableExpr) + items = [] + for item in Revise.LineSkippingIterator(rex.ex.args) + push!(items, isa(item, Expr) ? Revise.RelocatableExpr(item) : item) + end + items +end + +function get_docstring(obj) + while !isa(obj, AbstractString) + fn = fieldnames(typeof(obj)) + if :content ∈ fn + obj = obj.content[1] + elseif :code ∈ fn + obj = obj.code + else + error("unknown object ", obj) + end + end + return obj +end + +function get_code(f, typ) + # Julia 1.5 introduces ":code_coverage_effect" exprs + ci = code_typed(f, typ)[1].first + code = copy(ci.code) + while !isempty(code) && isexpr(code[1], :code_coverage_effect) + popfirst!(code) + end + return code +end + +function do_test(name) + runtest = isempty(ARGS) || name in ARGS + # Sometimes we get "no output received for 10 minutes" on CI, + # to debug this it may be useful to know what test is being run. + runtest && haskey(ENV, "CI") && println("Starting test ", name) + return runtest +end + +function rm_precompile(pkgname::AbstractString) + filepath = Base.cache_file_entry(Base.PkgId(pkgname)) + isa(filepath, Tuple) && (filepath = filepath[1]*filepath[2]) # Julia 1.3+ + for depot in DEPOT_PATH + fullpath = joinpath(depot, filepath) + isfile(fullpath) && rm(fullpath) + end +end + +function isreturning(stmt, val) + isa(stmt, Core.ReturnNode) || return false + return stmt.val == val +end +function isreturning_slot(stmt, val) + isa(stmt, Core.ReturnNode) || return false + v = stmt.val + isa(v, Core.SlotNumber) || isa(v, Core.Argument) || return false + return (isa(v, Core.SlotNumber) ? v.id : v.n) == val +end + +if !isempty(ARGS) && "REVISE_TESTS_WATCH_FILES" ∈ ARGS + Revise.watching_files[] = true + println("Running tests with `Revise.watching_files[] = true`") + idx = findall(isequal("REVISE_TESTS_WATCH_FILES"), ARGS) + deleteat!(ARGS, idx) +end diff --git a/packages/Revise/test/envs/use_exputils/install.jl b/packages/Revise/test/envs/use_exputils/install.jl new file mode 100644 index 0000000..367dc1f --- /dev/null +++ b/packages/Revise/test/envs/use_exputils/install.jl @@ -0,0 +1,8 @@ +# This gets run by setup.sh + +using Pkg + +Pkg.activate(@__DIR__) +Pkg.add(name="ExponentialUtilities", version=ARGS[1]) + +using ExponentialUtilities diff --git a/packages/Revise/test/envs/use_exputils/setup.sh b/packages/Revise/test/envs/use_exputils/setup.sh new file mode 100644 index 0000000..9a854e2 --- /dev/null +++ b/packages/Revise/test/envs/use_exputils/setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Ensure that packages that get updated indirectly by external +# manifest updates work as well as can be hoped. +# See https://github.com/timholy/Revise.jl/issues/647 + +# Install and compile two versions of ExponentialUtilities +dn=$(dirname "$BASH_SOURCE") +julia $dn/install.jl "1.10.0" +julia $dn/install.jl "1.9.0" diff --git a/packages/Revise/test/envs/use_exputils/switch_version.jl b/packages/Revise/test/envs/use_exputils/switch_version.jl new file mode 100644 index 0000000..b2d22bd --- /dev/null +++ b/packages/Revise/test/envs/use_exputils/switch_version.jl @@ -0,0 +1,25 @@ +# This test is intended to be run after `setup.sh` runs. ExponentialUtilities +# should be held at v1.9.0. + +using Revise, Pkg, Test + +const thisdir = dirname(@__FILE__) +Pkg.activate(thisdir) +# This is only needed on Pkg versions that don't notify +Revise.active_project_watcher() + +using ExponentialUtilities +id = Base.PkgId(ExponentialUtilities) +pkgdata = Revise.pkgdatas[id] +A = rand(3, 3); A = A'*A; A = A' + A; +@test_throws UndefVarError exponential!(A) # not present on v1.9 +# From a different process, switch the active version of ExponentialUtilities +run(Cmd(`julia -e 'using Pkg; Pkg.activate("."); Pkg.add(name="ExponentialUtilities", version="1.10.0")'`; dir=thisdir)) +sleep(0.2) +revise() +@test exponential!(A) isa Matrix # present on v1.9 +# ...and then switch back (check that it's bidirectional and also to reset state) +run(Cmd(`julia -e 'using Pkg; Pkg.activate("."); Pkg.add(name="ExponentialUtilities", version="1.9.0")'`; dir=thisdir)) +sleep(0.2) +revise() +@test_throws MethodError exponential!(A) # not present on v1.9 diff --git a/packages/Revise/test/fake_lang/new_test.program b/packages/Revise/test/fake_lang/new_test.program new file mode 100644 index 0000000..49cdc5b --- /dev/null +++ b/packages/Revise/test/fake_lang/new_test.program @@ -0,0 +1 @@ +2=x diff --git a/packages/Revise/test/fake_lang/test.program b/packages/Revise/test/fake_lang/test.program new file mode 100644 index 0000000..b2d292b --- /dev/null +++ b/packages/Revise/test/fake_lang/test.program @@ -0,0 +1,2 @@ +1=x +2=y diff --git a/packages/Revise/test/inotify.jl b/packages/Revise/test/inotify.jl new file mode 100644 index 0000000..a7dac2d --- /dev/null +++ b/packages/Revise/test/inotify.jl @@ -0,0 +1,11 @@ +using Revise, Test +# This test should only be run if you have a very small inotify limit + +@testset "inotify" begin + logs, _ = Test.collect_test_logs() do + Revise.track("revisetest.jl") + end + sleep(0.1) + @test !isempty(logs) + @test any(rec->occursin("inotify", rec.message), logs) +end diff --git a/packages/Revise/test/juliadir.jl b/packages/Revise/test/juliadir.jl new file mode 100644 index 0000000..a1b8599 --- /dev/null +++ b/packages/Revise/test/juliadir.jl @@ -0,0 +1,11 @@ +using Revise, InteractiveUtils, Test + +@eval Revise juliadir = ARGS[1] + +@test Revise.juliadir != Revise.basebuilddir +@test Revise.juliadir != Revise.fallback_juliadir() + +@show Revise.juliadir + +# https://github.com/timholy/Revise.jl/issues/697 +@test Revise.definition(@which(Float32(π))) isa Expr diff --git a/packages/Revise/test/non_jl_test.jl b/packages/Revise/test/non_jl_test.jl new file mode 100644 index 0000000..b1f5a5f --- /dev/null +++ b/packages/Revise/test/non_jl_test.jl @@ -0,0 +1,48 @@ +struct MyFile + file::String +end +Base.abspath(file::MyFile) = MyFile(Base.abspath(file.file)) +Base.isabspath(file::MyFile) = Base.isabspath(file.file) +Base.joinpath(str::String, file::MyFile) = MyFile(Base.joinpath(str, file.file)) +Base.normpath(file::MyFile) = MyFile(Base.normpath(file.file)) +Base.isfile(file::MyFile) = Base.isfile(file.file) +Base.findfirst(str::String, file::MyFile) = Base.findfirst(str, file.file) +Base.String(file::MyFile) = file.file + +function make_module(file::MyFile) + exprs = [] + for line in eachline(file.file) + val, name = split(line, '=') + push!(exprs, :(function $(Symbol(name))() $val end)) + end + Expr(:toplevel, :(baremodule fake_lang + $(exprs...) + end), :(using .fake_lang)) +end + +function Base.include(mod::Module, file::MyFile) + Core.eval(mod, make_module(file)) +end +Base.include(file::MyFile) = Base.include(Core.Main, file) + +using Revise +function Revise.parse_source!(mod_exprs_sigs::Revise.ModuleExprsSigs, file::MyFile, mod::Module; kwargs...) + ex = make_module(file) + Revise.process_source!(mod_exprs_sigs, ex, file, mod; kwargs...) +end + +path = joinpath(@__DIR__, "test.program") +try + cp(joinpath(@__DIR__, "fake_lang", "test.program"), path, force=true) + m=MyFile(path) + includet(m) + yry() # comes from test/common.jl + @test fake_lang.y() == "2" + @test fake_lang.x() == "1" + cp(joinpath(@__DIR__, "fake_lang", "new_test.program"), path, force=true) + Revise.revise() + @test fake_lang.x() == "2" + @test_throws MethodError fake_lang.y() +finally + rm(path, force=true) +end diff --git a/packages/Revise/test/pkgs/Dep442A/Project.toml b/packages/Revise/test/pkgs/Dep442A/Project.toml new file mode 100644 index 0000000..eb42e8c --- /dev/null +++ b/packages/Revise/test/pkgs/Dep442A/Project.toml @@ -0,0 +1,4 @@ +name = "Dep442A" +uuid = "76238f47-ed95-4e4a-a4d9-95a3fb1630ea" +authors = ["Tim Holy "] +version = "0.1.0" diff --git a/packages/Revise/test/pkgs/Dep442A/src/Dep442A.jl b/packages/Revise/test/pkgs/Dep442A/src/Dep442A.jl new file mode 100644 index 0000000..6989bba --- /dev/null +++ b/packages/Revise/test/pkgs/Dep442A/src/Dep442A.jl @@ -0,0 +1,7 @@ +module Dep442A + +export check442A + +check442A() = true + +end # module diff --git a/packages/Revise/test/pkgs/Dep442B/Project.toml b/packages/Revise/test/pkgs/Dep442B/Project.toml new file mode 100644 index 0000000..038e219 --- /dev/null +++ b/packages/Revise/test/pkgs/Dep442B/Project.toml @@ -0,0 +1,7 @@ +name = "Dep442B" +uuid = "880097ba-c503-4edb-bd3f-4c6394f19e96" +authors = ["Tim Holy "] +version = "0.1.0" + +[deps] +Requires = "ae029012-a4dd-5104-9daa-d747884805df" diff --git a/packages/Revise/test/pkgs/Dep442B/src/Dep442B.jl b/packages/Revise/test/pkgs/Dep442B/src/Dep442B.jl new file mode 100644 index 0000000..7241444 --- /dev/null +++ b/packages/Revise/test/pkgs/Dep442B/src/Dep442B.jl @@ -0,0 +1,18 @@ +module Dep442B + +using Requires + +export check442B + +check442B() = true + +function link_442A() + @debug "Loading 442A support into 442B" + include("support_442A.jl") +end + +function __init__() + @require Dep442A="76238f47-ed95-4e4a-a4d9-95a3fb1630ea" link_442A() +end + +end # module diff --git a/packages/Revise/test/pkgs/Dep442B/src/support_442A.jl b/packages/Revise/test/pkgs/Dep442B/src/support_442A.jl new file mode 100644 index 0000000..4ceac6a --- /dev/null +++ b/packages/Revise/test/pkgs/Dep442B/src/support_442A.jl @@ -0,0 +1 @@ +has442A() = true diff --git a/packages/Revise/test/pkgs/ExcludeFile/Project.toml b/packages/Revise/test/pkgs/ExcludeFile/Project.toml new file mode 100644 index 0000000..df61805 --- /dev/null +++ b/packages/Revise/test/pkgs/ExcludeFile/Project.toml @@ -0,0 +1,4 @@ +name = "ExcludeFile" +uuid = "b915cca1-7962-4ffb-a1c7-2bbdb2d9c14c" +authors = ["Tim Holy "] +version = "0.1.0" diff --git a/packages/Revise/test/pkgs/ExcludeFile/deps/dependency.txt b/packages/Revise/test/pkgs/ExcludeFile/deps/dependency.txt new file mode 100644 index 0000000..4422dba --- /dev/null +++ b/packages/Revise/test/pkgs/ExcludeFile/deps/dependency.txt @@ -0,0 +1 @@ +This is not a parseable Julia file diff --git a/packages/Revise/test/pkgs/ExcludeFile/src/ExcludeFile.jl b/packages/Revise/test/pkgs/ExcludeFile/src/ExcludeFile.jl new file mode 100644 index 0000000..bc6c903 --- /dev/null +++ b/packages/Revise/test/pkgs/ExcludeFile/src/ExcludeFile.jl @@ -0,0 +1,6 @@ +module ExcludeFile + +include_dependency(joinpath(dirname(@__DIR__), "deps", "dependency.txt")) +include("f.jl") + +end # module diff --git a/packages/Revise/test/pkgs/ExcludeFile/src/f.jl b/packages/Revise/test/pkgs/ExcludeFile/src/f.jl new file mode 100644 index 0000000..be011ee --- /dev/null +++ b/packages/Revise/test/pkgs/ExcludeFile/src/f.jl @@ -0,0 +1 @@ +f() = 1 diff --git a/packages/Revise/test/pkgs/Pkg442/Project.toml b/packages/Revise/test/pkgs/Pkg442/Project.toml new file mode 100644 index 0000000..a42619f --- /dev/null +++ b/packages/Revise/test/pkgs/Pkg442/Project.toml @@ -0,0 +1,8 @@ +name = "Pkg442" +uuid = "753a5fd8-985c-4be4-bc89-999c80d1e0e5" +authors = ["Tim Holy "] +version = "0.1.0" + +[deps] +Dep442A = "76238f47-ed95-4e4a-a4d9-95a3fb1630ea" +Dep442B = "880097ba-c503-4edb-bd3f-4c6394f19e96" diff --git a/packages/Revise/test/pkgs/Pkg442/src/Pkg442.jl b/packages/Revise/test/pkgs/Pkg442/src/Pkg442.jl new file mode 100644 index 0000000..8dbca00 --- /dev/null +++ b/packages/Revise/test/pkgs/Pkg442/src/Pkg442.jl @@ -0,0 +1,10 @@ +module Pkg442 + +using Dep442A +using Dep442B + +export check442 + +check442() = true + +end # module diff --git a/packages/Revise/test/polling.jl b/packages/Revise/test/polling.jl new file mode 100644 index 0000000..9aa3b51 --- /dev/null +++ b/packages/Revise/test/polling.jl @@ -0,0 +1,51 @@ +using Revise +using Test + +include("common.jl") + +@testset "Polling" begin + @test Revise.polling_files[] + + testdir = randtmp() + mkdir(testdir) + push!(LOAD_PATH, testdir) + dn = joinpath(testdir, "Polling", "src") + mkpath(dn) + srcfile = joinpath(dn, "Polling.jl") + joinpath(dn, "Polling.jl") + open(srcfile, "w") do io + println(io, """ +__precompile__(false) + +module Polling + +f() = 1 + +end +""") + end + sleep(0.5) # let the source file age a bit + @eval using Polling + @test Polling.f() == 1 + # I'm not sure why 2 sleeps are better than one, but here it seems to make a difference + sleep(0.1) + sleep(0.1) + open(srcfile, "w") do io + println(io, """ +__precompile__(false) + +module Polling + +f() = 2 + +end +""") + end + # Wait through the polling interval + yry() + sleep(7) + yry() + @test Polling.f() == 2 + + rm(testdir; force=true, recursive=true) +end diff --git a/packages/Revise/test/populate_compiled.jl b/packages/Revise/test/populate_compiled.jl new file mode 100644 index 0000000..1f3484a --- /dev/null +++ b/packages/Revise/test/populate_compiled.jl @@ -0,0 +1,16 @@ +# This runs only on Travis. The goal is to populate the `.julia/compiled/v*` directory +# with some additional files, so that `filter_valid_cachefiles` has to run. +# This is to catch problems like #460. + +using Pkg +using Test + +Pkg.add(PackageSpec(name="EponymTuples", version="0.2.0")) +using EponymTuples # force compilation +id = Base.PkgId(EponymTuples) +paths = Base.find_all_in_cache_path(id) +Pkg.rm("EponymTuples") # we don't need it anymore +path = first(paths) +base, ext = splitext(path) +mv(path, base*"blahblah"*ext) +Pkg.add(PackageSpec(name="EponymTuples")) diff --git a/packages/Revise/test/revisetest.jl b/packages/Revise/test/revisetest.jl new file mode 100644 index 0000000..56ce5b2 --- /dev/null +++ b/packages/Revise/test/revisetest.jl @@ -0,0 +1,24 @@ +__precompile__(false) + +module ReviseTest + +square(x) = x^2 + +cube(x) = x^4 # should be x^3, but this simulates a mistake + +module Internal + +mult2(x) = 2*x +mult3(x) = 4*x # oops +mult4(x) = -x + +""" +This has a docstring +""" +unchanged(x) = x + +unchanged2(@nospecialize(x)) = x + +end # Internal + +end # ReviseTest diff --git a/packages/Revise/test/revisetest_errors.jl b/packages/Revise/test/revisetest_errors.jl new file mode 100644 index 0000000..89066dd --- /dev/null +++ b/packages/Revise/test/revisetest_errors.jl @@ -0,0 +1,25 @@ +__precompile__(false) + +module ReviseTest + +square(x) = x^2 + +cube(x) = error("cube") + +fourth(x) = x^4 # this is an addition to the file + +module Internal + +mult2(x) = error("mult2") +mult3(x) = 3*x + +""" +This has a docstring +""" +unchanged(x) = x + +unchanged2(@nospecialize(x)) = x + +end # Internal + +end diff --git a/packages/Revise/test/revisetest_revised.jl b/packages/Revise/test/revisetest_revised.jl new file mode 100644 index 0000000..df2852d --- /dev/null +++ b/packages/Revise/test/revisetest_revised.jl @@ -0,0 +1,25 @@ +__precompile__(false) + +module ReviseTest + +square(x) = x^2 + +cube(x) = x^3 + +fourth(x) = x^4 # this is an addition to the file + +module Internal + +mult2(x) = 2*x +mult3(x) = 3*x + +""" +This has a docstring +""" +unchanged(x) = x + +unchanged2(@nospecialize(x)) = x + +end # Internal + +end diff --git a/packages/Revise/test/runtests.jl b/packages/Revise/test/runtests.jl new file mode 100644 index 0000000..57974a3 --- /dev/null +++ b/packages/Revise/test/runtests.jl @@ -0,0 +1,3676 @@ +# REVISE: DO NOT PARSE # For people with JULIA_REVISE_INCLUDE=1 +using Revise +using Revise.CodeTracking +using Revise.JuliaInterpreter +using Test + +@test isempty(detect_ambiguities(Revise)) + +using Pkg, Unicode, Distributed, InteractiveUtils, REPL, UUIDs +import LibGit2 +using Revise.OrderedCollections: OrderedSet +using Test: collect_test_logs +using Base.CoreLogging: Debug,Info + +using Revise.CodeTracking: line_is_decl + +# In addition to using this for the "More arg-modifying macros" test below, +# this package is used on Travis to test what happens when you have multiple +# *.ji files for the package. +using EponymTuples + +include("common.jl") + +throwing_function(bt) = bt[2] + +# A junk module that we can evaluate into +module ReviseTestPrivate +struct Inner + x::Float64 +end + +macro changeto1(args...) + return 1 +end + +macro donothing(ex) + esc(ex) +end + +macro addint(ex) + :($(esc(ex))::$(esc(Int))) +end + +# The following two submodules are for testing #199 +module A +f(x::Int) = 1 +end + +module B +f(x::Int) = 1 +module Core end +end + +end + +function private_module() + modname = gensym() + Core.eval(ReviseTestPrivate, :(module $modname end)) +end + +sig_type_exprs(ex) = Revise.sig_type_exprs(Main, ex) # just for testing purposes + +# accomodate changes in Dict printing w/ Julia version +const pair_op_compact = let io = IOBuffer() + print(IOContext(io, :compact=>true), Dict(1=>2)) + String(take!(io))[7:end-2] +end + +const issue639report = [] + +@testset "Revise" begin + do_test("PkgData") && @testset "PkgData" begin + # Related to #358 + id = Base.PkgId(Main) + pd = Revise.PkgData(id) + @test isempty(Revise.basedir(pd)) + end + + do_test("Package contents") && @testset "Package contents" begin + id = Base.PkgId(EponymTuples) + path, mods_files_mtimes = Revise.pkg_fileinfo(id) + @test occursin("EponymTuples", path) + end + + do_test("LineSkipping") && @testset "LineSkipping" begin + rex = Revise.RelocatableExpr(quote + f(x) = x^2 + g(x) = sin(x) + end) + @test length(rex.ex.args) == 4 # including the line number expressions + exs = collectexprs(rex) + @test length(exs) == 2 + @test isequal(exs[1], Revise.RelocatableExpr(:(f(x) = x^2))) + @test hash(exs[1]) == hash(Revise.RelocatableExpr(:(f(x) = x^2))) + @test !isequal(exs[2], Revise.RelocatableExpr(:(f(x) = x^2))) + @test isequal(exs[2], Revise.RelocatableExpr(:(g(x) = sin(x)))) + @test !isequal(exs[1], Revise.RelocatableExpr(:(g(x) = sin(x)))) + @test string(rex) == """ + quote + f(x) = begin + x ^ 2 + end + g(x) = begin + sin(x) + end + end""" + end + + do_test("Equality and hashing") && @testset "Equality and hashing" begin + # issue #233 + @test isequal(Revise.RelocatableExpr(:(x = 1)), Revise.RelocatableExpr(:(x = 1))) + @test !isequal(Revise.RelocatableExpr(:(x = 1)), Revise.RelocatableExpr(:(x = 1.0))) + @test hash(Revise.RelocatableExpr(:(x = 1))) == hash(Revise.RelocatableExpr(:(x = 1))) + @test hash(Revise.RelocatableExpr(:(x = 1))) != hash(Revise.RelocatableExpr(:(x = 1.0))) + @test hash(Revise.RelocatableExpr(:(x = 1))) != hash(Revise.RelocatableExpr(:(x = 2))) + end + + do_test("Parse errors") && @testset "Parse errors" begin + md = Revise.ModuleExprsSigs(Main) + @test_throws LoadError Revise.parse_source!(md, """ + begin # this block should parse correctly, cf. issue #109 + + end + f(x) = 1 + g(x) = 2 + h{x) = 3 # error + k(x) = 4 + """, "test", Main) + + # Issue #448 + testdir = newtestdir() + file = joinpath(testdir, "badfile.jl") + write(file, """ + function g() + while t + c = + k + end + """) + try + includet(file) + catch err + @test isa(err, LoadError) + @test err.file == file + @test endswith(err.error, "requires end") + end + end + + do_test("REPL input") && @testset "REPL input" begin + # issue #573 + retex = Revise.revise_first(nothing) + @test retex.head === :toplevel + @test length(retex.args) == 2 && retex.args[end] === nothing + end + + do_test("Signature extraction") && @testset "Signature extraction" begin + jidir = dirname(dirname(pathof(JuliaInterpreter))) + scriptfile = joinpath(jidir, "test", "toplevel_script.jl") + modex = :(module Toplevel include($scriptfile) end) + mod = eval(modex) + mexs = Revise.parse_source(scriptfile, mod) + Revise.instantiate_sigs!(mexs) + nms = names(mod; all=true) + modeval, modinclude = getfield(mod, :eval), getfield(mod, :include) + failed = [] + n = 0 + for fsym in nms + f = getfield(mod, fsym) + isa(f, Base.Callable) || continue + (f === modeval || f === modinclude) && continue + for m in methods(f) + # MyInt8 brings in lots of number & type machinery, which leads + # to wandering through Base files. At this point we just want + # to test whether we have the basics down, so for now avoid + # looking in any file other than the script + string(m.file) == scriptfile || continue + isa(definition(m), Expr) || push!(failed, m.sig) + n += 1 + end + end + @test isempty(failed) + @test n > length(nms)/2 + + # Method expressions with bad line number info + ex = quote + function nolineinfo(x) + y = x^2 + 2x + 1 + @warn "oops" + return y + end + end + ex2 = ex.args[end].args[end] + for (i, arg) in enumerate(ex2.args) + if isa(arg, LineNumberNode) + ex2.args[i] = LineNumberNode(0, :none) + end + end + mexs = Revise.ModuleExprsSigs(ReviseTestPrivate) + mexs[ReviseTestPrivate][Revise.RelocatableExpr(ex)] = nothing + logs, _ = Test.collect_test_logs() do + Revise.instantiate_sigs!(mexs; mode=:eval) + end + @test isempty(logs) + @test isdefined(ReviseTestPrivate, :nolineinfo) + end + + do_test("Comparison and line numbering") && @testset "Comparison and line numbering" begin + # We'll also use these tests to try out the logging system + rlogger = Revise.debug_logger() + + fl1 = joinpath(@__DIR__, "revisetest.jl") + fl2 = joinpath(@__DIR__, "revisetest_revised.jl") + fl3 = joinpath(@__DIR__, "revisetest_errors.jl") + + # Copy the files to a temporary file. This is to ensure that file name doesn't change + # in docstring macros and backtraces. + tmpfile = joinpath(tempdir(), randstring(10))*".jl" + push!(to_remove, tmpfile) + + cp(fl1, tmpfile) + include(tmpfile) # So the modules are defined + # test the "mistakes" + @test ReviseTest.cube(2) == 16 + @test ReviseTest.Internal.mult3(2) == 8 + @test ReviseTest.Internal.mult4(2) == -2 + # One method will be deleted, for log testing we need to grab it while we still have it + delmeth = first(methods(ReviseTest.Internal.mult4)) + mmult3 = @which ReviseTest.Internal.mult3(2) + + mexsold = Revise.parse_source(tmpfile, Main) + Revise.instantiate_sigs!(mexsold) + mcube = @which ReviseTest.cube(2) + + cp(fl2, tmpfile; force=true) + mexsnew = Revise.parse_source(tmpfile, Main) + mexsnew = Revise.eval_revised(mexsnew, mexsold) + @test ReviseTest.cube(2) == 8 + @test ReviseTest.Internal.mult3(2) == 6 + + @test length(mexsnew) == 3 + @test haskey(mexsnew, ReviseTest) && haskey(mexsnew, ReviseTest.Internal) + + dvs = collect(mexsnew[ReviseTest]) + @test length(dvs) == 3 + (def, val) = dvs[1] + @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(square(x) = x^2))) + @test val == [Tuple{typeof(ReviseTest.square),Any}] + @test Revise.firstline(Revise.unwrap(def)).line == 5 + m = @which ReviseTest.square(1) + @test m.line == 5 + @test whereis(m) == (tmpfile, 5) + @test Revise.RelocatableExpr(definition(m)) == Revise.unwrap(def) + (def, val) = dvs[2] + @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(cube(x) = x^3))) + @test val == [Tuple{typeof(ReviseTest.cube),Any}] + m = @which ReviseTest.cube(1) + @test m.line == 7 + @test whereis(m) == (tmpfile, 7) + @test Revise.RelocatableExpr(definition(m)) == Revise.unwrap(def) + (def, val) = dvs[3] + @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(fourth(x) = x^4))) + @test val == [Tuple{typeof(ReviseTest.fourth),Any}] + m = @which ReviseTest.fourth(1) + @test m.line == 9 + @test whereis(m) == (tmpfile, 9) + @test Revise.RelocatableExpr(definition(m)) == Revise.unwrap(def) + + dvs = collect(mexsnew[ReviseTest.Internal]) + @test length(dvs) == 5 + (def, val) = dvs[1] + @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(mult2(x) = 2*x))) + @test val == [Tuple{typeof(ReviseTest.Internal.mult2),Any}] + @test Revise.firstline(Revise.unwrap(def)).line == 13 + m = @which ReviseTest.Internal.mult2(1) + @test m.line == 11 + @test whereis(m) == (tmpfile, 13) + @test Revise.RelocatableExpr(definition(m)) == Revise.unwrap(def) + (def, val) = dvs[2] + @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(mult3(x) = 3*x))) + @test val == [Tuple{typeof(ReviseTest.Internal.mult3),Any}] + m = @which ReviseTest.Internal.mult3(1) + @test m.line == 14 + @test whereis(m) == (tmpfile, 14) + @test Revise.RelocatableExpr(definition(m)) == Revise.unwrap(def) + + @test_throws MethodError ReviseTest.Internal.mult4(2) + + function cmpdiff(record, msg; kwargs...) + record.message == msg + for (kw, val) in kwargs + logval = record.kwargs[kw] + for (v, lv) in zip(val, logval) + isa(v, Expr) && (v = Revise.RelocatableExpr(v)) + isa(lv, Expr) && (lv = Revise.RelocatableExpr(Revise.unwrap(lv))) + @test lv == v + end + end + return nothing + end + logs = filter(r->r.level==Debug && r.group=="Action", rlogger.logs) + @test length(logs) == 9 + cmpdiff(logs[1], "DeleteMethod"; deltainfo=(Tuple{typeof(ReviseTest.cube),Any}, MethodSummary(mcube))) + cmpdiff(logs[2], "DeleteMethod"; deltainfo=(Tuple{typeof(ReviseTest.Internal.mult3),Any}, MethodSummary(mmult3))) + cmpdiff(logs[3], "DeleteMethod"; deltainfo=(Tuple{typeof(ReviseTest.Internal.mult4),Any}, MethodSummary(delmeth))) + cmpdiff(logs[4], "Eval"; deltainfo=(ReviseTest, :(cube(x) = x^3))) + cmpdiff(logs[5], "Eval"; deltainfo=(ReviseTest, :(fourth(x) = x^4))) + stmpfile = Symbol(tmpfile) + cmpdiff(logs[6], "LineOffset"; deltainfo=(Any[Tuple{typeof(ReviseTest.Internal.mult2),Any}], LineNumberNode(11,stmpfile)=>LineNumberNode(13,stmpfile))) + cmpdiff(logs[7], "Eval"; deltainfo=(ReviseTest.Internal, :(mult3(x) = 3*x))) + cmpdiff(logs[8], "LineOffset"; deltainfo=(Any[Tuple{typeof(ReviseTest.Internal.unchanged),Any}], LineNumberNode(18,stmpfile)=>LineNumberNode(19,stmpfile))) + cmpdiff(logs[9], "LineOffset"; deltainfo=(Any[Tuple{typeof(ReviseTest.Internal.unchanged2),Any}], LineNumberNode(20,stmpfile)=>LineNumberNode(21,stmpfile))) + @test length(Revise.actions(rlogger)) == 6 # by default LineOffset is skipped + @test length(Revise.actions(rlogger; line=true)) == 9 + @test_broken length(Revise.diffs(rlogger)) == 2 + io = PipeBuffer() + foreach(rec -> show(io, rec), rlogger.logs) + foreach(rec -> show(io, rec; verbose=false), rlogger.logs) + @test count("Revise.LogRecord", read(io, String)) > 8 + empty!(rlogger.logs) + + # Backtraces. Note this doesn't test the line-number correction + # because both of these are revised definitions. + cp(fl3, tmpfile; force=true) + mexsold = mexsnew + mexsnew = Revise.parse_source(tmpfile, Main) + mexsnew = Revise.eval_revised(mexsnew, mexsold) + try + ReviseTest.cube(2) + @test false + catch err + @test isa(err, ErrorException) && err.msg == "cube" + bt = throwing_function(stacktrace(catch_backtrace())) + @test bt.func === :cube && bt.file == Symbol(tmpfile) && bt.line == 7 + end + try + ReviseTest.Internal.mult2(2) + @test false + catch err + @test isa(err, ErrorException) && err.msg == "mult2" + bt = throwing_function(stacktrace(catch_backtrace())) + @test bt.func === :mult2 && bt.file == Symbol(tmpfile) && bt.line == 13 + end + + logs = filter(r->r.level==Debug && r.group=="Action", rlogger.logs) + @test length(logs) == 4 + cmpdiff(logs[3], "Eval"; deltainfo=(ReviseTest, :(cube(x) = error("cube")))) + cmpdiff(logs[4], "Eval"; deltainfo=(ReviseTest.Internal, :(mult2(x) = error("mult2")))) + + # Turn off future logging + Revise.debug_logger(; min_level=Info) + + # Gensymmed symbols + rex1 = Revise.RelocatableExpr(macroexpand(Main, :(t = @elapsed(foo(x))))) + rex2 = Revise.RelocatableExpr(macroexpand(Main, :(t = @elapsed(foo(x))))) + @test isequal(rex1, rex2) + @test hash(rex1) == hash(rex2) + rex3 = Revise.RelocatableExpr(macroexpand(Main, :(t = @elapsed(bar(x))))) + @test !isequal(rex1, rex3) + @test hash(rex1) != hash(rex3) + sym1, sym2 = gensym(:hello), gensym(:hello) + rex1 = Revise.RelocatableExpr(:(x = $sym1)) + rex2 = Revise.RelocatableExpr(:(x = $sym2)) + @test isequal(rex1, rex2) + @test hash(rex1) == hash(rex2) + sym3 = gensym(:world) + rex3 = Revise.RelocatableExpr(:(x = $sym3)) + @test isequal(rex1, rex3) + @test hash(rex1) == hash(rex3) + + # coverage + rex = convert(Revise.RelocatableExpr, :(a = 1)) + @test Revise.striplines!(rex) isa Revise.RelocatableExpr + @test copy(rex) !== rex + end + + do_test("Display") && @testset "Display" begin + io = IOBuffer() + show(io, Revise.RelocatableExpr(:(@inbounds x[2]))) + str = String(take!(io)) + @test str == ":(@inbounds x[2])" + mod = private_module() + file = joinpath(@__DIR__, "revisetest.jl") + Base.include(mod, file) + mexs = Revise.parse_source(file, mod) + Revise.instantiate_sigs!(mexs) + # io = IOBuffer() + print(IOContext(io, :compact=>true), mexs) + str = String(take!(io)) + @test str == "OrderedCollections.OrderedDict($mod$(pair_op_compact)ExprsSigs(<1 expressions>, <0 signatures>), $mod.ReviseTest$(pair_op_compact)ExprsSigs(<2 expressions>, <2 signatures>), $mod.ReviseTest.Internal$(pair_op_compact)ExprsSigs(<6 expressions>, <5 signatures>))" + exs = mexs[getfield(mod, :ReviseTest)] + # io = IOBuffer() + print(IOContext(io, :compact=>true), exs) + @test String(take!(io)) == "ExprsSigs(<2 expressions>, <2 signatures>)" + print(IOContext(io, :compact=>false), exs) + str = String(take!(io)) + @test str == "ExprsSigs with the following expressions: \n :(square(x) = begin\n x ^ 2\n end)\n :(cube(x) = begin\n x ^ 4\n end)" + + sleep(0.1) # wait for EponymTuples to hit the cache + pkgdata = Revise.pkgdatas[Base.PkgId(EponymTuples)] + file = first(Revise.srcfiles(pkgdata)) + Revise.maybe_parse_from_cache!(pkgdata, file) + print(io, pkgdata) + str = String(take!(io)) + @test occursin("EponymTuples.jl\": FileInfo", str) + @test occursin(r"with cachefile.*EponymTuples.*ji", str) + print(IOContext(io, :compact=>true), pkgdata) + str = String(take!(io)) + @test occursin("1/1 parsed files", str) + end + + do_test("File paths") && @testset "File paths" begin + testdir = newtestdir() + for wf in (Revise.watching_files[] ? (true,) : (true, false)) + for (pcflag, fbase) in ((true, "pc"), (false, "npc"),) # precompiled & not + modname = uppercase(fbase) * (wf ? "WF" : "WD") + fbase = fbase * (wf ? "wf" : "wd") + pcexpr = pcflag ? "" : :(__precompile__(false)) + # Create a package with the following structure: + # src/PkgName.jl # PC.jl = precompiled, NPC.jl = nonprecompiled + # src/file2.jl + # src/subdir/file3.jl + # src/subdir/file4.jl + # exploring different ways of expressing the `include` statement + dn = joinpath(testdir, modname, "src") + mkpath(dn) + write(joinpath(dn, modname*".jl"), """ + $pcexpr + module $modname + + export $(fbase)1, $(fbase)2, $(fbase)3, $(fbase)4, $(fbase)5, using_macro_$(fbase) + + $(fbase)1() = 1 + + include("file2.jl") + include("subdir/file3.jl") + include(joinpath(@__DIR__, "subdir", "file4.jl")) + otherfile = "file5.jl" + include(otherfile) + + # Update order check: modifying `some_macro_` to return -6 doesn't change the + # return value of `using_macro_` (issue #20) unless `using_macro_` is also updated, + # *in this order*: + # 1. update the `@some_macro_` definition + # 2. update the `using_macro_` definition + macro some_macro_$(fbase)() + return 6 + end + using_macro_$(fbase)() = @some_macro_$(fbase)() + + end + """) + write(joinpath(dn, "file2.jl"), "$(fbase)2() = 2") + mkdir(joinpath(dn, "subdir")) + write(joinpath(dn, "subdir", "file3.jl"), "$(fbase)3() = 3") + write(joinpath(dn, "subdir", "file4.jl"), "$(fbase)4() = 4") + write(joinpath(dn, "file5.jl"), "$(fbase)5() = 5") + + sleep(mtimedelay) + @eval using $(Symbol(modname)) + sleep(mtimedelay) + fn1, fn2 = Symbol("$(fbase)1"), Symbol("$(fbase)2") + fn3, fn4 = Symbol("$(fbase)3"), Symbol("$(fbase)4") + fn5 = Symbol("$(fbase)5") + fn6 = Symbol("using_macro_$(fbase)") + @eval @test $(fn1)() == 1 + @eval @test $(fn2)() == 2 + @eval @test $(fn3)() == 3 + @eval @test $(fn4)() == 4 + @eval @test $(fn5)() == 5 + @eval @test $(fn6)() == 6 + m = @eval first(methods($fn1)) + rex = Revise.RelocatableExpr(definition(m)) + @test rex == Revise.RelocatableExpr(:( $fn1() = 1 )) + # Check that definition returns copies + rex2 = deepcopy(rex) + rex.ex.args[end].args[end] = 2 + @test Revise.RelocatableExpr(definition(m)) == rex2 + @test Revise.RelocatableExpr(definition(m)) != rex + # CodeTracking methods + m3 = first(methods(eval(fn3))) + m3file = joinpath(dn, "subdir", "file3.jl") + @test whereis(m3) == (m3file, 1) + @test signatures_at(m3file, 1) == [m3.sig] + @test signatures_at(eval(Symbol(modname)), joinpath("src", "subdir", "file3.jl"), 1) == [m3.sig] + + id = Base.PkgId(eval(Symbol(modname))) # for testing #596 + pkgdata = Revise.pkgdatas[id] + + # Change the definition of function 1 (easiest to just rewrite the whole file) + write(joinpath(dn, modname*".jl"), """ + $pcexpr + module $modname + export $(fbase)1, $(fbase)2, $(fbase)3, $(fbase)4, $(fbase)5, using_macro_$(fbase) + $(fbase)1() = -1 + include("file2.jl") + include("subdir/file3.jl") + include(joinpath(@__DIR__, "subdir", "file4.jl")) + otherfile = "file5.jl" + include(otherfile) + + macro some_macro_$(fbase)() + return -6 + end + using_macro_$(fbase)() = @some_macro_$(fbase)() + + end + """) # just for fun we skipped the whitespace + yry() + fi = pkgdata.fileinfos[1] + @test fi.extracted[] # issue 596 + @eval @test $(fn1)() == -1 + @eval @test $(fn2)() == 2 + @eval @test $(fn3)() == 3 + @eval @test $(fn4)() == 4 + @eval @test $(fn5)() == 5 + @eval @test $(fn6)() == 6 # because it hasn't been re-macroexpanded + @test revise(eval(Symbol(modname))) + @eval @test $(fn6)() == -6 + # Redefine function 2 + write(joinpath(dn, "file2.jl"), "$(fbase)2() = -2") + yry() + @eval @test $(fn1)() == -1 + @eval @test $(fn2)() == -2 + @eval @test $(fn3)() == 3 + @eval @test $(fn4)() == 4 + @eval @test $(fn5)() == 5 + @eval @test $(fn6)() == -6 + write(joinpath(dn, "subdir", "file3.jl"), "$(fbase)3() = -3") + yry() + @eval @test $(fn1)() == -1 + @eval @test $(fn2)() == -2 + @eval @test $(fn3)() == -3 + @eval @test $(fn4)() == 4 + @eval @test $(fn5)() == 5 + @eval @test $(fn6)() == -6 + write(joinpath(dn, "subdir", "file4.jl"), "$(fbase)4() = -4") + yry() + @eval @test $(fn1)() == -1 + @eval @test $(fn2)() == -2 + @eval @test $(fn3)() == -3 + @eval @test $(fn4)() == -4 + @eval @test $(fn5)() == 5 + @eval @test $(fn6)() == -6 + write(joinpath(dn, "file5.jl"), "$(fbase)5() = -5") + yry() + @eval @test $(fn1)() == -1 + @eval @test $(fn2)() == -2 + @eval @test $(fn3)() == -3 + @eval @test $(fn4)() == -4 + @eval @test $(fn5)() == -5 + @eval @test $(fn6)() == -6 + # Check that the list of files is complete + pkgdata = Revise.pkgdatas[Base.PkgId(modname)] + for file = [joinpath("src", modname*".jl"), joinpath("src", "file2.jl"), + joinpath("src", "subdir", "file3.jl"), + joinpath("src", "subdir", "file4.jl"), + joinpath("src", "file5.jl")] + @test Revise.hasfile(pkgdata, file) + end + end + end + # Remove the precompiled file(s) + rm_precompile("PCWF") + Revise.watching_files[] || rm_precompile("PCWD") + + # Submodules (issue #142) + srcdir = joinpath(testdir, "Mysupermodule", "src") + subdir = joinpath(srcdir, "Mymodule") + mkpath(subdir) + write(joinpath(srcdir, "Mysupermodule.jl"), """ + module Mysupermodule + include("Mymodule/Mymodule.jl") + end + """) + write(joinpath(subdir, "Mymodule.jl"), """ + module Mymodule + include("filesub.jl") + end + """) + write(joinpath(subdir, "filesub.jl"), "func() = 1") + sleep(mtimedelay) + @eval using Mysupermodule + sleep(mtimedelay) + @test Mysupermodule.Mymodule.func() == 1 + write(joinpath(subdir, "filesub.jl"), "func() = 2") + yry() + @test Mysupermodule.Mymodule.func() == 2 + rm_precompile("Mymodule") + rm_precompile("Mysupermodule") + + # Test files paths that can't be statically parsed + dn = joinpath(testdir, "LoopInclude", "src") + mkpath(dn) + write(joinpath(dn, "LoopInclude.jl"), """ + module LoopInclude + + export li_f, li_g + + for fn in ("file1.jl", "file2.jl") + include(fn) + end + + end + """) + write(joinpath(dn, "file1.jl"), "li_f() = 1") + write(joinpath(dn, "file2.jl"), "li_g() = 2") + sleep(mtimedelay) + @eval using LoopInclude + sleep(mtimedelay) + @test li_f() == 1 + @test li_g() == 2 + write(joinpath(dn, "file1.jl"), "li_f() = -1") + yry() + @test li_f() == -1 + rm_precompile("LoopInclude") + + # Multiple packages in the same directory (issue #228) + write(joinpath(testdir, "A228.jl"), """ + module A228 + using B228 + export f228 + f228(x) = 3 * g228(x) + end + """) + write(joinpath(testdir, "B228.jl"), """ + module B228 + export g228 + g228(x) = 4x + 2 + end + """) + sleep(mtimedelay) + using A228 + sleep(mtimedelay) + @test f228(3) == 42 + write(joinpath(testdir, "B228.jl"), """ + module B228 + export g228 + g228(x) = 4x + 1 + end + """) + yry() + @test f228(3) == 39 + rm_precompile("A228") + rm_precompile("B228") + + # uncoupled packages in the same directory (issue #339) + write(joinpath(testdir, "A339.jl"), """ + module A339 + f() = 1 + end + """) + write(joinpath(testdir, "B339.jl"), """ + module B339 + f() = 1 + end + """) + sleep(mtimedelay) + using A339, B339 + sleep(mtimedelay) + @test A339.f() == 1 + @test B339.f() == 1 + sleep(mtimedelay) + write(joinpath(testdir, "A339.jl"), """ + module A339 + f() = 2 + end + """) + yry() + @test A339.f() == 2 + @test B339.f() == 1 + sleep(mtimedelay) + write(joinpath(testdir, "B339.jl"), """ + module B339 + f() = 2 + end + """) + yry() + @test A339.f() == 2 + @test B339.f() == 2 + rm_precompile("A339") + rm_precompile("B339") + + pop!(LOAD_PATH) + end + + # issue #131 + do_test("Base & stdlib file paths") && @testset "Base & stdlib file paths" begin + @test isfile(Revise.basesrccache) + targetfn = Base.Filesystem.path_separator * joinpath("good", "path", "mydir", "myfile.jl") + @test Revise.fixpath("/some/bad/path/mydir/myfile.jl"; badpath="/some/bad/path", goodpath="/good/path") == targetfn + @test Revise.fixpath("/some/bad/path/mydir/myfile.jl"; badpath="/some/bad/path/", goodpath="/good/path") == targetfn + @test isfile(Revise.fixpath(Base.find_source_file("array.jl"))) + failedfiles = Tuple{String,String}[] + for (mod,file) = Base._included_files + fixedfile = Revise.fixpath(file) + if !isfile(fixedfile) + push!(failedfiles, (file, fixedfile)) + end + end + if !isempty(failedfiles) + display(failedfiles) + end + @test isempty(failedfiles) + end + + do_test("Namespace") && @testset "Namespace" begin + # Issues #579, #239, and #627 + testdir = newtestdir() + dn = joinpath(testdir, "Namespace", "src") + mkpath(dn) + write(joinpath(dn, "Namespace.jl"), """ + module Namespace + struct X end + cos(::X) = 20 + end + """) + sleep(mtimedelay) + @eval using Namespace + @test Namespace.cos(Namespace.X()) == 20 + @test_throws MethodError Base.cos(Namespace.X()) + sleep(mtimedelay) + write(joinpath(dn, "Namespace.jl"), """ + module Namespace + struct X end + sin(::Int) = 10 + Base.cos(::X) = 20 + # From #627 + module Foos + struct Foo end + end + using .Foos: Foo + end + """) + yry() + @test Namespace.sin(0) == 10 + @test Base.sin(0) == 0 + @test Base.cos(Namespace.X()) == 20 + @test_throws MethodError Namespace.cos(Namespace.X()) + + rm_precompile("Namespace") + pop!(LOAD_PATH) + end + + do_test("Multiple definitions") && @testset "Multiple definitions" begin + # This simulates a copy/paste/save "error" from one file to another + # ref https://github.com/timholy/CodeTracking.jl/issues/55 + testdir = newtestdir() + dn = joinpath(testdir, "Multidef", "src") + mkpath(dn) + write(joinpath(dn, "Multidef.jl"), """ + module Multidef + include("utils.jl") + end + """) + write(joinpath(dn, "utils.jl"), "repeated(x) = x+1") + sleep(mtimedelay) + @eval using Multidef + @test Multidef.repeated(3) == 4 + sleep(mtimedelay) + write(joinpath(dn, "Multidef.jl"), """ + module Multidef + include("utils.jl") + repeated(x) = x+1 + end + """) + yry() + @test Multidef.repeated(3) == 4 + sleep(mtimedelay) + write(joinpath(dn, "utils.jl"), "\n") + yry() + @test Multidef.repeated(3) == 4 + + rm_precompile("Multidef") + pop!(LOAD_PATH) + end + + do_test("Recursive types (issue #417)") && @testset "Recursive types (issue #417)" begin + testdir = newtestdir() + fn = joinpath(testdir, "recursive.jl") + write(fn, """ + module RecursiveTypes + struct Foo + x::Vector{Foo} + + Foo() = new(Foo[]) + end + end + """) + sleep(mtimedelay) + includet(fn) + @test isa(RecursiveTypes.Foo().x, Vector{RecursiveTypes.Foo}) + + pop!(LOAD_PATH) + end + + # issue #318 + do_test("Cross-module extension") && @testset "Cross-module extension" begin + testdir = newtestdir() + dnA = joinpath(testdir, "CrossModA", "src") + mkpath(dnA) + write(joinpath(dnA, "CrossModA.jl"), """ + module CrossModA + foo(x) = "default" + end + """) + dnB = joinpath(testdir, "CrossModB", "src") + mkpath(dnB) + write(joinpath(dnB, "CrossModB.jl"), """ + module CrossModB + import CrossModA + CrossModA.foo(x::Int) = 1 + end + """) + sleep(mtimedelay) + @eval using CrossModA, CrossModB + @test CrossModA.foo("") == "default" + @test CrossModA.foo(0) == 1 + sleep(mtimedelay) + write(joinpath(dnB, "CrossModB.jl"), """ + module CrossModB + import CrossModA + CrossModA.foo(x::Int) = 2 + end + """) + yry() + @test CrossModA.foo("") == "default" + @test CrossModA.foo(0) == 2 + write(joinpath(dnB, "CrossModB.jl"), """ + module CrossModB + import CrossModA + CrossModA.foo(x::Int) = 3 + end + """) + yry() + @test CrossModA.foo("") == "default" + @test CrossModA.foo(0) == 3 + + rm_precompile("CrossModA") + rm_precompile("CrossModB") + pop!(LOAD_PATH) + end + + # issue #36 + do_test("@__FILE__") && @testset "@__FILE__" begin + testdir = newtestdir() + dn = joinpath(testdir, "ModFILE", "src") + mkpath(dn) + write(joinpath(dn, "ModFILE.jl"), """ + module ModFILE + + mf() = @__FILE__, 1 + + end + """) + sleep(mtimedelay) + @eval using ModFILE + sleep(mtimedelay) + @test ModFILE.mf() == (joinpath(dn, "ModFILE.jl"), 1) + write(joinpath(dn, "ModFILE.jl"), """ + module ModFILE + + mf() = @__FILE__, 2 + + end + """) + yry() + @test ModFILE.mf() == (joinpath(dn, "ModFILE.jl"), 2) + rm_precompile("ModFILE") + pop!(LOAD_PATH) + end + + do_test("Revision order") && @testset "Revision order" begin + testdir = newtestdir() + dn = joinpath(testdir, "Order1", "src") + mkpath(dn) + write(joinpath(dn, "Order1.jl"), """ + module Order1 + include("file1.jl") + include("file2.jl") + end + """) + write(joinpath(dn, "file1.jl"), "# a comment") + write(joinpath(dn, "file2.jl"), "# a comment") + sleep(mtimedelay) + @eval using Order1 + sleep(mtimedelay) + # we want Revise to process files the order file1.jl, file2.jl, but let's save them in the opposite order + write(joinpath(dn, "file2.jl"), "f(::Ord1) = 1") + sleep(mtimedelay) + write(joinpath(dn, "file1.jl"), "struct Ord1 end") + yry() + @test Order1.f(Order1.Ord1()) == 1 + + # A case in which order cannot be determined solely from file order + dn = joinpath(testdir, "Order2", "src") + mkpath(dn) + write(joinpath(dn, "Order2.jl"), """ + module Order2 + include("file.jl") + end + """) + write(joinpath(dn, "file.jl"), "# a comment") + sleep(mtimedelay) + @eval using Order2 + sleep(mtimedelay) + write(joinpath(dn, "Order2.jl"), """ + module Order2 + include("file.jl") + f(::Ord2) = 1 + end + """) + sleep(mtimedelay) + write(joinpath(dn, "file.jl"), "struct Ord2 end") + @info "The following error message is expected for this broken test" + yry() + @test_broken Order2.f(Order2.Ord2()) == 1 + # Resolve it with retry + Revise.retry() + @test Order2.f(Order2.Ord2()) == 1 + + # Cross-module dependencies + dn3 = joinpath(testdir, "Order3", "src") + mkpath(dn3) + write(joinpath(dn3, "Order3.jl"), """ + module Order3 + using Order2 + end + """) + sleep(mtimedelay) + @eval using Order3 + sleep(mtimedelay) + write(joinpath(dn3, "Order3.jl"), """ + module Order3 + using Order2 + g(::Order2.Ord2a) = 1 + end + """) + sleep(mtimedelay) + write(joinpath(dn, "file.jl"), """ + struct Ord2 end + struct Ord2a end + """) + yry() + @test Order3.g(Order2.Ord2a()) == 1 + + rm_precompile("Order1") + rm_precompile("Order2") + pop!(LOAD_PATH) + end + + # issue #8 and #197 + do_test("Module docstring") && @testset "Module docstring" begin + testdir = newtestdir() + dn = joinpath(testdir, "ModDocstring", "src") + mkpath(dn) + write(joinpath(dn, "ModDocstring.jl"), """ + " Ahoy! " + module ModDocstring + + include("dependency.jl") + + f() = 1 + + end + """) + write(joinpath(dn, "dependency.jl"), "") + sleep(mtimedelay) + @eval using ModDocstring + sleep(mtimedelay) + @test ModDocstring.f() == 1 + ds = @doc(ModDocstring) + @test get_docstring(ds) == "Ahoy! " + + write(joinpath(dn, "ModDocstring.jl"), """ + " Ahoy! " + module ModDocstring + + include("dependency.jl") + + f() = 2 + + end + """) + yry() + @test ModDocstring.f() == 2 + ds = @doc(ModDocstring) + @test get_docstring(ds) == "Ahoy! " + + write(joinpath(dn, "ModDocstring.jl"), """ + " Hello! " + module ModDocstring + + include("dependency.jl") + + f() = 3 + + end + """) + yry() + @test ModDocstring.f() == 3 + ds = @doc(ModDocstring) + @test get_docstring(ds) == "Hello! " + rm_precompile("ModDocstring") + + # issue #197 + dn = joinpath(testdir, "ModDocstring2", "src") + mkpath(dn) + write(joinpath(dn, "ModDocstring2.jl"), """ + "docstring" + module ModDocstring2 + "docstring for .Sub" + module Sub + end + end + """) + sleep(mtimedelay) + @eval using ModDocstring2 + sleep(mtimedelay) + ds = @doc(ModDocstring2) + @test get_docstring(ds) == "docstring" + ds = @doc(ModDocstring2.Sub) + @test get_docstring(ds) == "docstring for .Sub" + write(joinpath(dn, "ModDocstring2.jl"), """ + "updated docstring" + module ModDocstring2 + "updated docstring for .Sub" + module Sub + end + end + """) + yry() + ds = @doc(ModDocstring2) + @test get_docstring(ds) == "updated docstring" + ds = @doc(ModDocstring2.Sub) + @test get_docstring(ds) == "updated docstring for .Sub" + rm_precompile("ModDocstring2") + + pop!(LOAD_PATH) + end + + do_test("Changing docstrings") && @testset "Changing docstring" begin + # Compiled mode covers most docstring changes, so we have to go to + # special effort to test the older interpreter-based solution. + testdir = newtestdir() + dn = joinpath(testdir, "ChangeDocstring", "src") + mkpath(dn) + write(joinpath(dn, "ChangeDocstring.jl"), """ + module ChangeDocstring + "f" f() = 1 + g() = 1 + end + """) + sleep(mtimedelay) + @eval using ChangeDocstring + sleep(mtimedelay) + @test ChangeDocstring.f() == 1 + ds = @doc(ChangeDocstring.f) + @test get_docstring(ds) == "f" + @test ChangeDocstring.g() == 1 + ds = @doc(ChangeDocstring.g) + @test get_docstring(ds) == "No documentation found." + # Ordinary route + write(joinpath(dn, "ChangeDocstring.jl"), """ + module ChangeDocstring + "h" f() = 1 + "g" g() = 1 + end + """) + yry() + ds = @doc(ChangeDocstring.f) + @test get_docstring(ds) == "h" + ds = @doc(ChangeDocstring.g) + @test get_docstring(ds) == "g" + + # Now manually change the docstring + ex = quote "g" f() = 1 end + lwr = Meta.lower(ChangeDocstring, ex) + frame = Frame(ChangeDocstring, lwr.args[1]) + methodinfo = Revise.MethodInfo() + docexprs = Revise.DocExprs() + ret = Revise.methods_by_execution!(JuliaInterpreter.finish_and_return!, methodinfo, + docexprs, frame, trues(length(frame.framecode.src.code)); mode=:sigs) + ds = @doc(ChangeDocstring.f) + @test get_docstring(ds) == "g" + + rm_precompile("ChangeDocstring") + + # Test for #583 + dn = joinpath(testdir, "FirstDocstring", "src") + mkpath(dn) + write(joinpath(dn, "FirstDocstring.jl"), """ + module FirstDocstring + g() = 1 + end + """) + sleep(mtimedelay) + @eval using FirstDocstring + sleep(mtimedelay) + @test FirstDocstring.g() == 1 + ds = @doc(FirstDocstring.g) + @test get_docstring(ds) == "No documentation found." + write(joinpath(dn, "FirstDocstring.jl"), """ + module FirstDocstring + "g" g() = 1 + end + """) + yry() + ds = @doc(FirstDocstring.g) + @test get_docstring(ds) == "g" + + rm_precompile("FirstDocstring") + pop!(LOAD_PATH) + end + + do_test("Undef in docstrings") && @testset "Undef in docstrings" begin + fn = Base.find_source_file("abstractset.jl") # has lots of examples of """str""" func1, func2 + mexsold = Revise.parse_source(fn, Base) + mexsnew = Revise.parse_source(fn, Base) + odict = mexsold[Base] + ndict = mexsnew[Base] + for (k, v) in odict + @test haskey(ndict, k) + end + end + + do_test("Macro docstrings (issue #309)") && @testset "Macro docstrings (issue #309)" begin + testdir = newtestdir() + dn = joinpath(testdir, "MacDocstring", "src") + mkpath(dn) + write(joinpath(dn, "MacDocstring.jl"), """ + module MacDocstring + + macro myconst(name, val) + quote + \"\"\" + mydoc + \"\"\" + const \$(esc(name)) = \$val + end + end + + @myconst c 1.2 + f() = 1 + + end # module + """) + sleep(mtimedelay) + @eval using MacDocstring + sleep(mtimedelay) + @test MacDocstring.f() == 1 + ds = @doc(MacDocstring.c) + @test strip(get_docstring(ds)) == "mydoc" + + write(joinpath(dn, "MacDocstring.jl"), """ + module MacDocstring + + macro myconst(name, val) + quote + \"\"\" + mydoc + \"\"\" + const \$(esc(name)) = \$val + end + end + + @myconst c 1.2 + f() = 2 + + end # module + """) + yry() + @test MacDocstring.f() == 2 + ds = @doc(MacDocstring.c) + @test strip(get_docstring(ds)) == "mydoc" + + rm_precompile("MacDocstring") + pop!(LOAD_PATH) + end + + # issue #165 + do_test("Changing @inline annotations") && @testset "Changing @inline annotations" begin + testdir = newtestdir() + dn = joinpath(testdir, "PerfAnnotations", "src") + mkpath(dn) + write(joinpath(dn, "PerfAnnotations.jl"), """ + module PerfAnnotations + + @inline hasinline(x) = x + check_hasinline(x) = hasinline(x) + + @noinline hasnoinline(x) = x + check_hasnoinline(x) = hasnoinline(x) + + notannot1(x) = x + check_notannot1(x) = notannot1(x) + + notannot2(x) = x + check_notannot2(x) = notannot2(x) + + end + """) + sleep(mtimedelay) + @eval using PerfAnnotations + sleep(mtimedelay) + @test PerfAnnotations.check_hasinline(3) == 3 + @test PerfAnnotations.check_hasnoinline(3) == 3 + @test PerfAnnotations.check_notannot1(3) == 3 + @test PerfAnnotations.check_notannot2(3) == 3 + code = get_code(PerfAnnotations.check_hasinline, Tuple{Int}) + @test length(code) == 1 && isreturning_slot(code[1], 2) + code = get_code(PerfAnnotations.check_hasnoinline, Tuple{Int}) + @test length(code) == 2 && code[1].head === :invoke + code = get_code(PerfAnnotations.check_notannot1, Tuple{Int}) + @test length(code) == 1 && isreturning_slot(code[1], 2) + code = get_code(PerfAnnotations.check_notannot2, Tuple{Int}) + @test length(code) == 1 && isreturning_slot(code[1], 2) + write(joinpath(dn, "PerfAnnotations.jl"), """ + module PerfAnnotations + + hasinline(x) = x + check_hasinline(x) = hasinline(x) + + hasnoinline(x) = x + check_hasnoinline(x) = hasnoinline(x) + + @inline notannot1(x) = x + check_notannot1(x) = notannot1(x) + + @noinline notannot2(x) = x + check_notannot2(x) = notannot2(x) + + end + """) + yry() + @test PerfAnnotations.check_hasinline(3) == 3 + @test PerfAnnotations.check_hasnoinline(3) == 3 + @test PerfAnnotations.check_notannot1(3) == 3 + @test PerfAnnotations.check_notannot2(3) == 3 + code = get_code(PerfAnnotations.check_hasinline, Tuple{Int}) + @test length(code) == 1 && isreturning_slot(code[1], 2) + code = get_code(PerfAnnotations.check_hasnoinline, Tuple{Int}) + @test length(code) == 1 && isreturning_slot(code[1], 2) + code = get_code(PerfAnnotations.check_notannot1, Tuple{Int}) + @test length(code) == 1 && isreturning_slot(code[1], 2) + code = get_code(PerfAnnotations.check_notannot2, Tuple{Int}) + @test length(code) == 2 && code[1].head === :invoke + rm_precompile("PerfAnnotations") + + pop!(LOAD_PATH) + end + + do_test("Revising macros") && @testset "Revising macros" begin + # issue #174 + testdir = newtestdir() + dn = joinpath(testdir, "MacroRevision", "src") + mkpath(dn) + write(joinpath(dn, "MacroRevision.jl"), """ + module MacroRevision + macro change(foodef) + foodef.args[2].args[2] = 1 + esc(foodef) + end + @change foo(x) = 0 + end + """) + sleep(mtimedelay) + @eval using MacroRevision + sleep(mtimedelay) + @test MacroRevision.foo("hello") == 1 + + write(joinpath(dn, "MacroRevision.jl"), """ + module MacroRevision + macro change(foodef) + foodef.args[2].args[2] = 2 + esc(foodef) + end + @change foo(x) = 0 + end + """) + yry() + @test MacroRevision.foo("hello") == 1 + revise(MacroRevision) + @test MacroRevision.foo("hello") == 2 + + write(joinpath(dn, "MacroRevision.jl"), """ + module MacroRevision + macro change(foodef) + foodef.args[2].args[2] = 3 + esc(foodef) + end + @change foo(x) = 0 + end + """) + yry() + @test MacroRevision.foo("hello") == 2 + revise(MacroRevision) + @test MacroRevision.foo("hello") == 3 + rm_precompile("MacroRevision") + + # issue #435 + dn = joinpath(testdir, "MacroSigs", "src") + mkpath(dn) + write(joinpath(dn, "MacroSigs.jl"), """ + module MacroSigs + end + """) + sleep(mtimedelay) + @eval using MacroSigs + sleep(mtimedelay) + write(joinpath(dn, "MacroSigs.jl"), """ + module MacroSigs + macro testmac(fname) + esc(quote + function some_fun end + \$fname() = 1 + end) + end + + @testmac blah + end + """) + yry() + @test MacroSigs.blah() == 1 + @test haskey(CodeTracking.method_info, (@which MacroSigs.blah()).sig) + rm_precompile("MacroSigs") + + # Issue #568 (a macro *execution* bug) + dn = joinpath(testdir, "MacroLineNos568", "src") + mkpath(dn) + write(joinpath(dn, "MacroLineNos568.jl"), """ + module MacroLineNos568 + using MacroTools: @q + + function my_fun end + + macro some_macro(value) + return esc(@q \$MacroLineNos568.my_fun() = \$value) + end + + @some_macro 20 + end + """) + sleep(mtimedelay) + @eval using MacroLineNos568 + sleep(mtimedelay) + @test MacroLineNos568.my_fun() == 20 + write(joinpath(dn, "MacroLineNos568.jl"), """ + module MacroLineNos568 + using MacroTools: @q + + function my_fun end + + macro some_macro(value) + return esc(@q \$MacroLineNos568.my_fun() = \$value) + end + + @some_macro 30 + end + """) + yry() + @test MacroLineNos568.my_fun() == 30 + rm_precompile("MacroLineNos568") + + pop!(LOAD_PATH) + end + + do_test("More arg-modifying macros") && @testset "More arg-modifying macros" begin + # issue #183 + testdir = newtestdir() + dn = joinpath(testdir, "ArgModMacros", "src") + mkpath(dn) + write(joinpath(dn, "ArgModMacros.jl"), """ + module ArgModMacros + + using EponymTuples + + const revision = Ref(0) + + function hyper_loglikelihood(@eponymargs(μ, σ, LΩ), @eponymargs(w̃s, α̃s, β̃s)) + revision[] = 1 + loglikelihood_normal(@eponymtuple(μ, σ, LΩ), vcat(w̃s, α̃s, β̃s)) + end + + loglikelihood_normal(@eponymargs(μ, σ, LΩ), stuff) = stuff + + end + """) + sleep(mtimedelay) + @eval using ArgModMacros + sleep(mtimedelay) + @test ArgModMacros.hyper_loglikelihood((μ=1, σ=2, LΩ=3), (w̃s=4, α̃s=5, β̃s=6)) == [4,5,6] + @test ArgModMacros.revision[] == 1 + write(joinpath(dn, "ArgModMacros.jl"), """ + module ArgModMacros + + using EponymTuples + + const revision = Ref(0) + + function hyper_loglikelihood(@eponymargs(μ, σ, LΩ), @eponymargs(w̃s, α̃s, β̃s)) + revision[] = 2 + loglikelihood_normal(@eponymtuple(μ, σ, LΩ), vcat(w̃s, α̃s, β̃s)) + end + + loglikelihood_normal(@eponymargs(μ, σ, LΩ), stuff) = stuff + + end + """) + yry() + @test ArgModMacros.hyper_loglikelihood((μ=1, σ=2, LΩ=3), (w̃s=4, α̃s=5, β̃s=6)) == [4,5,6] + @test ArgModMacros.revision[] == 2 + rm_precompile("ArgModMacros") + pop!(LOAD_PATH) + end + + do_test("Line numbers") && @testset "Line numbers" begin + # issue #27 + testdir = newtestdir() + modname = "LineNumberMod" + dn = joinpath(testdir, modname, "src") + mkpath(dn) + write(joinpath(dn, modname*".jl"), "module $modname include(\"incl.jl\") end") + write(joinpath(dn, "incl.jl"), """ + 0 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + + foo(x) = x+5 + + foo(y::Int) = y-51 + """) + sleep(mtimedelay) + @eval using LineNumberMod + sleep(mtimedelay) + lines = Int[] + files = String[] + for m in methods(LineNumberMod.foo) + push!(files, String(m.file)) + push!(lines, m.line) + end + @test all(f->endswith(string(f), "incl.jl"), files) + write(joinpath(dn, "incl.jl"), """ + 0 + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + + foo(x) = x+6 + + foo(y::Int) = y-51 + """) + yry() + for m in methods(LineNumberMod.foo) + @test endswith(string(m.file), "incl.jl") + @test m.line ∈ lines + end + rm_precompile("LineNumberMod") + pop!(LOAD_PATH) + end + + do_test("Line numbers in backtraces and warnings") && @testset "Line numbers in backtraces and warnings" begin + filename = randtmp() * ".jl" + write(filename, """ + function triggered(iserr::Bool, iswarn::Bool) + iserr && error("error") + iswarn && @warn "Information" + return nothing + end + """) + sleep(mtimedelay) + includet(filename) + sleep(mtimedelay) + try + triggered(true, false) + @test false + catch err + st = stacktrace(catch_backtrace()) + Revise.update_stacktrace_lineno!(st) + bt = throwing_function(st) + @test bt.file == Symbol(filename) && bt.line == 2 + end + io = IOBuffer() + if isdefined(Base, :methodloc_callback) + print(io, methods(triggered)) + mline = line_is_decl ? 1 : 2 + @test occursin(filename * ":$mline", String(take!(io))) + end + write(filename, """ + # A comment to change the line numbers + function triggered(iserr::Bool, iswarn::Bool) + iserr && error("error") + iswarn && @warn "Information" + return nothing + end + """) + yry() + try + triggered(true, false) + @test false + catch err + bt = throwing_function(Revise.update_stacktrace_lineno!(stacktrace(catch_backtrace()))) + @test bt.file == Symbol(filename) && bt.line == 3 + end + st = try + triggered(true, false) + @test false + catch err + stacktrace(catch_backtrace()) + end + targetstr = basename(filename * ":3") + Base.show_backtrace(io, st) + @test occursin(targetstr, String(take!(io))) + # Long stacktraces take a different path, test this too + while length(st) < 100 + st = vcat(st, st) + end + Base.show_backtrace(io, st) + @test occursin(targetstr, String(take!(io))) + if isdefined(Base, :methodloc_callback) + print(io, methods(triggered)) + mline = line_is_decl ? 2 : 3 + @test occursin(basename(filename * ":$mline"), String(take!(io))) + end + + push!(to_remove, filename) + end + + # Issue #43 + do_test("New submodules") && @testset "New submodules" begin + testdir = newtestdir() + dn = joinpath(testdir, "Submodules", "src") + mkpath(dn) + write(joinpath(dn, "Submodules.jl"), """ + module Submodules + f() = 1 + end + """) + sleep(mtimedelay) + @eval using Submodules + sleep(mtimedelay) + @test Submodules.f() == 1 + write(joinpath(dn, "Submodules.jl"), """ + module Submodules + f() = 1 + module Sub + g() = 2 + end + end + """) + yry() + @test Submodules.f() == 1 + @test Submodules.Sub.g() == 2 + rm_precompile("Submodules") + pop!(LOAD_PATH) + end + + do_test("Timing (issue #341)") && @testset "Timing (issue #341)" begin + testdir = newtestdir() + dn = joinpath(testdir, "Timing", "src") + mkpath(dn) + write(joinpath(dn, "Timing.jl"), """ + module Timing + f(x) = 1 + end + """) + sleep(mtimedelay) + @eval using Timing + sleep(mtimedelay) + @test Timing.f(nothing) == 1 + tmpfile = joinpath(dn, "Timing_temp.jl") + write(tmpfile, """ + module Timing + f(x) = 2 + end + """) + yry() + @test Timing.f(nothing) == 1 + mv(tmpfile, pathof(Timing), force=true) + yry() + @test Timing.f(nothing) == 2 + + rm_precompile("Timing") + end + + do_test("Method deletion") && @testset "Method deletion" begin + Core.eval(Base, :(revisefoo(x::Float64) = 1)) # to test cross-module method scoping + testdir = newtestdir() + dn = joinpath(testdir, "MethDel", "src") + mkpath(dn) + write(joinpath(dn, "MethDel.jl"), """ + __precompile__(false) # "clean" Base doesn't have :revisefoo + module MethDel + f(x) = 1 + f(x::Int) = 2 + g(x::Vector{T}, y::T) where T = 1 + g(x::Array{T,N}, y::T) where N where T = 2 + g(::Array, ::Any) = 3 + h(x::Array{T}, y::T) where T = g(x, y) + k(::Int; badchoice=1) = badchoice + Base.revisefoo(x::Int) = 2 + struct Private end + Base.revisefoo(::Private) = 3 + + dfltargs(x::Int8, y::Int=0, z::Float32=1.0f0) = x+y+z + + hasmacro1(@nospecialize(x)) = x + hasmacro2(@nospecialize(x::Int)) = x + hasmacro3(@nospecialize(x::Int), y::Float64) = x + + hasdestructure1(x, (count, name)) = name^count + hasdestructure2(x, (count, name)::Tuple{Int,Any}) = name^count + + struct A end + struct B end + + checkunion(a::Union{Nothing, A}) = 1 + + methgensym(::Vector{<:Integer}) = 1 + + mapf(fs, x) = (fs[1](x), mapf(Base.tail(fs), x)...) + mapf(::Tuple{}, x) = () + + for T in (Int, Float64, String) + @eval mytypeof(x::\$T) = \$T + end + + @generated function firstparam(A::AbstractArray) + T = A.parameters[1] + return :(\$T) + end + + end + """) + sleep(mtimedelay) + @eval using MethDel + sleep(mtimedelay) + @test MethDel.f(1.0) == 1 + @test MethDel.f(1) == 2 + @test MethDel.g(rand(3), 1.0) == 1 + @test MethDel.g(rand(3, 3), 1.0) == 2 + @test MethDel.g(Int[], 1.0) == 3 + @test MethDel.h(rand(3), 1.0) == 1 + @test MethDel.k(1) == 1 + @test MethDel.k(1; badchoice=2) == 2 + @test MethDel.hasmacro1(1) == 1 + @test MethDel.hasmacro2(1) == 1 + @test MethDel.hasmacro3(1, 0.0) == 1 + @test MethDel.hasdestructure1(0, (3, "hi")) == "hihihi" + @test MethDel.hasdestructure2(0, (3, "hi")) == "hihihi" + @test Base.revisefoo(1.0) == 1 + @test Base.revisefoo(1) == 2 + @test Base.revisefoo(MethDel.Private()) == 3 + @test MethDel.dfltargs(Int8(2)) == 3.0f0 + @test MethDel.dfltargs(Int8(2), 5) == 8.0f0 + @test MethDel.dfltargs(Int8(2), 5, -17.0f0) == -10.0f0 + @test MethDel.checkunion(nothing) == 1 + @test MethDel.methgensym([1]) == 1 + @test_throws MethodError MethDel.methgensym([1.0]) + @test MethDel.mapf((x->x+1, x->x+0.1), 3) == (4, 3.1) + @test MethDel.mytypeof(1) === Int + @test MethDel.mytypeof(1.0) === Float64 + @test MethDel.mytypeof("hi") === String + @test MethDel.firstparam(rand(2,2)) === Float64 + write(joinpath(dn, "MethDel.jl"), """ + module MethDel + f(x) = 1 + g(x::Array{T,N}, y::T) where N where T = 2 + h(x::Array{T}, y::T) where T = g(x, y) + k(::Int; goodchoice=-1) = goodchoice + dfltargs(x::Int8, yz::Tuple{Int,Float32}=(0,1.0f0)) = x+yz[1]+yz[2] + + struct A end + struct B end + + checkunion(a::Union{Nothing, B}) = 2 + + methgensym(::Vector{<:Real}) = 1 + + mapf(fs::F, x) where F = (fs[1](x), mapf(Base.tail(fs), x)...) + mapf(::Tuple{}, x) = () + + for T in (Int, String) + @eval mytypeof(x::\$T) = \$T + end + + end + """) + yry() + @test MethDel.f(1.0) == 1 + @test MethDel.f(1) == 1 + @test MethDel.g(rand(3), 1.0) == 2 + @test MethDel.g(rand(3, 3), 1.0) == 2 + @test_throws MethodError MethDel.g(Int[], 1.0) + @test MethDel.h(rand(3), 1.0) == 2 + @test_throws MethodError MethDel.k(1; badchoice=2) + @test MethDel.k(1) == -1 + @test MethDel.k(1; goodchoice=10) == 10 + @test_throws MethodError MethDel.hasmacro1(1) + @test_throws MethodError MethDel.hasmacro2(1) + @test_throws MethodError MethDel.hasmacro3(1, 0.0) + @test_throws MethodError MethDel.hasdestructure1(0, (3, "hi")) + @test_throws MethodError MethDel.hasdestructure2(0, (3, "hi")) + @test Base.revisefoo(1.0) == 1 + @test_throws MethodError Base.revisefoo(1) + @test_throws MethodError Base.revisefoo(MethDel.Private()) + @test MethDel.dfltargs(Int8(2)) == 3.0f0 + @test MethDel.dfltargs(Int8(2), (5,-17.0f0)) == -10.0f0 + @test_throws MethodError MethDel.dfltargs(Int8(2), 5) == 8.0f0 + @test_throws MethodError MethDel.dfltargs(Int8(2), 5, -17.0f0) == -10.0f0 + @test MethDel.checkunion(nothing) == 2 + @test MethDel.methgensym([1]) == 1 + @test MethDel.methgensym([1.0]) == 1 + @test length(methods(MethDel.methgensym)) == 1 + @test MethDel.mapf((x->x+1, x->x+0.1), 3) == (4, 3.1) + @test length(methods(MethDel.mapf)) == 2 + @test MethDel.mytypeof(1) === Int + @test_throws MethodError MethDel.mytypeof(1.0) + @test MethDel.mytypeof("hi") === String + @test_throws MethodError MethDel.firstparam(rand(2,2)) + + Base.delete_method(first(methods(Base.revisefoo))) + + # Test for specificity in deletion + ex1 = :(methspecificity(x::Int) = 1) + ex2 = :(methspecificity(x::Integer) = 2) + Core.eval(ReviseTestPrivate, ex1) + Core.eval(ReviseTestPrivate, ex2) + exsig1 = Revise.RelocatableExpr(ex1)=>[Tuple{typeof(ReviseTestPrivate.methspecificity),Int}] + exsig2 = Revise.RelocatableExpr(ex2)=>[Tuple{typeof(ReviseTestPrivate.methspecificity),Integer}] + f_old, f_new = Revise.ExprsSigs(exsig1, exsig2), Revise.ExprsSigs(exsig2) + Revise.delete_missing!(f_old, f_new) + m = @which ReviseTestPrivate.methspecificity(1) + @test m.sig.parameters[2] === Integer + Revise.delete_missing!(f_old, f_new) + m = @which ReviseTestPrivate.methspecificity(1) + @test m.sig.parameters[2] === Integer + end + + do_test("revise_file_now") && @testset "revise_file_now" begin + # Very rarely this is used for debugging + testdir = newtestdir() + dn = joinpath(testdir, "ReviseFileNow", "src") + mkpath(dn) + fn = joinpath(dn, "ReviseFileNow.jl") + write(fn, """ + module ReviseFileNow + f(x) = 1 + end + """) + sleep(mtimedelay) + @eval using ReviseFileNow + @test ReviseFileNow.f(0) == 1 + sleep(mtimedelay) + pkgdata = Revise.pkgdatas[Base.PkgId(ReviseFileNow)] + write(fn, """ + module ReviseFileNow + f(x) = 2 + end + """) + try + Revise.revise_file_now(pkgdata, "foo") + catch err + @test isa(err, ErrorException) + @test occursin("not currently being tracked", err.msg) + end + Revise.revise_file_now(pkgdata, relpath(fn, pkgdata)) + @test ReviseFileNow.f(0) == 2 + + rm_precompile("ReviseFileNow") + end + + do_test("Evaled toplevel") && @testset "Evaled toplevel" begin + testdir = newtestdir() + dnA = joinpath(testdir, "ToplevelA", "src"); mkpath(dnA) + dnB = joinpath(testdir, "ToplevelB", "src"); mkpath(dnB) + dnC = joinpath(testdir, "ToplevelC", "src"); mkpath(dnC) + write(joinpath(dnA, "ToplevelA.jl"), """ + module ToplevelA + @eval using ToplevelB + g() = 2 + end""") + write(joinpath(dnB, "ToplevelB.jl"), """ + module ToplevelB + using ToplevelC + end""") + write(joinpath(dnC, "ToplevelC.jl"), """ + module ToplevelC + export f + f() = 1 + end""") + sleep(mtimedelay) + using ToplevelA + sleep(mtimedelay) + @test ToplevelA.ToplevelB.f() == 1 + @test ToplevelA.g() == 2 + write(joinpath(dnA, "ToplevelA.jl"), """ + module ToplevelA + @eval using ToplevelB + g() = 3 + end""") + yry() + @test ToplevelA.ToplevelB.f() == 1 + @test ToplevelA.g() == 3 + + rm_precompile("ToplevelA") + rm_precompile("ToplevelB") + rm_precompile("ToplevelC") + end + + do_test("struct inner functions") && @testset "struct inner functions" begin + # issue #599 + testdir = newtestdir() + dn = joinpath(testdir, "StructInnerFuncs", "src"); mkpath(dn) + write(joinpath(dn, "StructInnerFuncs.jl"), """ + module StructInnerFuncs + mutable struct A + x::Int + + A(x) = new(f(x)) + f(x) = x^2 + end + g(x) = 1 + end""") + sleep(mtimedelay) + using StructInnerFuncs + sleep(mtimedelay) + @test StructInnerFuncs.A(2).x == 4 + @test StructInnerFuncs.g(3) == 1 + write(joinpath(dn, "StructInnerFuncs.jl"), """ + module StructInnerFuncs + mutable struct A + x::Int + + A(x) = new(f(x)) + f(x) = x^2 + end + g(x) = 2 + end""") + yry() + @test StructInnerFuncs.A(2).x == 4 + @test StructInnerFuncs.g(3) == 2 + + rm_precompile("StructInnerFuncs") + end + + do_test("Issue 606") && @testset "Issue 606" begin + # issue #606 + testdir = newtestdir() + dn = joinpath(testdir, "Issue606", "src"); mkpath(dn) + write(joinpath(dn, "Issue606.jl"), """ + module Issue606 + function convert_output_relations() + function add_default_zero!(dict::Dict{K, V})::Dict{K, V} where + {K <: Tuple, V} + if K == Tuple{} && isempty(dict) + dict[()] = 0.0 + end + return dict + end + + function convert_to_sorteddict( + relation::Union{Dict{K, Tuple{Float64}}} + ) where K <: Tuple + return add_default_zero!(Dict{K, Float64}((k, v[1]) for (k, v) in relation)) + end + + function convert_to_sorteddict(relation::Dict{<:Tuple, Float64}) + return add_default_zero!(relation) + end + + return "HELLO" + end + end""") + sleep(mtimedelay) + using Issue606 + sleep(mtimedelay) + @test Issue606.convert_output_relations() == "HELLO" + write(joinpath(dn, "Issue606.jl"), """ + module Issue606 + function convert_output_relations() + function add_default_zero!(dict::Dict{K, V})::Dict{K, V} where + {K <: Tuple, V} + if K == Tuple{} && isempty(dict) + dict[()] = 0.0 + end + return dict + end + + function convert_to_sorteddict( + relation::Union{Dict{K, Tuple{Float64}}} + ) where K <: Tuple + return add_default_zero!(Dict{K, Float64}((k, v[1]) for (k, v) in relation)) + end + + function convert_to_sorteddict(relation::Dict{<:Tuple, Float64}) + return add_default_zero!(relation) + end + + return "HELLO2" + end + end""") + yry() + @test Issue606.convert_output_relations() == "HELLO2" + + rm_precompile("Issue606") + end + + do_test("Revision errors") && @testset "Revision errors" begin + testdir = newtestdir() + dn = joinpath(testdir, "RevisionErrors", "src") + mkpath(dn) + fn = joinpath(dn, "RevisionErrors.jl") + write(fn, """ + module RevisionErrors + f(x) = 1 + struct Vec{N, T <: Union{Float32,Float64}} + data::NTuple{N, T} + end + g(x) = 1 + end + """) + sleep(mtimedelay) + @eval using RevisionErrors + sleep(mtimedelay) + @test RevisionErrors.f(0) == 1 + write(fn, """ + module RevisionErrors + f{x) = 2 + struct Vec{N, T <: Union{Float32,Float64}} + data::NTuple{N, T} + end + g(x) = 1 + end + """) + logs, _ = Test.collect_test_logs() do + yry() + end + + function check_revision_error(rec, ErrorType, msg, line) + @test rec.message == "Failed to revise $fn" + exc = rec.kwargs[:exception] + if exc isa Revise.ReviseEvalException + exc, st = exc.exc, exc.stacktrace + else + exc, bt = exc + st = stacktrace(bt) + end + @test exc isa ErrorType + if ErrorType === LoadError + @test exc.file == fn + @test exc.line == line + @test occursin(msg, exc.error) + elseif ErrorType === UndefVarError + @test msg == exc.var + end + @test length(st) == 1 + end + + # test errors are reported the the first time + check_revision_error(logs[1], LoadError, "missing comma or }", 2) + # Check that there's an informative warning + rec = logs[2] + @test startswith(rec.message, "The running code does not match") + @test occursin("RevisionErrors.jl", rec.message) + + # test errors are not re-reported + logs, _ = Test.collect_test_logs() do + yry() + end + @test isempty(logs) + + # test error re-reporting + logs,_ = Test.collect_test_logs() do + Revise.errors() + end + check_revision_error(logs[1], LoadError, "missing comma or }", 2) + + write(joinpath(dn, "RevisionErrors.jl"), """ + module RevisionErrors + f(x) = 2 + struct Vec{N, T <: Union{Float32,Float64}} + data::NTuple{N, T} + end + g(x) = 1 + end + """) + logs, _ = Test.collect_test_logs() do + yry() + end + @test isempty(logs) + @test RevisionErrors.f(0) == 2 + + # issue #421 + write(joinpath(dn, "RevisionErrors.jl"), """ + module RevisionErrors + f(x) = 2 + struct Vec{N, T <: Union{Float32,Float64}} + data::NTuple{N, T} + end + function g(x) = 1 + end + """) + logs, _ = Test.collect_test_logs() do + yry() + end + check_revision_error(logs[1], LoadError, "unexpected \"=\"", 6) + + write(joinpath(dn, "RevisionErrors.jl"), """ + module RevisionErrors + f(x) = 2 + struct Vec{N, T <: Union{Float32,Float64}} + data::NTuple{N, T} + end + g(x) = 1 + end + """) + logs, _ = Test.collect_test_logs() do + yry() + end + @test isempty(logs) + + write(joinpath(dn, "RevisionErrors.jl"), """ + module RevisionErrors + f(x) = 2 + struct Vec{N, T <: Union{Float32,Float64}} + data::NTuple{N, T} + end + g(x) = 1 + foo(::Vector{T}) = 3 + end + """) + logs, _ = Test.collect_test_logs() do + yry() + end + check_revision_error(logs[1], UndefVarError, :T, 6) + + # issue #541 + sleep(mtimedelay) + write(joinpath(dn, "RevisionErrors.jl"), """ + module RevisionErrors + f(x) = 2 + struct Vec{N, T <: Union{Float32,Float64}} + data::NTuple{N, T} + end + g(x} = 2 + end + """) + @test try + revise(throw=true) + false + catch err + isa(err, LoadError) && occursin("""unexpected "}" """, err.error) + end + sleep(mtimedelay) + write(joinpath(dn, "RevisionErrors.jl"), """ + module RevisionErrors + f(x) = 2 + struct Vec{N, T <: Union{Float32,Float64}} + data::NTuple{N, T} + end + g(x) = 2 + end + """) + yry() + @test RevisionErrors.g(0) == 2 + + rm_precompile("RevisionErrors") + empty!(Revise.queue_errors) + + testfile = joinpath(testdir, "Test301.jl") + write(testfile, """ + module Test301 + mutable struct Struct301 + x::Int + unset + + Struct301(x::Integer) = new(x) + end + f(s) = s.unset + const s = Struct301(1) + if f(s) + g() = 1 + else + g() = 2 + end + end + """) + logfile = joinpath(tempdir(), randtmp()*".log") + open(logfile, "w") do io + redirect_stderr(io) do + includet(testfile) + end + end + sleep(mtimedelay) + lines = readlines(logfile) + @test lines[1] == "ERROR: UndefRefError: access to undefined reference" + @test any(str -> occursin(r"f\(.*Test301\.Struct301\)", str), lines) + @test any(str -> endswith(str, "Test301.jl:10"), lines) + + logfile = joinpath(tempdir(), randtmp()*".log") + open(logfile, "w") do io + redirect_stderr(io) do + includet("callee_error.jl") + end + end + sleep(mtimedelay) + lines = readlines(logfile) + @test lines[1] == "ERROR: BoundsError: attempt to access 3-element $(Vector{Int}) at index [4]" + @test any(str -> endswith(str, "callee_error.jl:12"), lines) + @test_throws UndefVarError CalleeError.foo(0.1f0) + end + + do_test("Retry on InterruptException") && @testset "Retry on InterruptException" begin + function check_revision_interrupt(logs) + rec = logs[1] + @test rec.message == "Failed to revise $fn" + exc = rec.kwargs[:exception] + if exc isa Revise.ReviseEvalException + exc, st = exc.exc, exc.stacktrace + else + exc, bt = exc + st = stacktrace(bt) + end + @test exc isa InterruptException + if length(logs) > 1 + rec = logs[2] + @test startswith(rec.message, "The running code does not match") + end + end + + testdir = newtestdir() + dn = joinpath(testdir, "RevisionInterrupt", "src") + mkpath(dn) + fn = joinpath(dn, "RevisionInterrupt.jl") + write(fn, """ + module RevisionInterrupt + f(x) = 1 + end + """) + sleep(mtimedelay) + @eval using RevisionInterrupt + sleep(mtimedelay) + @test RevisionInterrupt.f(0) == 1 + + # Interpreted & compiled mode + n = 1 + for errthrow in ("throw(InterruptException())", """ + eval(quote # this forces interpreted mode + throw(InterruptException()) + end)""") + n += 1 + write(fn, """ + module RevisionInterrupt + $errthrow + f(x) = $n + end + """) + logs, _ = Test.collect_test_logs() do + yry() + end + check_revision_interrupt(logs) + # This method gets deleted because it's redefined to f(x) = 2, + # but the error prevents it from getting that far. + # @test RevisionInterrupt.f(0) == 1 + # Check that InterruptException triggers a retry (issue #418) + logs, _ = Test.collect_test_logs() do + yry() + end + check_revision_interrupt(logs) + # @test RevisionInterrupt.f(0) == 1 + write(fn, """ + module RevisionInterrupt + f(x) = $n + end + """) + logs, _ = Test.collect_test_logs() do + yry() + end + @test isempty(logs) + @test RevisionInterrupt.f(0) == n + end + end + + do_test("Modify @enum") && @testset "Modify @enum" begin + testdir = newtestdir() + dn = joinpath(testdir, "ModifyEnum", "src") + mkpath(dn) + write(joinpath(dn, "ModifyEnum.jl"), """ + module ModifyEnum + @enum Fruit apple=1 orange=2 + end + """) + sleep(mtimedelay) + @eval using ModifyEnum + sleep(mtimedelay) + @test Int(ModifyEnum.apple) == 1 + @test ModifyEnum.apple isa ModifyEnum.Fruit + @test_throws UndefVarError Int(ModifyEnum.kiwi) + write(joinpath(dn, "ModifyEnum.jl"), """ + module ModifyEnum + @enum Fruit apple=1 orange=2 kiwi=3 + end + """) + yry() + @test Int(ModifyEnum.kiwi) == 3 + @test Base.instances(ModifyEnum.Fruit) === (ModifyEnum.apple, ModifyEnum.orange, ModifyEnum.kiwi) + rm_precompile("ModifyEnum") + pop!(LOAD_PATH) + end + + do_test("get_def") && @testset "get_def" begin + testdir = newtestdir() + dn = joinpath(testdir, "GetDef", "src") + mkpath(dn) + write(joinpath(dn, "GetDef.jl"), """ + module GetDef + + f(x) = 1 + f(v::AbstractVector) = 2 + f(v::AbstractVector{<:Integer}) = 3 + + foo(x::T, y::Integer=1; kw1="hello", kwargs...) where T<:Number = error("stop") + bar(x) = foo(x; kw1="world") + + end + """) + sleep(mtimedelay) + @eval using GetDef + sleep(mtimedelay) + @test GetDef.f(1.0) == 1 + @test GetDef.f([1.0]) == 2 + @test GetDef.f([1]) == 3 + m = @which GetDef.f([1]) + ex = Revise.RelocatableExpr(definition(m)) + @test ex isa Revise.RelocatableExpr + @test isequal(ex, Revise.RelocatableExpr(:(f(v::AbstractVector{<:Integer}) = 3))) + + st = try GetDef.bar(5.0) catch err stacktrace(catch_backtrace()) end + m = st[2].linfo.def + def = Revise.RelocatableExpr(definition(m)) + @test def == Revise.RelocatableExpr(:(foo(x::T, y::Integer=1; kw1="hello", kwargs...) where T<:Number = error("stop"))) + + rm_precompile("GetDef") + + # This method identifies itself as originating from @irrational, defined in Base, but + # the module of the method is listed as Base.MathConstants. + m = @which Float32(π) + @test definition(m) isa Expr + end + + do_test("Pkg exclusion") && @testset "Pkg exclusion" begin + push!(Revise.dont_watch_pkgs, :Example) + push!(Revise.silence_pkgs, :Example) + @eval import Example + id = Base.PkgId(Example) + @test !haskey(Revise.pkgdatas, id) + # Ensure that silencing works + sfile = Revise.silencefile[] # remember the original + try + sfiletemp = tempname() + Revise.silencefile[] = sfiletemp + Revise.silence("GSL") + @test isfile(sfiletemp) + pkgs = readlines(sfiletemp) + @test any(p->p=="GSL", pkgs) + rm(sfiletemp) + finally + Revise.silencefile[] = sfile + end + pop!(LOAD_PATH) + end + + do_test("Manual track") && @testset "Manual track" begin + srcfile = joinpath(tempdir(), randtmp()*".jl") + write(srcfile, "revise_f(x) = 1") + sleep(mtimedelay) + includet(srcfile) + sleep(mtimedelay) + @test revise_f(10) == 1 + @test length(signatures_at(srcfile, 1)) == 1 + write(srcfile, "revise_f(x) = 2") + yry() + @test revise_f(10) == 2 + push!(to_remove, srcfile) + + # Do it again with a relative path + curdir = pwd() + cd(tempdir()) + srcfile = randtmp()*".jl" + write(srcfile, "revise_floc(x) = 1") + sleep(mtimedelay) + include(joinpath(pwd(), srcfile)) + @test revise_floc(10) == 1 + Revise.track(srcfile) + sleep(mtimedelay) + write(srcfile, "revise_floc(x) = 2") + yry() + @test revise_floc(10) == 2 + # Call track again & make sure it doesn't track twice + Revise.track(srcfile) + id = Base.PkgId(Main) + pkgdata = Revise.pkgdatas[id] + @test count(isequal(srcfile), pkgdata.info.files) == 1 + push!(to_remove, joinpath(tempdir(), srcfile)) + cd(curdir) + + # Empty files (issue #253) + srcfile = joinpath(tempdir(), randtmp()*".jl") + write(srcfile, "\n") + sleep(mtimedelay) + includet(srcfile) + sleep(mtimedelay) + @test basename(srcfile) ∈ Revise.watched_files[dirname(srcfile)] + push!(to_remove, srcfile) + + # Double-execution (issue #263) + srcfile = joinpath(tempdir(), randtmp()*".jl") + write(srcfile, "println(\"executed\")") + sleep(mtimedelay) + logfile = joinpath(tempdir(), randtmp()*".log") + open(logfile, "w") do io + redirect_stdout(io) do + includet(srcfile) + end + end + sleep(mtimedelay) + lines = readlines(logfile) + @test length(lines) == 1 && chomp(lines[1]) == "executed" + # In older versions of Revise, it would do the work again when the file + # changed. Starting with 3.0, Revise modifies methods and docstrings but + # does not "do work." + write(srcfile, "println(\"executed again\")") + open(logfile, "w") do io + redirect_stdout(io) do + yry() + end + end + lines = readlines(logfile) + @test isempty(lines) + + # tls path (issue #264) + srcdir = joinpath(tempdir(), randtmp()) + mkpath(srcdir) + push!(to_remove, srcdir) + srcfile1 = joinpath(srcdir, randtmp()*".jl") + srcfile2 = joinpath(srcdir, randtmp()*".jl") + write(srcfile1, "includet(\"$(basename(srcfile2))\")") + write(srcfile2, "f264() = 1") + sleep(mtimedelay) + include(srcfile1) + sleep(mtimedelay) + @test f264() == 1 + write(srcfile2, "f264() = 2") + yry() + @test f264() == 2 + + # recursive `includet`s (issue #302) + testdir = newtestdir() + srcfile1 = joinpath(testdir, "Test302.jl") + write(srcfile1, """ + module Test302 + struct Parameters{T} + control::T + end + function Parameters(control = nothing; kw...) + Parameters(control) + end + function (p::Parameters)(; kw...) + p + end + end + """) + srcfile2 = joinpath(testdir, "test2.jl") + write(srcfile2, """ + includet(joinpath(@__DIR__, "Test302.jl")) + using .Test302 + """) + sleep(mtimedelay) + includet(srcfile2) + sleep(mtimedelay) + p = Test302.Parameters{Int}(3) + @test p() == p + write(srcfile1, """ + module Test302 + struct Parameters{T} + control::T + end + function Parameters(control = nothing; kw...) + Parameters(control) + end + function (p::Parameters)(; kw...) + 0 + end + end + """) + yry() + @test p() == 0 + + # Double-execution prevention (issue #639) + empty!(issue639report) + srcfile1 = joinpath(testdir, "file1.jl") + srcfile2 = joinpath(testdir, "file2.jl") + write(srcfile1, """ + include(joinpath(@__DIR__, "file2.jl")) + push!($(@__MODULE__).issue639report, '1') + """) + write(srcfile2, "push!($(@__MODULE__).issue639report, '2')") + sleep(mtimedelay) + includet(srcfile1) + @test issue639report == ['2', '1'] + + # Non-included dependency (issue #316) + testdir = newtestdir() + dn = joinpath(testdir, "LikePlots", "src"); mkpath(dn) + write(joinpath(dn, "LikePlots.jl"), """ + module LikePlots + plot() = 0 + backend() = include(joinpath(@__DIR__, "backends/backend.jl")) + end + """) + sd = joinpath(dn, "backends"); mkpath(sd) + write(joinpath(sd, "backend.jl"), "f() = 1") + sleep(mtimedelay) + @eval using LikePlots + @test LikePlots.plot() == 0 + @test_throws UndefVarError LikePlots.f() + sleep(mtimedelay) + Revise.track(LikePlots, joinpath(sd, "backend.jl")) + LikePlots.backend() + @test LikePlots.f() == 1 + sleep(2*mtimedelay) + write(joinpath(sd, "backend.jl"), "f() = 2") + yry() + @test LikePlots.f() == 2 + pkgdata = Revise.pkgdatas[Base.PkgId(LikePlots)] + @test joinpath("src", "backends", "backend.jl") ∈ Revise.srcfiles(pkgdata) + # No duplications from Revise.track with either relative or absolute paths + Revise.track(LikePlots, joinpath(sd, "backend.jl")) + @test length(Revise.srcfiles(pkgdata)) == 2 + cd(dn) do + Revise.track(LikePlots, joinpath("backends", "backend.jl")) + @test length(Revise.srcfiles(pkgdata)) == 2 + end + + rm_precompile("LikePlots") + + # Issue #475 + srcfile = joinpath(tempdir(), randtmp()*".jl") + write(srcfile, """ + a475 = 0.8 + a475 = 0.7 + a475 = 0.8 + """) + includet(srcfile) + @test a475 == 0.8 + + end + + do_test("Auto-track user scripts") && @testset "Auto-track user scripts" begin + srcfile = joinpath(tempdir(), randtmp()*".jl") + push!(to_remove, srcfile) + write(srcfile, "revise_g() = 1") + sleep(mtimedelay) + # By default user scripts are not tracked + # issue #358: but if the user is tracking all includes... + user_track_includes = Revise.tracking_Main_includes[] + Revise.tracking_Main_includes[] = false + include(srcfile) + yry() + @test revise_g() == 1 + write(srcfile, "revise_g() = 2") + yry() + @test revise_g() == 1 + # Turn on tracking of user scripts + empty!(Revise.included_files) # don't track files already loaded (like this one) + Revise.tracking_Main_includes[] = true + try + srcfile = joinpath(tempdir(), randtmp()*".jl") + push!(to_remove, srcfile) + write(srcfile, "revise_g() = 1") + sleep(mtimedelay) + include(srcfile) + yry() + @test revise_g() == 1 + write(srcfile, "revise_g() = 2") + yry() + @test revise_g() == 2 + + # issue #257 + logs, _ = Test.collect_test_logs() do # just to prevent noisy warning + try include("nonexistent1.jl") catch end + yry() + try include("nonexistent2.jl") catch end + yry() + end + finally + Revise.tracking_Main_includes[] = user_track_includes # restore old behavior + end + end + + do_test("Distributed") && @testset "Distributed" begin + # The d31474 test below is from + # https://discourse.julialang.org/t/how-do-i-make-revise-jl-work-in-multiple-workers-environment/31474 + newprocs = addprocs(2) + newproc = newprocs[end] + Revise.init_worker.(newprocs) + allworkers = [myid(); newprocs] + dirname = randtmp() + mkdir(dirname) + @everywhere push_LOAD_PATH!(dirname) = push!(LOAD_PATH, dirname) # Don't want to share this LOAD_PATH + for p in allworkers + remotecall_wait(push_LOAD_PATH!, p, dirname) + end + push!(to_remove, dirname) + modname = "ReviseDistributed" + dn = joinpath(dirname, modname, "src") + mkpath(dn) + s31474 = """ + function d31474() + r = @spawnat $newproc sqrt(4) + fetch(r) + end + """ + write(joinpath(dn, modname*".jl"), """ + module ReviseDistributed + using Distributed + + f() = π + g(::Int) = 0 + $s31474 + + end + """) + sleep(mtimedelay) + using ReviseDistributed + sleep(mtimedelay) + @everywhere using ReviseDistributed + for p in allworkers + @test remotecall_fetch(ReviseDistributed.f, p) == π + @test remotecall_fetch(ReviseDistributed.g, p, 1) == 0 + end + @test ReviseDistributed.d31474() == 2.0 + s31474 = """ + function d31474() + r = @spawnat $newproc sqrt(9) + fetch(r) + end + """ + write(joinpath(dn, modname*".jl"), """ + module ReviseDistributed + + f() = 3.0 + $s31474 + + end + """) + yry() + @test_throws MethodError ReviseDistributed.g(1) + for p in allworkers + @test remotecall_fetch(ReviseDistributed.f, p) == 3.0 + @test_throws RemoteException remotecall_fetch(ReviseDistributed.g, p, 1) + end + @test ReviseDistributed.d31474() == 3.0 + rmprocs(allworkers[2:3]...; waitfor=10) + rm_precompile("ReviseDistributed") + pop!(LOAD_PATH) + end + + + do_test("Distributed on worker") && @testset "Distributed on worker" begin + # https://github.com/timholy/Revise.jl/pull/527 + favorite_proc, boring_proc = addprocs(2) + + Distributed.remotecall_eval(Main, [favorite_proc, boring_proc], :(ENV["JULIA_REVISE_WORKER_ONLY"] = "1")) + + dirname = randtmp() + mkdir(dirname) + push!(to_remove, dirname) + + @everywhere push_LOAD_PATH!(dirname) = push!(LOAD_PATH, dirname) # Don't want to share this LOAD_PATH + remotecall_wait(push_LOAD_PATH!, favorite_proc, dirname) + + modname = "ReviseDistributedOnWorker" + dn = joinpath(dirname, modname, "src") + mkpath(dn) + + s527_old = """ + module ReviseDistributedOnWorker + + f() = π + g(::Int) = 0 + + end + """ + write(joinpath(dn, modname*".jl"), s527_old) + + # In the first tests, we only load Revise on our favorite process. The other (boring) process should be unaffected by the upcoming tests. + Distributed.remotecall_eval(Main, [favorite_proc], :(using Revise)) + sleep(mtimedelay) + Distributed.remotecall_eval(Main, [favorite_proc], :(using ReviseDistributedOnWorker)) + sleep(mtimedelay) + + @test Distributed.remotecall_eval(Main, favorite_proc, :(ReviseDistributedOnWorker.f())) == π + @test Distributed.remotecall_eval(Main, favorite_proc, :(ReviseDistributedOnWorker.g(1))) == 0 + + # we only loaded ReviseDistributedOnWorker on our favorite process + @test_throws RemoteException Distributed.remotecall_eval(Main, boring_proc, :(ReviseDistributedOnWorker.f())) + @test_throws RemoteException Distributed.remotecall_eval(Main, boring_proc, :(ReviseDistributedOnWorker.g(1))) + + s527_new = """ + module ReviseDistributedOnWorker + + f() = 3.0 + + end + """ + write(joinpath(dn, modname*".jl"), s527_new) + sleep(mtimedelay) + Distributed.remotecall_eval(Main, [favorite_proc], :(Revise.revise())) + sleep(mtimedelay) + + + @test Distributed.remotecall_eval(Main, favorite_proc, :(ReviseDistributedOnWorker.f())) == 3.0 + @test_throws RemoteException Distributed.remotecall_eval(Main, favorite_proc, :(ReviseDistributedOnWorker.g(1))) + + @test_throws RemoteException Distributed.remotecall_eval(Main, boring_proc, :(ReviseDistributedOnWorker.f())) + @test_throws RemoteException Distributed.remotecall_eval(Main, boring_proc, :(ReviseDistributedOnWorker.g(1))) + + # In the second part, we'll also load Revise on the boring process, which should have no effect. + Distributed.remotecall_eval(Main, [boring_proc], :(using Revise)) + + write(joinpath(dn, modname*".jl"), s527_old) + + sleep(mtimedelay) + @test !Distributed.remotecall_eval(Main, favorite_proc, :(Revise.revision_queue |> isempty)) + @test Distributed.remotecall_eval(Main, boring_proc, :(Revise.revision_queue |> isempty)) + + Distributed.remotecall_eval(Main, [favorite_proc, boring_proc], :(Revise.revise())) + sleep(mtimedelay) + + + @test Distributed.remotecall_eval(Main, favorite_proc, :(ReviseDistributedOnWorker.f())) == π + @test Distributed.remotecall_eval(Main, favorite_proc, :(ReviseDistributedOnWorker.g(1))) == 0 + + @test_throws RemoteException Distributed.remotecall_eval(Main, boring_proc, :(ReviseDistributedOnWorker.f())) + @test_throws RemoteException Distributed.remotecall_eval(Main, boring_proc, :(ReviseDistributedOnWorker.g(1))) + + rmprocs(favorite_proc, boring_proc; waitfor=10) + end + + do_test("Git") && @testset "Git" begin + loc = Base.find_package("Revise") + if occursin("dev", loc) + repo, path = Revise.git_repo(loc) + @test repo != nothing + files = Revise.git_files(repo) + @test "README.md" ∈ files + src = Revise.git_source(loc, "946d588328c2eb5fe5a56a21b4395379e41092e0") + @test startswith(src, "__precompile__") + src = Revise.git_source(loc, "eae5e000097000472280e6183973a665c4243b94") # 2nd commit in Revise's history + @test src == "module Revise\n\n# package code goes here\n\nend # module\n" + else + @warn "skipping git tests because Revise is not under development" + end + # Issue #135 + if !Sys.iswindows() + randdir = randtmp() + modname = "ModuleWithNewFile" + push!(to_remove, randdir) + push!(LOAD_PATH, randdir) + randdir = joinpath(randdir, modname) + mkpath(joinpath(randdir, "src")) + mainjl = joinpath(randdir, "src", modname*".jl") + LibGit2.with(LibGit2.init(randdir)) do repo + write(mainjl, """ + module $modname + end + """) + LibGit2.add!(repo, joinpath("src", modname*".jl")) + test_sig = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time(); digits=0), 0) + LibGit2.commit(repo, "New file test"; author=test_sig, committer=test_sig) + end + sleep(mtimedelay) + @eval using $(Symbol(modname)) + sleep(mtimedelay) + mod = @eval $(Symbol(modname)) + id = Base.PkgId(mod) + extrajl = joinpath(randdir, "src", "extra.jl") + write(extrajl, "println(\"extra\")") + write(mainjl, """ + module $modname + include("extra.jl") + end + """) + sleep(mtimedelay) + repo = LibGit2.GitRepo(randdir) + LibGit2.add!(repo, joinpath("src", "extra.jl")) + pkgdata = Revise.pkgdatas[id] + logs, _ = Test.collect_test_logs() do + Revise.track_subdir_from_git!(pkgdata, joinpath(randdir, "src"); commit="HEAD") + end + yry() + @test Revise.hasfile(pkgdata, mainjl) + @test startswith(logs[end].message, "skipping src/extra.jl") || startswith(logs[end-1].message, "skipping src/extra.jl") + rm_precompile("ModuleWithNewFile") + pop!(LOAD_PATH) + end + end + + do_test("Recipes") && @testset "Recipes" begin + # https://github.com/JunoLab/Juno.jl/issues/257#issuecomment-473856452 + meth = @which gcd(10, 20) + sigs = signatures_at(Base.find_source_file(String(meth.file)), meth.line) # this should track Base + + # Tracking Base + # issue #250 + @test_throws ErrorException("use Revise.track(Base) or Revise.track()") Revise.track(joinpath(Revise.juliadir, "base", "intfuncs.jl")) + + id = Base.PkgId(Base) + pkgdata = Revise.pkgdatas[id] + @test any(k->endswith(k, "number.jl"), Revise.srcfiles(pkgdata)) + @test length(filter(k->endswith(k, "file.jl"), Revise.srcfiles(pkgdata))) == 1 + m = @which show([1,2,3]) + @test definition(m) isa Expr + m = @which redirect_stdout() + @test definition(m).head ∈ (:function, :(=)) + + # Tracking stdlibs + Revise.track(Unicode) + id = Base.PkgId(Unicode) + pkgdata = Revise.pkgdatas[id] + @test any(k->endswith(k, "Unicode.jl"), Revise.srcfiles(pkgdata)) + m = first(methods(Unicode.isassigned)) + @test definition(m) isa Expr + @test isfile(whereis(m)[1]) + + # Submodule of Pkg (note that package is developed outside the + # Julia repo, this tests new cases) + id = Revise.get_tracked_id(Pkg.Types) + pkgdata = Revise.pkgdatas[id] + @test definition(first(methods(Pkg.API.add))) isa Expr + + # Test that we skip over files that don't end in ".jl" + logs, _ = Test.collect_test_logs() do + Revise.track(REPL) + end + @test isempty(logs) + + Revise.get_tracked_id(Core) # just test that this doesn't error + + # Determine whether a git repo is available. Travis & Appveyor do not have this. + repo, path = Revise.git_repo(Revise.juliadir) + if repo != nothing + # Tracking Core.Compiler + Revise.track(Core.Compiler) + id = Base.PkgId(Core.Compiler) + pkgdata = Revise.pkgdatas[id] + @test any(k->endswith(k, "optimize.jl"), Revise.srcfiles(pkgdata)) + m = first(methods(Core.Compiler.typeinf_code)) + @test definition(m) isa Expr + else + @test_throws Revise.GitRepoException Revise.track(Core.Compiler) + @warn "skipping Core.Compiler tests due to lack of git repo" + end + end + + do_test("CodeTracking #48") && @testset "CodeTracking #48" begin + m = @which sum([1]; dims=1) + file, line = whereis(m) + @test endswith(file, "reducedim.jl") && line > 1 + end + + do_test("Methods at REPL") && @testset "Methods at REPL" begin + if isdefined(Base, :active_repl) + hp = Base.active_repl.interface.modes[1].hist + fstr = "__fREPL__(x::Int16) = 0" + histidx = length(hp.history) + 1 - hp.start_idx + ex = Base.parse_input_line(fstr; filename="REPL[$histidx]") + f = Core.eval(Main, ex) + if ex.head === :toplevel + ex = ex.args[end] + end + push!(hp.history, fstr) + m = first(methods(f)) + @test !isempty(signatures_at(String(m.file), m.line)) + @test isequal(Revise.RelocatableExpr(definition(m)), Revise.RelocatableExpr(ex)) + @test definition(String, m)[1] == fstr + + # Test that revisions work (https://github.com/timholy/CodeTracking.jl/issues/38) + fstr = "__fREPL__(x::Int16) = 1" + histidx = length(hp.history) + 1 - hp.start_idx + ex = Base.parse_input_line(fstr; filename="REPL[$histidx]") + f = Core.eval(Main, ex) + if ex.head === :toplevel + ex = ex.args[end] + end + push!(hp.history, fstr) + m = first(methods(f)) + @test isequal(Revise.RelocatableExpr(definition(m)), Revise.RelocatableExpr(ex)) + @test definition(String, m)[1] == fstr + @test !isempty(signatures_at(String(m.file), m.line)) + + pop!(hp.history) + pop!(hp.history) + else + @warn "REPL tests skipped" + end + end + + do_test("baremodule") && @testset "baremodule" begin + testdir = newtestdir() + dn = joinpath(testdir, "Baremodule", "src") + mkpath(dn) + write(joinpath(dn, "Baremodule.jl"), """ + baremodule Baremodule + f() = 1 + end + """) + sleep(mtimedelay) + @eval using Baremodule + sleep(mtimedelay) + @test Baremodule.f() == 1 + write(joinpath(dn, "Baremodule.jl"), """ + module Baremodule + f() = 2 + end + """) + yry() + @test Baremodule.f() == 2 + rm_precompile("Baremodule") + pop!(LOAD_PATH) + end + + do_test("module style 2-argument includes (issue #670)") && @testset "module style 2-argument includes (issue #670)" begin + testdir = newtestdir() + dn = joinpath(testdir, "B670", "src") + mkpath(dn) + write(joinpath(dn, "A670.jl"), """ + x = 6 + y = 7 + """) + sleep(mtimedelay) + write(joinpath(dn, "B670.jl"), """ + module B670 + x = 5 + end + """) + sleep(mtimedelay) + write(joinpath(dn, "C670.jl"), """ + using B670 + Base.include(B670, "A670.jl") + """) + sleep(mtimedelay) + @eval using B670 + path = joinpath(dn, "C670.jl") + @eval include($path) + @test B670.x == 6 + @test B670.y == 7 + rm_precompile("B670") + end +end + +do_test("Utilities") && @testset "Utilities" begin + # Used by Rebugger but still lives here + io = IOBuffer() + Revise.println_maxsize(io, "a"^100; maxchars=50) + str = String(take!(io)) + @test startswith(str, "a"^25) + @test endswith(chomp(chomp(str)), "a"^24) + @test occursin("…", str) +end + +do_test("Switching free/dev") && @testset "Switching free/dev" begin + function make_a2d(path, val, mode="r"; generate=true) + # Create a new "read-only package" (which mimics how Pkg works when you `add` a package) + cd(path) do + pkgpath = normpath(joinpath(path, "A2D")) + srcpath = joinpath(pkgpath, "src") + if generate + Pkg.generate("A2D") + else + mkpath(srcpath) + end + filepath = joinpath(srcpath, "A2D.jl") + write(filepath, """ + module A2D + f() = $val + end + """) + chmod(filepath, mode=="r" ? 0o100444 : 0o100644) + return pkgpath + end + end + # Create a new package depot + depot = mktempdir() + old_depots = copy(DEPOT_PATH) + empty!(DEPOT_PATH) + push!(DEPOT_PATH, depot) + # Skip cloning the General registry since that is slow and unnecessary + ENV["JULIA_PKG_SERVER"] = "" + registries = isdefined(Pkg.Types, :DEFAULT_REGISTRIES) ? Pkg.Types.DEFAULT_REGISTRIES : Pkg.Registry.DEFAULT_REGISTRIES + old_registries = copy(registries) + empty!(registries) + # Ensure we start fresh with no dependencies + old_project = Base.ACTIVE_PROJECT[] + Base.ACTIVE_PROJECT[] = joinpath(depot, "environments", "v$(VERSION.major).$(VERSION.minor)", "Project.toml") + mkpath(dirname(Base.ACTIVE_PROJECT[])) + write(Base.ACTIVE_PROJECT[], "[deps]") + ropkgpath = make_a2d(depot, 1) + Pkg.develop(PackageSpec(path=ropkgpath)) + sleep(mtimedelay) + @eval using A2D + sleep(mtimedelay) + @test Base.invokelatest(A2D.f) == 1 + for dir in keys(Revise.watched_files) + @test !startswith(dir, ropkgpath) + end + devpath = joinpath(depot, "dev") + mkpath(devpath) + mfile = Revise.manifest_file() + schedule(Task(Revise.TaskThunk(Revise.watch_manifest, (mfile,)))) + sleep(mtimedelay) + pkgdevpath = make_a2d(devpath, 2, "w"; generate=false) + cp(joinpath(ropkgpath, "Project.toml"), joinpath(devpath, "A2D/Project.toml")) + Pkg.develop(PackageSpec(path=pkgdevpath)) + yry() + @test Base.invokelatest(A2D.f) == 2 + Pkg.develop(PackageSpec(path=ropkgpath)) + yry() + @test Base.invokelatest(A2D.f) == 1 + for dir in keys(Revise.watched_files) + @test !startswith(dir, ropkgpath) + end + + # Restore internal Pkg data + empty!(DEPOT_PATH) + append!(DEPOT_PATH, old_depots) + for pr in old_registries + push!(registries, pr) + end + Base.ACTIVE_PROJECT[] = old_project + + push!(to_remove, depot) +end + +# in v1.8 and higher, a package can't be loaded at all when its precompilation failed +@static if Base.VERSION < v"1.8.0-DEV.1451" +do_test("Broken dependencies (issue #371)") && @testset "Broken dependencies (issue #371)" begin + testdir = newtestdir() + srcdir = joinpath(testdir, "DepPkg371", "src") + filepath = joinpath(srcdir, "DepPkg371.jl") + cd(testdir) do + Pkg.generate("DepPkg371") + write(filepath, """ + module DepPkg371 + using OrderedCollections # undeclared dependency + greet() = "Hello world!" + end + """) + end + sleep(mtimedelay) + @info "A warning about not having OrderedCollection in dependencies is expected" + @eval using DepPkg371 + @test DepPkg371.greet() == "Hello world!" + sleep(mtimedelay) + write(filepath, """ + module DepPkg371 + using OrderedCollections # undeclared dependency + greet() = "Hello again!" + end + """) + yry() + @test DepPkg371.greet() == "Hello again!" + + rm_precompile("DepPkg371") + pop!(LOAD_PATH) +end +end # @static if VERSION ≤ v"1.7" + +do_test("Non-jl include_dependency (issue #388)") && @testset "Non-jl include_dependency (issue #388)" begin + push!(LOAD_PATH, joinpath(@__DIR__, "pkgs")) + @eval using ExcludeFile + sleep(0.01) + pkgdata = Revise.pkgdatas[Base.PkgId(UUID("b915cca1-7962-4ffb-a1c7-2bbdb2d9c14c"), "ExcludeFile")] + files = Revise.srcfiles(pkgdata) + @test length(files) == 2 + @test joinpath("src", "ExcludeFile.jl") ∈ files + @test joinpath("src", "f.jl") ∈ files + @test joinpath("deps", "dependency.txt") ∉ files +end + +do_test("New files & Requires.jl") && @testset "New files & Requires.jl" begin + # Issue #107 + testdir = newtestdir() + dn = joinpath(testdir, "NewFile", "src") + mkpath(dn) + write(joinpath(dn, "NewFile.jl"), """ + module NewFile + f() = 1 + module SubModule + struct NewType end + end + end + """) + sleep(mtimedelay) + @eval using NewFile + @test NewFile.f() == 1 + @test_throws UndefVarError NewFile.g() + sleep(mtimedelay) + write(joinpath(dn, "g.jl"), "g() = 2") + write(joinpath(dn, "NewFile.jl"), """ + module NewFile + include("g.jl") + f() = 1 + module SubModule + struct NewType end + end + end + """) + yry() + @test NewFile.f() == 1 + @test NewFile.g() == 2 + sd = joinpath(dn, "subdir") + mkpath(sd) + write(joinpath(sd, "h.jl"), "h(::NewType) = 3") + write(joinpath(dn, "NewFile.jl"), """ + module NewFile + include("g.jl") + f() = 1 + module SubModule + struct NewType end + include("subdir/h.jl") + end + end + """) + yry() + @test NewFile.f() == 1 + @test NewFile.g() == 2 + @test NewFile.SubModule.h(NewFile.SubModule.NewType()) == 3 + + dn = joinpath(testdir, "DeletedFile", "src") + mkpath(dn) + write(joinpath(dn, "DeletedFile.jl"), """ + module DeletedFile + include("g.jl") + f() = 1 + end + """) + write(joinpath(dn, "g.jl"), "g() = 1") + sleep(mtimedelay) + @eval using DeletedFile + @test DeletedFile.f() == DeletedFile.g() == 1 + sleep(mtimedelay) + write(joinpath(dn, "DeletedFile.jl"), """ + module DeletedFile + f() = 1 + end + """) + rm(joinpath(dn, "g.jl")) + yry() + @test DeletedFile.f() == 1 + @test_throws MethodError DeletedFile.g() + + rm_precompile("NewFile") + rm_precompile("DeletedFile") + + # https://discourse.julialang.org/t/revise-with-requires/19347 + dn = joinpath(testdir, "TrackRequires", "src") + mkpath(dn) + write(joinpath(dn, "TrackRequires.jl"), """ + module TrackRequires + using Requires + const called_onearg = Ref(false) + onearg(x) = called_onearg[] = true + module SubModule + abstract type SuperType end + end + function __init__() + @require EndpointRanges="340492b5-2a47-5f55-813d-aca7ddf97656" begin + export testfunc + include("testfile.jl") + end + @require CatIndices="aafaddc9-749c-510e-ac4f-586e18779b91" onearg(1) + @require IndirectArrays="9b13fd28-a010-5f03-acff-a1bbcff69959" @eval SubModule include("st.jl") + @require RoundingIntegers="d5f540fe-1c90-5db3-b776-2e2f362d9394" begin + fn = joinpath(@__DIR__, "subdir", "anotherfile.jl") + include(fn) + @require Revise="295af30f-e4ad-537b-8983-00126c2a3abe" Revise.track(TrackRequires, fn) + end + @require UnsafeArrays="c4a57d5a-5b31-53a6-b365-19f8c011fbd6" begin + fn = joinpath(@__DIR__, "subdir", "yetanotherfile.jl") + include(fn) + end + end + end # module + """) + write(joinpath(dn, "testfile.jl"), "testfunc() = 1") + write(joinpath(dn, "st.jl"), """ + struct NewType <: SuperType end + h(::NewType) = 3 + """) + sd = mkpath(joinpath(dn, "subdir")) + write(joinpath(sd, "anotherfile.jl"), "ftrack() = 1") + write(joinpath(sd, "yetanotherfile.jl"), "fauto() = 1") + sleep(mtimedelay) + @eval using TrackRequires + notified = isdefined(TrackRequires.Requires, :withnotifications) + notified || @warn "Requires does not support notifications" + @test_throws UndefVarError TrackRequires.testfunc() + @test_throws UndefVarError TrackRequires.SubModule.h(TrackRequires.SubModule.NewType()) + @eval using EndpointRanges # to trigger Requires + sleep(mtimedelay) + notified && @test TrackRequires.testfunc() == 1 + write(joinpath(dn, "testfile.jl"), "testfunc() = 2") + yry() + notified && @test TrackRequires.testfunc() == 2 + @test_throws UndefVarError TrackRequires.SubModule.h(TrackRequires.SubModule.NewType()) + # Issue #477 + @eval using IndirectArrays + sleep(mtimedelay) + notified && @test TrackRequires.SubModule.h(TrackRequires.SubModule.NewType()) == 3 + # Check a non-block expression + warnfile = randtmp() + open(warnfile, "w") do io + redirect_stderr(io) do + @eval using CatIndices + sleep(0.5) + end + end + notified && @test TrackRequires.called_onearg[] + @test isempty(read(warnfile, String)) + # Issue #431 + @test_throws UndefVarError TrackRequires.ftrack() + if !(get(ENV, "CI", nothing) == "true" && Base.VERSION.major == 1 && Base.VERSION.minor == 8) # circumvent CI hang + @eval using RoundingIntegers + sleep(2) # allow time for the @async in all @require blocks to finish + if notified + @test TrackRequires.ftrack() == 1 + id = Base.PkgId(TrackRequires) + pkgdata = Revise.pkgdatas[id] + sf = Revise.srcfiles(pkgdata) + @test count(name->occursin("@require", name), sf) == 1 + @test count(name->occursin("anotherfile", name), sf) == 1 + @test !any(isequal("."), sf) + idx = findfirst(name->occursin("anotherfile", name), sf) + @test !isabspath(sf[idx]) + end + end + @test_throws UndefVarError TrackRequires.fauto() + @eval using UnsafeArrays + sleep(2) # allow time for the @async in all @require blocks to finish + if notified + @test TrackRequires.fauto() == 1 + id = Base.PkgId(TrackRequires) + pkgdata = Revise.pkgdatas[id] + sf = Revise.srcfiles(pkgdata) + @test count(name->occursin("@require", name), sf) == 1 + @test count(name->occursin("yetanotherfile", name), sf) == 1 + @test !any(isequal("."), sf) + idx = findfirst(name->occursin("yetanotherfile", name), sf) + @test !isabspath(sf[idx]) + end + + # Ensure it also works if the Requires dependency is pre-loaded + dn = joinpath(testdir, "TrackRequires2", "src") + mkpath(dn) + write(joinpath(dn, "TrackRequires2.jl"), """ + module TrackRequires2 + using Requires + function __init__() + @require EndpointRanges="340492b5-2a47-5f55-813d-aca7ddf97656" begin + export testfunc + include("testfile.jl") + end + @require MappedArrays="dbb5928d-eab1-5f90-85c2-b9b0edb7c900" begin + export othertestfunc + include("testfile2.jl") + end + end + end # module + """) + write(joinpath(dn, "testfile.jl"), "testfunc() = 1") + write(joinpath(dn, "testfile2.jl"), "othertestfunc() = -1") + sleep(mtimedelay) + @eval using TrackRequires2 + sleep(mtimedelay) + notified && @test TrackRequires2.testfunc() == 1 + @test_throws UndefVarError TrackRequires2.othertestfunc() + write(joinpath(dn, "testfile.jl"), "testfunc() = 2") + yry() + notified && @test TrackRequires2.testfunc() == 2 + @test_throws UndefVarError TrackRequires2.othertestfunc() + @eval using MappedArrays + @test TrackRequires2.othertestfunc() == -1 + sleep(mtimedelay) + write(joinpath(dn, "testfile2.jl"), "othertestfunc() = -2") + yry() + notified && @test TrackRequires2.othertestfunc() == -2 + + # Issue #442 + push!(LOAD_PATH, joinpath(@__DIR__, "pkgs")) + @eval using Pkg442 + sleep(0.01) + @test check442() + @test Pkg442.check442A() + @test Pkg442.check442B() + @test Pkg442.Dep442B.has442A() + pop!(LOAD_PATH) + + rm_precompile("TrackRequires") + rm_precompile("TrackRequires2") + pop!(LOAD_PATH) +end + +do_test("entr") && @testset "entr" begin + srcfile1 = joinpath(tempdir(), randtmp()*".jl"); push!(to_remove, srcfile1) + srcfile2 = joinpath(tempdir(), randtmp()*".jl"); push!(to_remove, srcfile2) + revise(throw=true) # force compilation + write(srcfile1, "Core.eval(Main, :(__entr__ = 1))") + touch(srcfile2) + Core.eval(Main, :(__entr__ = 0)) + sleep(mtimedelay) + try + @sync begin + @test Main.__entr__ == 0 + + @async begin + entr([srcfile1, srcfile2]; pause=0.5) do + include(srcfile1) + end + end + sleep(1) + @test Main.__entr__ == 1 # callback should have been run (postpone=false) + + # File modification + write(srcfile1, "Core.eval(Main, :(__entr__ = 2))") + sleep(1) + @test Main.__entr__ == 2 # callback should have been called + + # Two events in quick succession (w.r.t. the `pause` argument) + write(srcfile1, "Core.eval(Main, :(__entr__ += 1))") + sleep(0.1) + touch(srcfile2) + sleep(1) + @test Main.__entr__ == 3 # callback should have been called only once + + + write(srcfile1, "error(\"stop\")") + sleep(mtimedelay) + end + @test false + catch err + while err isa CompositeException + err = err.exceptions[1] + if err isa TaskFailedException + err = err.task.exception + end + if err isa CapturedException + err = err.ex + end + end + @test isa(err, LoadError) + @test err.error.msg == "stop" + end + + # Callback should have been removed + @test isempty(Revise.user_callbacks_by_file[srcfile1]) + + + # Watch directories (#470) + try + @sync let + srcdir = joinpath(tempdir(), randtmp()) + mkdir(srcdir) + + trigger = joinpath(srcdir, "trigger.txt") + + counter = Ref(0) + stop = Ref(false) + + @async begin + entr([srcdir]; pause=0.5) do + counter[] += 1 + stop[] && error("stop watching directory") + end + end + sleep(1) + @test length(readdir(srcdir)) == 0 # directory should still be empty + @test counter[] == 1 # postpone=false + + # File creation + touch(trigger) + sleep(1) + @test counter[] == 2 + + # File modification + touch(trigger) + sleep(1) + @test counter[] == 3 + + # File deletion -> the directory should be empty again + rm(trigger) + sleep(1) + @test length(readdir(srcdir)) == 0 + @test counter[] == 4 + + # Two events in quick succession (w.r.t. the `pause` argument) + touch(trigger) # creation + sleep(0.1) + touch(trigger) # modification + sleep(1) + @test counter[] == 5 # Callback should have been called only once + + # Stop + stop[] = true + touch(trigger) + end + + # `entr` should have errored by now + @test false + catch err + while err isa CompositeException + err = err.exceptions[1] + if err isa TaskFailedException + err = err.task.exception + end + if err isa CapturedException + err = err.ex + end + end + @test isa(err, ErrorException) + @test err.msg == "stop watching directory" + end +end + +const A354_result = Ref(0) + +# issue #354 +do_test("entr with modules") && @testset "entr with modules" begin + + testdir = newtestdir() + modname = "A354" + srcfile = joinpath(testdir, modname * ".jl") + + setvalue(x) = write(srcfile, "module $modname test() = $x end") + + setvalue(1) + + # these sleeps may not be needed... + sleep(mtimedelay) + @eval using A354 + sleep(mtimedelay) + + A354_result[] = 0 + + @async begin + sleep(mtimedelay) + setvalue(2) + # belt and suspenders -- make sure we trigger entr: + sleep(mtimedelay) + touch(srcfile) + sleep(mtimedelay) + end + + try + entr([], [A354], postpone=true) do + A354_result[] = A354.test() + error() + end + catch err + end + + @test A354_result[] == 2 + + rm_precompile(modname) + +end + +# issue #469 +do_test("entr with all files") && @testset "entr with all files" begin + testdir = newtestdir() + modname = "A469" + srcfile = joinpath(testdir, modname * ".jl") + write(srcfile, "module $modname test() = 469 end") + + sleep(mtimedelay) + @eval using A469 + sleep(mtimedelay) + result = Ref(0) + + try + @sync begin + @async begin + # Watch all files known to Revise + # (including `srcfile`) + entr([]; all=true, postpone=true) do + result[] = 1 + error("stop") + end + end + sleep(mtimedelay) + @test result[] == 0 + + # Trigger the callback + touch(srcfile) + end + @test false + catch err + while err isa CompositeException + err = err.exceptions[1] + if err isa TaskFailedException + err = err.task.exception + end + if err isa CapturedException + err = err.ex + end + end + @test isa(err, ErrorException) + @test err.msg == "stop" + end + + # If we got to this point, the callback should have been triggered. But + # let's check nonetheless + @test result[] == 1 + + rm_precompile(modname) +end + +do_test("callbacks") && @testset "callbacks" begin + + append(path, x...) = open(path, append=true) do io + write(io, x...) + end + + mktemp() do path, _ + contents = Ref("") + key = Revise.add_callback([path]) do + contents[] = read(path, String) + end + + sleep(mtimedelay) + + append(path, "abc") + sleep(mtimedelay) + revise() + @test contents[] == "abc" + + sleep(mtimedelay) + + append(path, "def") + sleep(mtimedelay) + revise() + @test contents[] == "abcdef" + + Revise.remove_callback(key) + sleep(mtimedelay) + + append(path, "ghi") + sleep(mtimedelay) + revise() + @test contents[] == "abcdef" + end + + testdir = newtestdir() + modname = "A355" + srcfile = joinpath(testdir, modname * ".jl") + + setvalue(x) = write(srcfile, "module $modname test() = $x end") + + setvalue(1) + + sleep(mtimedelay) + @eval using A355 + sleep(mtimedelay) + + A355_result = Ref(0) + + Revise.add_callback([], [A355]) do + A355_result[] = A355.test() + end + + sleep(mtimedelay) + setvalue(2) + # belt and suspenders -- make sure we trigger entr: + sleep(mtimedelay) + touch(srcfile) + + yry() + + @test A355_result[] == 2 + + rm_precompile(modname) + + # Issue 574 - ad-hoc revision of a file, combined with add_callback() + A574_path = joinpath(testdir, "A574.jl") + + set_foo_A574(x) = write(A574_path, "foo_574() = $x") + + set_foo_A574(1) + includet(@__MODULE__, A574_path) + @test Base.invokelatest(foo_574) == 1 + + foo_A574_result = Ref(0) + key = Revise.add_callback([A574_path]) do + foo_A574_result[] = foo_574() + end + + sleep(mtimedelay) + set_foo_A574(2) + sleep(mtimedelay) + revise() + @test Base.invokelatest(foo_574) == 2 + @test foo_A574_result[] == 2 + + Revise.remove_callback(key) + + sleep(mtimedelay) + set_foo_A574(3) + sleep(mtimedelay) + revise() + @test Base.invokelatest(foo_574) == 3 + @test foo_A574_result[] == 2 # <- callback removed - no longer updated +end + +do_test("includet with mod arg (issue #689)") && @testset "includet with mod arg (issue #689)" begin + testdir = newtestdir() + + common = joinpath(testdir, "common.jl") + write(common, """ + module Common + const foo = 2 + end + """) + + routines = joinpath(testdir, "routines.jl") + write(routines, """ + module Routines + using Revise + includet(@__MODULE__, raw"$common") + using .Common + end + """) + + codes = joinpath(testdir, "codes.jl") + write(codes, """ + module Codes + using Revise + includet(@__MODULE__, raw"$common") + using .Common + end + """) + + driver = joinpath(testdir, "driver.jl") + write(driver, """ + module Driver + using Revise + includet(@__MODULE__, raw"$routines") + using .Routines + includet(@__MODULE__, raw"$codes") + using .Codes + end + """) + + includet(@__MODULE__, driver) + @test parentmodule(Driver.Routines.Common) == Driver.Routines + @test Base.moduleroot(Driver.Routines.Common) == Main + + @test parentmodule(Driver.Codes.Common) == Driver.Codes + @test Base.moduleroot(Driver.Codes.Common) == Main + + @test Driver.Routines.Common.foo == 2 + @test Driver.Codes.Common.foo == 2 +end + +do_test("misc - coverage") && @testset "misc - coverage" begin + @test Revise.ReviseEvalException("undef", UndefVarError(:foo)).loc isa String + @test !Revise.throwto_repl(UndefVarError(:foo)) + + @test endswith(Revise.fallback_juliadir(), "julia") + + @test isnothing(Revise.revise(REPL.REPLBackend())) +end + +do_test("deprecated") && @testset "deprecated" begin + @test_logs (:warn, r"`steal_repl_backend` has been removed.*") Revise.async_steal_repl_backend() + @test_logs (:warn, r"`steal_repl_backend` has been removed.*") Revise.wait_steal_repl_backend() +end + +println("beginning cleanup") +GC.gc(); GC.gc() + +@testset "Cleanup" begin + logs, _ = Test.collect_test_logs() do + warnfile = randtmp() + open(warnfile, "w") do io + redirect_stderr(io) do + for name in to_remove + try + rm(name; force=true, recursive=true) + deleteat!(LOAD_PATH, findall(LOAD_PATH .== name)) + catch + end + end + for i = 1:3 + yry() + GC.gc() + end + end + end + msg = Revise.watching_files[] ? "is not an existing file" : "is not an existing directory" + isempty(ARGS) && !Sys.isapple() && @test occursin(msg, read(warnfile, String)) + rm(warnfile) + end +end + +GC.gc(); GC.gc(); GC.gc() # work-around for https://github.com/JuliaLang/julia/issues/28306 + +# see #532 Fix InitError opening none existent Project.toml +function load_in_empty_project_test() + # This will try to load Revise in a julia seccion + # with an empty enviroment (missing Project.toml) + + julia = Base.julia_cmd() + revise_proj = escape_string(Base.active_project()) + @assert isfile(revise_proj) + + src = """ + import Pkg + Pkg.activate("fake_env") + @assert !isfile(Base.active_project()) + + # force to load the package env Revise version + empty!(LOAD_PATH) + push!(LOAD_PATH, "$revise_proj") + + @info "A warning about no Manifest.toml file found is expected" + try; using Revise + catch err + # just fail for this error (see #532) + err isa InitError && rethrow(err) + end + """ + cmd = `$julia --project=@. -E $src` + + @test begin + wait(run(cmd)) + true + end +end + +do_test("Import in empty enviroment (issue #532)") && @testset "Import in empty enviroment (issue #532)" begin + load_in_empty_project_test(); +end + +include("backedges.jl") + +do_test("Base signatures") && @testset "Base signatures" begin + println("beginning signatures tests") + # Using the extensive repository of code in Base as a testbed + include("sigtest.jl") +end + +include("non_jl_test.jl") diff --git a/packages/Revise/test/sigtest.jl b/packages/Revise/test/sigtest.jl new file mode 100644 index 0000000..dff689f --- /dev/null +++ b/packages/Revise/test/sigtest.jl @@ -0,0 +1,133 @@ +using Revise, Test +using Revise.CodeTracking +using Revise.LoweredCodeUtils + +function isdefinedmod(mod::Module) + # Not all modules---e.g., LibGit2---are reachable without loading the stdlib + names = fullname(mod) + pmod = Main + for n in names + isdefined(pmod, n) || return false + pmod = getfield(pmod, n) + end + return true +end +function reljpath(path) + for subdir in ("base/", "stdlib/", "test/") + s = split(path, subdir) + if length(s) == 2 + return subdir * s[end] + end + end + return path +end +function filepredicate(file, reffiles) + bfile = Base.find_source_file(file) + bfile === nothing && return false # when the file is "none" + return reljpath(bfile) ∈ reffiles +end +function signature_diffs(mod::Module, signatures; filepredicate=nothing) + extras = copy(signatures) + modeval, modinclude = getfield(mod, :eval), getfield(mod, :include) + failed = [] + nmethods = 0 + for fsym in names(mod; all=true) + isdefined(mod, fsym) || continue + f = getfield(mod, fsym) + isa(f, Base.Callable) || continue + (f === modeval || f === modinclude) && continue + for m in methods(f) + nmethods += 1 + if haskey(signatures, m.sig) + delete!(extras, m.sig) + else + if filepredicate !== nothing + filepredicate(String(m.file)) || continue # find signatures only in selected files + end + push!(failed, m.sig) + end + end + end + return failed, extras, nmethods +end +function extracttype(T) + p1 = T.parameters[1] + isa(p1, Type) && return p1 + isa(p1, TypeVar) && return p1.ub + error("unrecognized type ", T) +end +if isdefined(Core, :TypeofVararg) + istva(T) = isa(T, Core.TypeofVararg) +else + istva(T) = false +end +function in_module_or_core(T, mod::Module) + if isa(T, TypeVar) + return in_module_or_core(T.ub, mod) + end + if isa(T, UnionAll) + T = Base.unwrap_unionall(T) + end + T === Union{} && return true + if isa(T, Union) + in_module_or_core(T.a, mod) || return false + return in_module_or_core(T.b, mod) + end + if istva(T) + isdefined(T, :T) || return true + return in_module_or_core(T.T, mod) + end + Tname = T.name + if Tname.name === :Type + return in_module_or_core(extracttype(T), mod) + end + Tmod = Tname.module + return Tmod === mod || Tmod === Core +end + +module Lowering end + +@testset ":lambda expressions" begin + ex = quote + mutable struct InnerC + x::Int + valid::Bool + + function InnerC(x; notvalid::Bool=false) + return new(x, !notvalid) + end + end + end + sigs, _ = Revise.eval_with_signatures(Lowering, ex) + @test length(sigs) >= 2 +end + +basefiles = Set{String}() +@time for (i, (mod, file)) in enumerate(Base._included_files) + endswith(file, "sysimg.jl") && continue + file = Revise.fixpath(file) + push!(basefiles, reljpath(file)) + mexs = Revise.parse_source(file, mod) + Revise.instantiate_sigs!(mexs; always_rethrow=true) +end +failed, extras, nmethods = signature_diffs(Base, CodeTracking.method_info; filepredicate = fn->filepredicate(fn, basefiles)) +# In some cases, the above doesn't really select the file-of-origin. For example, anything +# defined with an @enum gets attributed to Enum.jl rather than the file in which @enum is used. +realfailed = similar(failed, 0) +for sig in failed + ft = Base.unwrap_unionall(sig).parameters[1] + match(r"^getfield\(Base, Symbol\(\"##\d", string(ft)) === nothing || continue # exclude anonymous functions + all(T->in_module_or_core(T, Base), Base.unwrap_unionall(sig).parameters[2:end]) || continue + push!(realfailed, sig) +end +if false # change to true to see the failures + world = Base.get_world_counter() + for tt in realfailed + println(tt) + mms = Base._methods_by_ftype(tt, -1, world) + for mm in mms + println(mm.method) + end + end +end +@test length(realfailed) < 60 # big enough for some cushion in case new "difficult" methods get added diff --git a/packages/Revise/test/start_late.jl b/packages/Revise/test/start_late.jl new file mode 100644 index 0000000..db101f7 --- /dev/null +++ b/packages/Revise/test/start_late.jl @@ -0,0 +1,14 @@ +# For this test, Julia should be started without Revise and then it should be added to the running session +# Catches #664 + +using Test + +@async(Base.run_main_repl(true, true, false, true, false)) +while !isdefined(Base, :active_repl_backend) + sleep(0.5) +end + +using Revise +@test Revise.revise_first ∈ Base.active_repl_backend.ast_transforms + +exit() diff --git a/packages/TestEnv/.github/workflows/TagBot.yml b/packages/TestEnv/.github/workflows/TagBot.yml new file mode 100644 index 0000000..f49313b --- /dev/null +++ b/packages/TestEnv/.github/workflows/TagBot.yml @@ -0,0 +1,15 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/packages/TestEnv/.gitignore b/packages/TestEnv/.gitignore new file mode 100644 index 0000000..5f65356 --- /dev/null +++ b/packages/TestEnv/.gitignore @@ -0,0 +1,8 @@ +*.DS_Store +*.jl.cov +*.jl.*.cov +*.jl.mem +Manifest.toml +docs/build +docs/site +.vscode/settings.json \ No newline at end of file diff --git a/packages/TestEnv/LICENSE b/packages/TestEnv/LICENSE new file mode 100644 index 0000000..d4439c8 --- /dev/null +++ b/packages/TestEnv/LICENSE @@ -0,0 +1,49 @@ +The TestEnv.jl package is licensed under the MIT "Expat" License: + +> Copyright (c) 2018-2021: Lyndon White and Malcolm Miller. +> +> Permission is hereby granted, free of charge, to any person obtaining a copy +> of this software and associated documentation files (the "Software"), to deal +> in the Software without restriction, including without limitation the rights +> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is +> furnished to do so, subject to the following conditions: +> +> The above copyright notice and this permission notice shall be included in all +> copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. +> + +Some of the code (in particular /src/runner.jl), is based on code from the Julia Base library https://github.com/JuliaLang/julia + + +> Copyright (c) 2009-2021: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, +> and other contributors: +> +> https://github.com/JuliaLang/julia/contributors +> +> Permission is hereby granted, free of charge, to any person obtaining +> a copy of this software and associated documentation files (the +> "Software"), to deal in the Software without restriction, including +> without limitation the rights to use, copy, modify, merge, publish, +> distribute, sublicense, and/or sell copies of the Software, and to +> permit persons to whom the Software is furnished to do so, subject to +> the following conditions: +> +> The above copyright notice and this permission notice shall be +> included in all copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +> NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +> LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +> OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +> WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/packages/TestEnv/Project.toml b/packages/TestEnv/Project.toml new file mode 100644 index 0000000..eb52465 --- /dev/null +++ b/packages/TestEnv/Project.toml @@ -0,0 +1,21 @@ +name = "TestEnv" +uuid = "1e6cf692-eddd-4d53-88a5-2d735e33781b" +version = "2.0.0" + +[deps] +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[compat] +ChainRulesCore = "=1.0.2" +MCMCDiagnosticTools = "=0.1.0" +YAXArrays = "0.1.3" +julia = "1" + +[extras] +ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +MCMCDiagnosticTools = "be115224-59cd-429b-ad48-344e309966f0" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c" + +[targets] +test = ["ChainRulesCore", "MCMCDiagnosticTools", "Test", "YAXArrays"] diff --git a/packages/TestEnv/README.md b/packages/TestEnv/README.md new file mode 100644 index 0000000..f476808 --- /dev/null +++ b/packages/TestEnv/README.md @@ -0,0 +1,76 @@ +# TestEnv + +[![Code Style: Blue](https://img.shields.io/badge/code%20style-blue-4495d1.svg)](https://github.com/invenia/BlueStyle) +[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac) + + +This is a 1-function package: `TestEnv.activate`. +It lets you activate the test enviroment from a given package. +Just like `Pkg.activate` lets you activate it's main enviroment. + + +Consider for example **ChainRules.jl** has as a test-only dependency of **ChainRulesTestUtils.jl**, +not a main dependency + +```julia +pkg> activate ~/.julia/dev/ChainRules + +julia> using TestEnv; + +julia> TestEnv.activate(); + +julia> using ChainRulesTestUtils +``` + +Use `Pkg.activate` to re-activate the previous environment, e.g. `Pkg.activate("~/.julia/dev/ChainRules")`. + +You can also pass in the name of a package, to activate that package and it's test dependencies: +`TestEnv.activate("Javis")` for example would activate Javis.jl's test environment. + +Finally you can pass in a function to run in this environment. +```julia +using TestEnv, ReTest +TestEnv.activate("Example") do + retest() +end +``` + +## Where is the code? +The astute reader has probably notice that the default branch of this git repo is basically empty. +This is because we keep all the code in other branches. +One per minor release: `release-1.0`, `release-1.1` etc. +We do this because TestEnv.jl accesses a whole ton of interals of [Pkg](https://github.com/JuliaLang/Pkg.jl). +These internals change basically every single release. +Maintaining compatibility in a single branch for multiple julia versions leads to code that is a nightmare. +As such, we instead maintain 1 branch per julia minor version. +And we tag releases off that branch with major and minor versions matching the julia version supported, but with patch versions allowed to change freely. + + - [release-1.0](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.0) contains the code to support julia v1.0.x + - [release-1.1](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.1) contains the code to support julia v1.1.x + - [release-1.2](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.2) contains the code to support julia v1.2.x + - [release-1.3](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.3) contains the code to support julia v1.3.x + - [release-1.4](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.4) contains the code to support julia v1.4.x, v1.5.x, and v1.6.x + - This was a rare goldern ages where the internals of Pkg did not change for almost a year. + - [release-1.7](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.7) contains the code to support julia v1.7.x + - [release-1.8](https://github.com/JuliaTesting/TestEnv.jl/tree/release-1.8) contains the code to support julia v1.8.x + + +**Do not make PRs against this COVER branch.** +Except to update this README. +Instead you probably want to PR a branch for some current version of Julia. + +This is a bit weird for semver. +New features *can* be added in patch release, but they must be ported to all later branches, and patch releases must be made there also. +For the this reason: we *only* support the latest patch release of any branch. +Older ones may be yanked if they start causing issues for people. + + +## What should I put in my Project.toml `[compat]` section +If using this as a dependency of a package that supports many versions of julia you may wonder what to put in your Project.toml's [compat] section. +Do not fear, the package manager has your back. +If you put in your `[compat]` for `TestEnv=`: `1` or equivalently `1.0` or `1.0` or `1.0.0` or `^1`, or `^1.0` or `^1.0` or `^1.0.0`, +then the package manager is free to choose any compatible version `v` with `1.0.0 <= v < 2.0.0`. +It will thus chose the corret minor version of TestEnv that is compatible with the loaded version of Julia. + +### See also: + - [Discourse Release Announcement](https://discourse.julialang.org/t/ann-testenv-jl-activate-your-test-enviroment-so-you-can-use-your-test-dependencies/65739) diff --git a/src/TestEnv.jl b/packages/TestEnv/src/TestEnv.jl similarity index 100% rename from src/TestEnv.jl rename to packages/TestEnv/src/TestEnv.jl diff --git a/src/julia-1.0/TestEnv.jl b/packages/TestEnv/src/julia-1.0/TestEnv.jl similarity index 100% rename from src/julia-1.0/TestEnv.jl rename to packages/TestEnv/src/julia-1.0/TestEnv.jl diff --git a/src/julia-1.0/activate_do.jl b/packages/TestEnv/src/julia-1.0/activate_do.jl similarity index 100% rename from src/julia-1.0/activate_do.jl rename to packages/TestEnv/src/julia-1.0/activate_do.jl diff --git a/src/julia-1.0/activate_set.jl b/packages/TestEnv/src/julia-1.0/activate_set.jl similarity index 100% rename from src/julia-1.0/activate_set.jl rename to packages/TestEnv/src/julia-1.0/activate_set.jl diff --git a/src/julia-1.0/common.jl b/packages/TestEnv/src/julia-1.0/common.jl similarity index 100% rename from src/julia-1.0/common.jl rename to packages/TestEnv/src/julia-1.0/common.jl diff --git a/src/julia-1.1/TestEnv.jl b/packages/TestEnv/src/julia-1.1/TestEnv.jl similarity index 100% rename from src/julia-1.1/TestEnv.jl rename to packages/TestEnv/src/julia-1.1/TestEnv.jl diff --git a/src/julia-1.1/activate_do.jl b/packages/TestEnv/src/julia-1.1/activate_do.jl similarity index 100% rename from src/julia-1.1/activate_do.jl rename to packages/TestEnv/src/julia-1.1/activate_do.jl diff --git a/src/julia-1.1/activate_set.jl b/packages/TestEnv/src/julia-1.1/activate_set.jl similarity index 100% rename from src/julia-1.1/activate_set.jl rename to packages/TestEnv/src/julia-1.1/activate_set.jl diff --git a/src/julia-1.1/common.jl b/packages/TestEnv/src/julia-1.1/common.jl similarity index 100% rename from src/julia-1.1/common.jl rename to packages/TestEnv/src/julia-1.1/common.jl diff --git a/src/julia-1.2/TestEnv.jl b/packages/TestEnv/src/julia-1.2/TestEnv.jl similarity index 100% rename from src/julia-1.2/TestEnv.jl rename to packages/TestEnv/src/julia-1.2/TestEnv.jl diff --git a/src/julia-1.2/activate_do.jl b/packages/TestEnv/src/julia-1.2/activate_do.jl similarity index 100% rename from src/julia-1.2/activate_do.jl rename to packages/TestEnv/src/julia-1.2/activate_do.jl diff --git a/src/julia-1.2/activate_set.jl b/packages/TestEnv/src/julia-1.2/activate_set.jl similarity index 100% rename from src/julia-1.2/activate_set.jl rename to packages/TestEnv/src/julia-1.2/activate_set.jl diff --git a/src/julia-1.2/common.jl b/packages/TestEnv/src/julia-1.2/common.jl similarity index 100% rename from src/julia-1.2/common.jl rename to packages/TestEnv/src/julia-1.2/common.jl diff --git a/src/julia-1.3/TestEnv.jl b/packages/TestEnv/src/julia-1.3/TestEnv.jl similarity index 100% rename from src/julia-1.3/TestEnv.jl rename to packages/TestEnv/src/julia-1.3/TestEnv.jl diff --git a/src/julia-1.3/activate_do.jl b/packages/TestEnv/src/julia-1.3/activate_do.jl similarity index 100% rename from src/julia-1.3/activate_do.jl rename to packages/TestEnv/src/julia-1.3/activate_do.jl diff --git a/src/julia-1.3/activate_set.jl b/packages/TestEnv/src/julia-1.3/activate_set.jl similarity index 100% rename from src/julia-1.3/activate_set.jl rename to packages/TestEnv/src/julia-1.3/activate_set.jl diff --git a/src/julia-1.3/common.jl b/packages/TestEnv/src/julia-1.3/common.jl similarity index 100% rename from src/julia-1.3/common.jl rename to packages/TestEnv/src/julia-1.3/common.jl diff --git a/src/julia-1.4/TestEnv.jl b/packages/TestEnv/src/julia-1.4/TestEnv.jl similarity index 100% rename from src/julia-1.4/TestEnv.jl rename to packages/TestEnv/src/julia-1.4/TestEnv.jl diff --git a/src/julia-1.4/activate_do.jl b/packages/TestEnv/src/julia-1.4/activate_do.jl similarity index 100% rename from src/julia-1.4/activate_do.jl rename to packages/TestEnv/src/julia-1.4/activate_do.jl diff --git a/src/julia-1.4/activate_set.jl b/packages/TestEnv/src/julia-1.4/activate_set.jl similarity index 100% rename from src/julia-1.4/activate_set.jl rename to packages/TestEnv/src/julia-1.4/activate_set.jl diff --git a/src/julia-1.4/common.jl b/packages/TestEnv/src/julia-1.4/common.jl similarity index 100% rename from src/julia-1.4/common.jl rename to packages/TestEnv/src/julia-1.4/common.jl diff --git a/src/julia-1.7/TestEnv.jl b/packages/TestEnv/src/julia-1.7/TestEnv.jl similarity index 100% rename from src/julia-1.7/TestEnv.jl rename to packages/TestEnv/src/julia-1.7/TestEnv.jl diff --git a/src/julia-1.7/activate_do.jl b/packages/TestEnv/src/julia-1.7/activate_do.jl similarity index 100% rename from src/julia-1.7/activate_do.jl rename to packages/TestEnv/src/julia-1.7/activate_do.jl diff --git a/src/julia-1.7/activate_set.jl b/packages/TestEnv/src/julia-1.7/activate_set.jl similarity index 100% rename from src/julia-1.7/activate_set.jl rename to packages/TestEnv/src/julia-1.7/activate_set.jl diff --git a/src/julia-1.7/common.jl b/packages/TestEnv/src/julia-1.7/common.jl similarity index 100% rename from src/julia-1.7/common.jl rename to packages/TestEnv/src/julia-1.7/common.jl diff --git a/src/julia-1.8/TestEnv.jl b/packages/TestEnv/src/julia-1.8/TestEnv.jl similarity index 100% rename from src/julia-1.8/TestEnv.jl rename to packages/TestEnv/src/julia-1.8/TestEnv.jl diff --git a/src/julia-1.8/activate_do.jl b/packages/TestEnv/src/julia-1.8/activate_do.jl similarity index 100% rename from src/julia-1.8/activate_do.jl rename to packages/TestEnv/src/julia-1.8/activate_do.jl diff --git a/src/julia-1.8/activate_set.jl b/packages/TestEnv/src/julia-1.8/activate_set.jl similarity index 100% rename from src/julia-1.8/activate_set.jl rename to packages/TestEnv/src/julia-1.8/activate_set.jl diff --git a/src/julia-1.8/common.jl b/packages/TestEnv/src/julia-1.8/common.jl similarity index 100% rename from src/julia-1.8/common.jl rename to packages/TestEnv/src/julia-1.8/common.jl diff --git a/src/julia-1.9/TestEnv.jl b/packages/TestEnv/src/julia-1.9/TestEnv.jl similarity index 100% rename from src/julia-1.9/TestEnv.jl rename to packages/TestEnv/src/julia-1.9/TestEnv.jl diff --git a/src/julia-1.9/activate_do.jl b/packages/TestEnv/src/julia-1.9/activate_do.jl similarity index 100% rename from src/julia-1.9/activate_do.jl rename to packages/TestEnv/src/julia-1.9/activate_do.jl diff --git a/src/julia-1.9/activate_set.jl b/packages/TestEnv/src/julia-1.9/activate_set.jl similarity index 100% rename from src/julia-1.9/activate_set.jl rename to packages/TestEnv/src/julia-1.9/activate_set.jl diff --git a/src/julia-1.9/common.jl b/packages/TestEnv/src/julia-1.9/common.jl similarity index 100% rename from src/julia-1.9/common.jl rename to packages/TestEnv/src/julia-1.9/common.jl diff --git a/packages/TestItemServer/Project.toml b/packages/TestItemServer/Project.toml new file mode 100644 index 0000000..a91bef5 --- /dev/null +++ b/packages/TestItemServer/Project.toml @@ -0,0 +1,25 @@ +name = "TestItemServer" +uuid = "1076c532-7f93-4758-ac65-93afa8eae143" + +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" +Mmap = "a63ad114-7e13-5084-954f-fe012c677804" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[compat] +julia = "1" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/packages/TestItemServer/README.md b/packages/TestItemServer/README.md new file mode 100644 index 0000000..f1dbfc3 --- /dev/null +++ b/packages/TestItemServer/README.md @@ -0,0 +1,3 @@ +# TestItemServer.jl + +This package implements the functionality that runs inside a test item test process. The package is not registered, but instead vendored by both the VS Code extension and the TestItemRunner.jl package. diff --git a/packages/TestItemServer/src/TestItemServer.jl b/packages/TestItemServer/src/TestItemServer.jl new file mode 100644 index 0000000..b434878 --- /dev/null +++ b/packages/TestItemServer/src/TestItemServer.jl @@ -0,0 +1,327 @@ +module TestItemServer + +include("pkg_imports.jl") + +import .JSONRPC: @dict_readable +import Test, Pkg + +include("testserver_protocol.jl") +include("helper.jl") + +const conn_endpoint = Ref{Union{Nothing,JSONRPC.JSONRPCEndpoint}}(nothing) +const TESTSETUPS = Dict{Symbol,Any}() + +function run_update_testsetups(conn, params::TestserverUpdateTestsetupsRequestParams) + new_testsetups = Dict(i.name => i for i in params.testsetups) + + # Delete all existing test setups that are not in the new list + for i in keys(TESTSETUPS) + if !haskey(new_testsetups, i) + delete!(TESTSETUPS, i) + end + end + + for i in params.testsetups + # We only add new if not there before or if the code changed + if !haskey(TESTSETUPS, i.name) || (haskey(TESTSETUPS, i.name) && TESTSETUPS[i.name].code != i.code) + TESTSETUPS[Symbol(i.name)] = ( + name = i.name, + uri = i.uri, + line = i.line, + column = i.column, + code = i.code, + evaled = false + ) + end + end +end + +function withpath(f, path) + tls = task_local_storage() + hassource = haskey(tls, :SOURCE_PATH) + hassource && (path′ = tls[:SOURCE_PATH]) + tls[:SOURCE_PATH] = path + try + return f() + finally + hassource ? (tls[:SOURCE_PATH] = path′) : delete!(tls, :SOURCE_PATH) + end +end + + +function run_revise_handler(conn, params::Nothing) + try + Revise.revise(throw=true) + return "success" + catch err + Base.display_error(err, catch_backtrace()) + return "failed" + end +end + +function flatten_failed_tests!(ts, out) + append!(out, i for i in ts.results if !(i isa Test.Pass)) + + for cts in ts.children + flatten_failed_tests!(cts, out) + end +end + +function format_error_message(err, bt) + try + return Base.invokelatest(sprint, Base.display_error, err, bt) + catch err + # TODO We could probably try to output an even better error message here that + # takes into account `err`. And in the callsites we should probably also + # handle this better. + return "Error while trying to format an error message" + end +end + +function run_testitem_handler(conn, params::TestserverRunTestitemRequestParams) + for i in params.testsetups + if !haskey(TESTSETUPS, Symbol(i)) + ret = TestserverRunTestitemRequestParamsReturn( + "errored", + [ + TestMessage( + "The specified testsetup $i does not exist.", + Location( + params.uri, + Range(Position(params.line, 0), Position(params.line, 0)) + ) + ) + ], + missing + ) + return ret + end + + setup_details = TESTSETUPS[Symbol(i)] + + if !setup_details.evaled + mod = Core.eval(Main.Testsetups, :(module $(Symbol(i)) end)) + + code = string('\n'^setup_details.line, ' '^setup_details.column, setup_details.code) + + filepath = uri2filepath(setup_details.uri) + + t0 = time_ns() + try + withpath(filepath) do + Base.invokelatest(include_string, mod, code, filepath) + end + catch err + elapsed_time = (time_ns() - t0) / 1e6 # Convert to milliseconds + + bt = catch_backtrace() + st = stacktrace(bt) + + error_message = format_error_message(err, bt) + + if err isa LoadError + error_filepath = err.file + error_line = err.line + else + error_filepath = string(st[1].file) + error_line = st[1].line + end + + return TestserverRunTestitemRequestParamsReturn( + "errored", + [ + TestMessage( + error_message, + Location( + isabspath(error_filepath) ? filepath2uri(error_filepath) : "", + Range(Position(max(0, error_line - 1), 0), Position(max(0, error_line - 1), 0)) + ) + ) + ], + elapsed_time + ) + end + end + end + + mod = Core.eval(Main, :(module $(gensym()) end)) + + if params.useDefaultUsings + try + Core.eval(mod, :(using Test)) + catch + return TestserverRunTestitemRequestParamsReturn( + "errored", + [ + TestMessage( + "Unable to load the `Test` package. Please ensure that `Test` is listed as a test dependency in the Project.toml for the package.", + Location( + params.uri, + Range(Position(params.line, 0), Position(params.line, 0)) + ) + ) + ], + nothing + ) + end + + if params.packageName!="" + try + Core.eval(mod, :(using $(Symbol(params.packageName)))) + catch err + bt = catch_backtrace() + error_message = format_error_message(err, bt) + + return TestserverRunTestitemRequestParamsReturn( + "errored", + [ + TestMessage( + error_message, + Location( + params.uri, + Range(Position(params.line, 0), Position(params.line, 0)) + ) + ) + ], + nothing + ) + end + end + end + + for i in params.testsetups + try + Core.eval(mod, :(using ..Testsetups: $(Symbol(i)))) + catch + return TestserverRunTestitemRequestParamsReturn( + "errored", + [ + TestMessage( + "Unable to load the `$i` testsetup.", + Location( + params.uri, + Range(Position(params.line, 0), Position(params.line, 0)) + ) + ) + ], + nothing + ) + end + end + + filepath = uri2filepath(params.uri) + + code = string('\n'^params.line, ' '^params.column, params.code) + + ts = Test.DefaultTestSet("$filepath:$(params.name)") + + Test.push_testset(ts) + + elapsed_time = UInt64(0) + + t0 = time_ns() + try + withpath(filepath) do + Base.invokelatest(include_string, mod, code, filepath) + elapsed_time = (time_ns() - t0) / 1e6 # Convert to milliseconds + end + catch err + elapsed_time = (time_ns() - t0) / 1e6 # Convert to milliseconds + + Test.pop_testset() + + bt = catch_backtrace() + st = stacktrace(bt) + + error_message = format_error_message(err, bt) + + if err isa LoadError + error_filepath = err.file + error_line = err.line + else + error_filepath = string(st[1].file) + error_line = st[1].line + end + + return TestserverRunTestitemRequestParamsReturn( + "errored", + [ + TestMessage( + error_message, + Location( + isabspath(error_filepath) ? filepath2uri(error_filepath) : "", + Range(Position(max(0, error_line - 1), 0), Position(max(0, error_line - 1), 0)) + ) + ) + ], + elapsed_time + ) + end + + ts = Test.pop_testset() + + try + Test.finish(ts) + + return TestserverRunTestitemRequestParamsReturn("passed", missing, elapsed_time) + catch err + if err isa Test.TestSetException + failed_tests = Test.filter_errors(ts) + + return TestserverRunTestitemRequestParamsReturn( + "failed", + [TestMessage(sprint(Base.show, i), Location(filepath2uri(string(i.source.file)), Range(Position(i.source.line - 1, 0), Position(i.source.line - 1, 0)))) for i in failed_tests], + elapsed_time + ) + else + rethrow(err) + end + end +end + +function serve_in_env(conn) + conn_endpoint[] = JSONRPC.JSONRPCEndpoint(conn, conn) + @debug "connected" + run(conn_endpoint[]) + @debug "running" + + msg_dispatcher = JSONRPC.MsgDispatcher() + + msg_dispatcher[testserver_revise_request_type] = run_revise_handler + msg_dispatcher[testserver_run_testitem_request_type] = run_testitem_handler + msg_dispatcher[testserver_update_testsetups_type] = run_update_testsetups + + while conn_endpoint[] isa JSONRPC.JSONRPCEndpoint && isopen(conn) + msg = JSONRPC.get_next_message(conn_endpoint[]) + + JSONRPC.dispatch_msg(conn_endpoint[], msg_dispatcher, msg) + end +end + +function serve(conn, project_path, package_path, package_name; is_dev=false, crashreporting_pipename::Union{AbstractString,Nothing}=nothing) + if project_path=="" + Pkg.activate(temp=true) + + Pkg.develop(path=package_path) + + TestEnv.activate(package_name) do + serve_in_env(conn) + end + else + Pkg.activate(project_path) + + if package_name!="" + TestEnv.activate(package_name) do + serve_in_env(conn) + end + else + serve_in_env(conn) + end + end +end + +function __init__() + Core.eval(Main, :(module Testsetups end)) +end + +end diff --git a/packages/TestItemServer/src/helper.jl b/packages/TestItemServer/src/helper.jl new file mode 100644 index 0000000..7c62aaf --- /dev/null +++ b/packages/TestItemServer/src/helper.jl @@ -0,0 +1,62 @@ +# TODO Use our new Uri2 once it is ready +function uri2filepath(uri::AbstractString) + parsed_uri = try + URIParser.URI(uri) + catch + error("Cannot parse `$uri`.") + end + + if parsed_uri.scheme !== "file" + return nothing + end + + path_unescaped = URIParser.unescape(parsed_uri.path) + host_unescaped = URIParser.unescape(parsed_uri.host) + + value = "" + + if host_unescaped != "" && length(path_unescaped) > 1 + # unc path: file://shares/c$/far/boo + value = "//$host_unescaped$path_unescaped" + elseif length(path_unescaped) >= 3 && + path_unescaped[1] == '/' && + isascii(path_unescaped[2]) && isletter(path_unescaped[2]) && + path_unescaped[3] == ':' + # windows drive letter: file:///c:/far/boo + value = lowercase(path_unescaped[2]) * path_unescaped[3:end] + else + # other path + value = path_unescaped + end + + if Sys.iswindows() + value = replace(value, '/' => '\\') + end + + value = normpath(value) + + return value +end + +# TODO Use our new Uri2 once it is ready +function filepath2uri(file::String) + isabspath(file) || error("Relative path `$file` is not valid.") + if Sys.iswindows() + file = normpath(file) + file = replace(file, "\\" => "/") + file = URIParser.escape(file) + file = replace(file, "%2F" => "/") + if startswith(file, "//") + # UNC path \\foo\bar\foobar + return string("file://", file[3:end]) + else + # windows drive letter path + return string("file:///", file) + end + else + file = normpath(file) + file = URIParser.escape(file) + file = replace(file, "%2F" => "/") + return string("file://", file) + end +end diff --git a/packages/TestItemServer/src/pkg_imports.jl b/packages/TestItemServer/src/pkg_imports.jl new file mode 100644 index 0000000..ca8e8a1 --- /dev/null +++ b/packages/TestItemServer/src/pkg_imports.jl @@ -0,0 +1,48 @@ +include("../../TestEnv/src/TestEnv.jl") +include("../../URIParser/src/URIParser.jl") +include("../../JSON/src/JSON.jl") +include("../../OrderedCollections/src/OrderedCollections.jl") +include("../../CodeTracking/src/CodeTracking.jl") + +module JSONRPC +import ..JSON +import UUIDs +include("../../JSONRPC/src/packagedef.jl") +end + +module JuliaInterpreter +using ..CodeTracking +include("../../JuliaInterpreter/src/packagedef.jl") +end + +module LoweredCodeUtils +using ..JuliaInterpreter +using ..JuliaInterpreter: SSAValue, SlotNumber, Frame +using ..JuliaInterpreter: @lookup, moduleof, pc_expr, step_expr!, is_global_ref, is_quotenode_egal, whichtt, + next_until!, finish_and_return!, get_return, nstatements, codelocation, linetable, + is_return, lookup_return, is_GotoIfNot, is_ReturnNode + +include("../../LoweredCodeUtils/src/packagedef.jl") +end + +module Revise +using ..OrderedCollections +using ..LoweredCodeUtils +using ..CodeTracking +using ..JuliaInterpreter +using ..CodeTracking: PkgFiles, basedir, srcfiles, line_is_decl, basepath +using ..JuliaInterpreter: whichtt, is_doc_expr, step_expr!, finish_and_return!, get_return, + @lookup, moduleof, scopeof, pc_expr, is_quotenode_egal, + linetable, codelocs, LineTypes, is_GotoIfNot, isassign, isidentical +using ..LoweredCodeUtils: next_or_nothing!, trackedheads, callee_matches +include("../../Revise/src/packagedef.jl") +end + +# module DebugAdapter +# import ..JuliaInterpreter +# import ..JSON +# import ..JSONRPC +# import ..JSONRPC: @dict_readable, Outbound + +# include("../../DebugAdapter/src/packagedef.jl") +# end diff --git a/packages/TestItemServer/src/testserver_protocol.jl b/packages/TestItemServer/src/testserver_protocol.jl new file mode 100644 index 0000000..46a1f23 --- /dev/null +++ b/packages/TestItemServer/src/testserver_protocol.jl @@ -0,0 +1,60 @@ +@dict_readable struct Position + line::Int + character::Int +end + +struct Range + start::Position + stop::Position +end +function Range(d::Dict) + Range(Position(d["start"]), Position(d["end"])) +end +function JSON.lower(a::Range) + Dict("start" => a.start, "end" => a.stop) +end + +JSONRPC.@dict_readable struct Location + uri::String + range::Range +end + +JSONRPC.@dict_readable struct TestMessage + message::String + # expectedOutput?: string; + # actualOutput?: string; + location::Union{Missing,Location} +end + +JSONRPC.@dict_readable struct TestserverRunTestitemRequestParams <: JSONRPC.Outbound + uri::String + name::String + packageName::String + useDefaultUsings::Bool + testsetups::Vector{String} + line::Int + column::Int + code::String +end + +JSONRPC.@dict_readable struct TestserverRunTestitemRequestParamsReturn <: JSONRPC.Outbound + status::String + message::Union{Vector{TestMessage},Missing} + duration::Union{Float64,Missing} +end + +JSONRPC.@dict_readable struct TestsetupDetails <: JSONRPC.Outbound + name::String + uri::String + line::Int + column::Int + code::String +end + +JSONRPC.@dict_readable struct TestserverUpdateTestsetupsRequestParams <: JSONRPC.Outbound + testsetups::Vector{TestsetupDetails} +end + +const testserver_revise_request_type = JSONRPC.RequestType("testserver/revise", Nothing, String) +const testserver_run_testitem_request_type = JSONRPC.RequestType("testserver/runtestitem", TestserverRunTestitemRequestParams, TestserverRunTestitemRequestParamsReturn) +const testserver_update_testsetups_type = JSONRPC.RequestType("testserver/updateTestsetups", TestserverUpdateTestsetupsRequestParams, Nothing) diff --git a/packages/URIParser/.appveyor.yml b/packages/URIParser/.appveyor.yml new file mode 100644 index 0000000..6e78c30 --- /dev/null +++ b/packages/URIParser/.appveyor.yml @@ -0,0 +1,45 @@ +environment: + matrix: + - julia_version: 0.7 + - julia_version: 1.0 + - julia_version: 1.1 + - julia_version: 1.2 + - julia_version: 1.3 + - julia_version: 1.4 + - julia_version: nightly + +platform: + - x86 # 32-bit + - x64 # 64-bit + +matrix: + allow_failures: + - julia_version: nightly + +branches: + only: + - master + - /release-.*/ + +notifications: + - provider: Email + on_build_success: false + on_build_failure: false + on_build_status_changed: false + +install: + - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) + +build_script: + - echo "%JL_BUILD_SCRIPT%" + - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" + +test_script: + - echo "%JL_TEST_SCRIPT%" + - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" + +# # Uncomment to support code coverage upload. Should only be enabled for packages +# # which would have coverage gaps without running on Windows +# on_success: +# - echo "%JL_CODECOV_SCRIPT%" +# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" diff --git a/packages/URIParser/.github/workflows/TagBot.yml b/packages/URIParser/.github/workflows/TagBot.yml new file mode 100644 index 0000000..d77d3a0 --- /dev/null +++ b/packages/URIParser/.github/workflows/TagBot.yml @@ -0,0 +1,11 @@ +name: TagBot +on: + schedule: + - cron: 0 * * * * +jobs: + TagBot: + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/packages/URIParser/.gitignore b/packages/URIParser/.gitignore new file mode 100644 index 0000000..cf63da8 --- /dev/null +++ b/packages/URIParser/.gitignore @@ -0,0 +1,3 @@ +*.cov +.DS_Store +.vscode diff --git a/packages/URIParser/.travis.yml b/packages/URIParser/.travis.yml new file mode 100644 index 0000000..db118ac --- /dev/null +++ b/packages/URIParser/.travis.yml @@ -0,0 +1,18 @@ +language: julia +os: + - linux + - osx +julia: + - 0.7 + - 1.0 + - 1.1 + - 1.2 + - 1.3 + - 1.4 + - nightly +matrix: + allow_failures: + - julia: nightly +notifications: + email: false +codecov: true diff --git a/packages/URIParser/LICENSE.md b/packages/URIParser/LICENSE.md new file mode 100644 index 0000000..eb75778 --- /dev/null +++ b/packages/URIParser/LICENSE.md @@ -0,0 +1,23 @@ +URIParser.jl is licensed under the MIT License: + +> Copyright (c) 2013: Massachusetts Institute of Technology +> Copyright (c) 2013: Keno Fischer and other contributors. +> +> Permission is hereby granted, free of charge, to any person obtaining +> a copy of this software and associated documentation files (the +> "Software"), to deal in the Software without restriction, including +> without limitation the rights to use, copy, modify, merge, publish, +> distribute, sublicense, and/or sell copies of the Software, and to +> permit persons to whom the Software is furnished to do so, subject to +> the following conditions: +> +> The above copyright notice and this permission notice shall be +> included in all copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +> NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +> LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +> OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +> WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/packages/URIParser/Project.toml b/packages/URIParser/Project.toml new file mode 100644 index 0000000..d5c099f --- /dev/null +++ b/packages/URIParser/Project.toml @@ -0,0 +1,15 @@ +name = "URIParser" +uuid = "30578b45-9adc-5946-b283-645ec420af67" +version = "0.4.1" + +[deps] +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +julia = "0.7, 1" + +[targets] +test = ["Test"] diff --git a/packages/URIParser/README.md b/packages/URIParser/README.md new file mode 100644 index 0000000..d4b3250 --- /dev/null +++ b/packages/URIParser/README.md @@ -0,0 +1,82 @@ +# URIParser.jl + +This Julia package provides URI parsing according to [RFC 3986](http://tools.ietf.org/html/rfc3986). + +**This package is deprecated**. Please open new issues in [URIs.jl](https://github.com/JuliaWeb/URIs.jl). + +[![Build Status](https://travis-ci.org/JuliaWeb/URIParser.jl.svg?branch=master)](https://travis-ci.org/JuliaWeb/URIParser.jl) +[![codecov.io](http://codecov.io/github/JuliaWeb/URIParser.jl/coverage.svg?branch=master)](http://codecov.io/github/JuliaWeb/URIParser.jl?branch=master) + +[![URIParser](http://pkg.julialang.org/badges/URIParser_0.3.svg)](http://pkg.julialang.org/?pkg=URIParser&ver=0.3) +[![URIParser](http://pkg.julialang.org/badges/URIParser_0.4.svg)](http://pkg.julialang.org/?pkg=URIParser&ver=0.4) + +The main interaction with the package is through the `URI` constructor, which takes a string argument, e.g. + +```julia +julia> using URIParser + +julia> URI("hdfs://user:password@hdfshost:9000/root/folder/file.csv") +URI(hdfs://user:password@hdfshost:9000/root/folder/file.csv) + +julia> URI("https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag") +URI(https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag) + +julia> URI("news:comp.infosystems.www.servers.unix") +URI(news:comp.infosystems.www.servers.unix) +``` + +Additionally, there is a method for taking the parts of the URI individually, as well as a convenience method taking `host` and `path` which constructs a valid HTTP URL: + +```julia +julia> URI("hdfs","hdfshost",9000,"/root/folder/file.csv","","","user:password") +URI(hdfs://user:password@hdfshost:9000/root/folder/file.csv) + +julia> URI("google.com","/some/path") +URI(http://google.com:80/some/path) +``` + +Afterwards, you may either pass the API struct directly to another package (probably the more common use case) or extract parts of the URI as follows: + +```julia +julia> uri = URI("https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag") +URI(https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag) + +julia> uri.scheme +"https" + +julia> uri.host +"httphost" + +julia> dec(uri.port) +"9000" + +julia> uri.path +"/path1/path2;paramstring" + +julia> uri.query +"q=a&p=r" + +julia> uri.fragment +"frag" + +julia> uri.specifies_authority +true +``` + +The `specifies_authority` may need some extra explanation. The reson for its existence is that RFC 3986 differentiates between empty authorities and missing authorities, but there is not way to distinguish these by just looking at the fields. As an example: + +```julia +julia> URI("file:///a/b/c").specifies_authority +true + +julia> URI("file:///a/b/c").host +"" + +julia> URI("file:a/b/c").specifies_authority +false + +julia> URI("file:a/b/c").host +"" +``` + +Now, while the `file` scheme consideres these to be equivalent, this may not necessarily be true for all schemes and thus the distinction is necessary. diff --git a/packages/URIParser/src/URIParser.jl b/packages/URIParser/src/URIParser.jl new file mode 100644 index 0000000..f71a147 --- /dev/null +++ b/packages/URIParser/src/URIParser.jl @@ -0,0 +1,18 @@ +module URIParser + +export URI +export defrag, userinfo, path_params, query_params, isvalid +export escape, escape_form, escape_with, unescape, unescape_form + +using Unicode +using Base.Unicode: isascii + +import Base: isequal, isvalid, show, print, (==) + +include("parser.jl") +include("esc.jl") +include("utils.jl") +include("precompile.jl") +_precompile_() + +end # module diff --git a/packages/URIParser/src/esc.jl b/packages/URIParser/src/esc.jl new file mode 100644 index 0000000..6106b5e --- /dev/null +++ b/packages/URIParser/src/esc.jl @@ -0,0 +1,64 @@ +const escaped_regex = r"%([0-9a-fA-F]{2})" + +# Escaping +const control_array = vcat(map(UInt8, 0:parse(Int, "1f", base=16))) +const control = String(control_array)*"\x7f" +const space = String(" ") +const delims = String("%<>\"") +const unwise = String("(){}|\\^`") + +const reserved = String(",;/?:@&=+\$![]'*#") +# Strings to be escaped +# (Delims goes first so '%' gets escaped first.) +const unescaped = delims * reserved * control * space * unwise +const unescaped_form = delims * reserved * control * unwise + + +function unescape(str) + r = UInt8[] + l = length(str) + i = 1 + while i <= l + c = str[i] + i += 1 + if c == '%' + c = parse(UInt8, str[i:i+1], base=16) + i += 2 + end + push!(r, c) + end + return String(r) +end +unescape_form(str) = unescape(replace(str, "+" => " ")) + +hex_string(x) = string('%', uppercase(string(x, base=16, pad=2))) + +# Escapes chars (in second string); also escapes all non-ASCII chars. +function escape_with(str, use) + str = String(str) + out = IOBuffer() + chars = Set(use) + i = firstindex(str) + e = lastindex(str) + while i <= e + i_next = nextind(str, i) + if i_next == i + 1 + _char = str[i] + if _char in chars + write(out, hex_string(Int(_char))) + else + write(out, _char) + end + else + while i < i_next + write(out, hex_string(codeunits(str)[i])) + i += 1 + end + end + i = i_next + end + String(take!(out)) +end + +escape(str) = escape_with(str, unescaped) +escape_form(str) = replace(escape_with(str, unescaped_form), " " => "+") diff --git a/packages/URIParser/src/parser.jl b/packages/URIParser/src/parser.jl new file mode 100644 index 0000000..91d82aa --- /dev/null +++ b/packages/URIParser/src/parser.jl @@ -0,0 +1,360 @@ +isalnum(c) = isletter(c) || isnumeric(c) + +is_url_char(c) = ((@assert UInt32(c) < 0x80); 'A' <= c <= '~' || '$' <= c <= '>' || c == '\f' || c == '\t') +is_mark(c) = (c == '-') || (c == '_') || (c == '.') || (c == '!') || (c == '~') || + (c == '*') || (c == '\'') || (c == '(') || (c == ')') +is_userinfo_char(c) = isalnum(c) || is_mark(c) || (c == '%') || (c == ';') || + (c == ':') || (c == '&') || (c == '+') || (c == '$' || c == ',') +isnum(c) = ('0' <= c <= '9') +ishex(c) = (isnum(c) || 'a' <= lowercase(c) <= 'f') +is_host_char(c) = isalnum(c) || (c == '.') || (c == '-') || (c == '_') || (c == "~") + + +struct URI + scheme::String + host::String + port::UInt16 + path::String + query::String + fragment::String + userinfo::String + specifies_authority::Bool + URI(scheme,host,port,path,query="",fragment="",userinfo="",specifies_authority=false) = + new(scheme,host,UInt16(port),path,query,fragment,userinfo,specifies_authority) +end + +==(a::URI,b::URI) = isequal(a,b) +isequal(a::URI,b::URI) = (a.scheme == b.scheme) && + (a.host == b.host) && + (a.port == b.port) && + (a.path == b.path) && + (a.query == b.query) && + (a.fragment == b.fragment) && + (a.userinfo == b.userinfo) + +URI(host,path) = URI("http",host,UInt16(80),path,"","","",true) +URI(uri::URI; + scheme=uri.scheme, + host=uri.host, + port=uri.port, + path=uri.path, + query=uri.query, + fragment=uri.fragment, + userinfo=uri.userinfo, + specifies_authority=uri.specifies_authority) = +URI( scheme, host, port, path, query, fragment, userinfo, specifies_authority) + + +# URL parser based on the http-parser package by Joyent +# Licensed under the BSD license + +# Parse authority (user@host:port) +# return (host,port,user) +function parse_authority(authority,seen_at) + host="" + port="" + user="" + last_state = state = seen_at ? :http_userinfo_start : :http_host_start + i = firstindex(authority) + li = s = 0 + while true + if li > ncodeunits(authority) + last_state = state + state = :done + end + + if s == 0 + s = li + end + + if state != last_state + r = s:prevind(authority,li) + s = li + if last_state == :http_userinfo + user = authority[r] + elseif last_state == :http_host || last_state == :http_host_v6 + host = authority[r] + elseif last_state == :http_host_port + port = authority[r] + end + end + + if state == :done + break + end + + if i > ncodeunits(authority) + li = i + continue + end + + li = i + (ch,i) = iterate(authority,i) + + last_state = state + if state == :http_userinfo || state == :http_userinfo_start + if ch == '@' + state = :http_host_start + elseif is_userinfo_char(ch) + state = :http_userinfo + else + error("Unexpected character '$ch' in userinfo") + end + elseif state == :http_host_start + if ch == '[' + state = :http_host_v6_start + elseif ch == '%' + pos_escape_char2 = nextind(authority, li, 2) + if pos_escape_char2 > ncodeunits(authority) + error("Invalid escape sequence in host") + end + + pos_escape_char1 = nextind(authority, i) + + if !ishex(authority[pos_escape_char1]) || !ishex(authority[pos_escape_char2]) + error("Invalid escape sequence in host") + end + state = :http_host + elseif is_host_char(ch) + state = :http_host + else + error("Unexpected character '$ch' at the beginning of the host string") + end + elseif state == :http_host + if ch == ':' + state = :http_host_port_start + elseif ch == '%' + pos_escape_char2 = nextind(authority, li, 2) + if pos_escape_char2 > ncodeunits(authority) + error("Invalid escape sequence in host") + end + + pos_escape_char1 = nextind(authority, i) + + if ishex(authority[pos_escape_char1]) && ishex(authority[pos_escape_char2]) + li = pos_escape_char2 + (ch,i) = iterate(authority,li) + else + error("Invalid escape sequence in host") + end + elseif !is_host_char(ch) + error("Unexpected character '$ch' in host") + end + elseif state == :http_host_v6_end + if ch != ':' + error("Only port allowed in authority after IPv6 address") + end + state = :http_host_port_start + elseif state == :http_host_v6 || state == :http_host_v6_start + if ch == ']' && state == :http_host_v6 + state = :http_host_v6_end + elseif ishex(ch) || ch == ':' || ch == '.' + state = :http_host_v6 + else + error("Unrecognized character in IPv6 address") + end + elseif state == :http_host_port || state == :http_host_port_start + if !isnum(ch) + error("Port must be numeric (decimal)") + end + state = :http_host_port + else + error("Unexpected state $state") + end + end + (host, UInt16(port == "" ? 0 : parse(Int, port, base=10)), user) +end + +function parse_url(url) + scheme = "" + host = "" + server = "" + port = 80 + query = "" + fragment = "" + username = "" + pass = "" + path = "/" + last_state = state = :req_spaces_before_url + seen_at = false + specifies_authority = false + + i = firstindex(url) + li = s = 0 + while true + if li > ncodeunits(url) + last_state = state + state = :done + end + + if s == 0 + s = li + end + + if state != last_state + r = s:prevind(url,li) + s = li + if last_state == :req_scheme + scheme = url[r] + elseif last_state == :req_server_start + specifies_authority = true + elseif last_state == :req_server + server = url[r] + elseif last_state == :req_query_string + query = url[r] + elseif last_state == :req_path + path = url[r] + elseif last_state == :req_fragment + fragment = url[r] + end + end + + if state == :done + break + end + + if i > ncodeunits(url) + li = i + continue + end + + li = i + (ch,i) = iterate(url,i) + + if !isascii(ch) + error("Non-ASCII characters not supported in URIs. Encode the URL and try again.") + end + + last_state = state + + if state == :req_spaces_before_url + if ch == '/' || ch == '*' + state = :req_path + elseif isletter(ch) + state = :req_scheme + else + error("Unexpected start of URL") + end + elseif state == :req_scheme + if ch == ':' + state = :req_scheme_slash + elseif !(isletter(ch) || isdigit(ch) || ch == '+' || ch == '-' || ch == '.') + error("Unexpected character $ch after scheme") + end + elseif state == :req_scheme_slash + if ch == '/' + state = :req_scheme_slash_slash + elseif is_url_char(ch) + state = :req_path + else + error("Expecting scheme:path scheme:/path format not scheme:$ch") + end + elseif state == :req_scheme_slash_slash + if ch == '/' + state = :req_server_start + elseif is_url_char(ch) + s -= 1 + state = :req_path + else + error("Expecting scheme:// or scheme: format not scheme:/$ch") + end + elseif state == :req_server_start || state == :req_server + # In accordence with RFC3986: + # 'The authority component is preceded by a double slash ("//") and isterminated by the next slash ("/")' + # This is different from the joyent http-parser, which considers empty hosts to be invalid. c.f. also the + # following part of RFC 3986: + # "If the URI scheme defines a default for host, then that default + # applies when the host subcomponent is undefined or when the + # registered name is empty (zero length). For example, the "file" URI + # scheme is defined so that no authority, an empty host, and + # "localhost" all mean the end-user's machine, whereas the "http" + # scheme considers a missing authority or empty host invalid." + if ch == '/' + state = :req_path + elseif ch == '?' + state = :req_query_string_start + elseif ch == '@' + seen_at = true + state = :req_server + elseif is_userinfo_char(ch) || ch == '[' || ch == ']' + state = :req_server + else + error("Unexpected character $ch in server") + end + elseif state == :req_path + if ch == '?' + state = :req_query_string_start + elseif ch == '#' + state = :req_fragment_start + elseif !is_url_char(ch) && ch != '@' + error("Path contained unexpected character") + end + elseif state == :req_query_string_start || state == :req_query_string + if ch == '?' + state = :req_query_string + elseif ch == '#' + state = :req_fragment_start + elseif !is_url_char(ch) + error("Query string contained unexpected character") + else + state = :req_query_string + end + elseif state == :req_fragment_start + if ch == '?' + state = :req_fragment + elseif ch == '#' + state = :req_fragment_start + elseif ch != '#' && !is_url_char(ch) + error("Start of fragment contained unexpected character") + else + state = :req_fragment + end + elseif state == :req_fragment + if !is_url_char(ch) && ch != '?' && ch != '#' + error("Fragment contained unexpected character") + end + else + error("Unrecognized state") + end + end + host, port, user = parse_authority(server,seen_at) + URI(lowercase(scheme),host,port,path,query,fragment,user,specifies_authority) +end + +URI(url) = parse_url(url) + +show(io::IO, uri::URI) = print(io,"URI(",uri,")") + +function print(io::IO, uri::URI) + if uri.specifies_authority || !isempty(uri.host) + print(io,uri.scheme,"://") + if !isempty(uri.userinfo) + print(io,uri.userinfo,'@') + end + if ':' in uri.host #is IPv6 + print(io,'[',uri.host,']') + else + print(io,uri.host) + end + if uri.port != 0 + print(io,':',Int(uri.port)) + end + else + print(io,uri.scheme,":") + end + print(io,uri.path) + if !isempty(uri.query) + print(io,"?",uri.query) + end + if !isempty(uri.fragment) + print(io,"#",uri.fragment) + end +end + +function show(io::IO, ::MIME"text/html", uri::URI) + print(io, "") + print(io, uri) + print(io, "") +end diff --git a/packages/URIParser/src/precompile.jl b/packages/URIParser/src/precompile.jl new file mode 100644 index 0000000..726672f --- /dev/null +++ b/packages/URIParser/src/precompile.jl @@ -0,0 +1,13 @@ +function _precompile_() + ccall(:jl_generating_output, Cint, ()) == 1 || return nothing + precompile(URIParser.parse_url, (String,)) + precompile(URIParser.parse_authority, (String, Bool,)) + precompile(URIParser.escape_with, (String, String,)) + precompile(URIParser.is_host_char, (Char,)) + precompile(URIParser.is_url_char, (Char,)) + precompile(URIParser.isnum, (Char,)) + precompile(URIParser.is_mark, (Char,)) + precompile(URIParser.is_userinfo_char, (Char,)) + precompile(URIParser.escape, (String,)) + precompile(URIParser.ishex, (Char,)) +end diff --git a/packages/URIParser/src/utils.jl b/packages/URIParser/src/utils.jl new file mode 100644 index 0000000..ce04cd1 --- /dev/null +++ b/packages/URIParser/src/utils.jl @@ -0,0 +1,73 @@ +## +# Splits the userinfo portion of an URI in the format user:password and +# returns the components as tuple. +# +# Note: This is just a convenience method, and this form of usage is +# deprecated as of rfc3986. +# See: http://tools.ietf.org/html/rfc3986#section-3.2.1 +function userinfo(uri::URI) + @warn("Use of the format user:password is deprecated (rfc3986)") + uinfo = uri.userinfo + sep = findfirst(isequal(':'), uinfo) + l = length(uinfo) + username = uinfo[1:(sep-1)] + password = ((sep == l) || (sep == 0)) ? "" : uinfo[(sep+1):l] + (username, password) +end + +## +# Splits the path into components and parameters +# See: http://tools.ietf.org/html/rfc3986#section-3.3 +function path_params(uri::URI, seps=[';',',','=']) + elems = split(uri.path, '/', keepempty = false) + p = Array[] + for elem in elems + pp = split(elem, seps) + push!(p, pp) + end + p +end + +## +# Splits the query into key value pairs +function query_params(query::AbstractString) + elems = split(query, '&', keepempty = false) + d = Dict{AbstractString, AbstractString}() + for elem in elems + pp = split(elem, "=", keepempty = true) + if length(pp) == 2 + d[unescape(pp[1])] = unescape(pp[2]) + else + d[elem] = "" #gracefully degrade for ill formed query params + end + end + d +end +query_params(uri::URI) = query_params(uri.query::AbstractString) + + +## +# Create equivalent URI without the fragment +defrag(uri::URI) = URI(uri.scheme, uri.host, uri.port, uri.path, uri.query, "", uri.userinfo, uri.specifies_authority) + +## +# Validate known URI formats +const uses_authority = ["hdfs", "ftp", "http", "gopher", "nntp", "telnet", "imap", "wais", "file", "mms", "https", "shttp", "snews", "prospero", "rtsp", "rtspu", "rsync", "svn", "svn+ssh", "sftp" ,"nfs", "git", "git+ssh", "ldap"] +const uses_params = ["ftp", "hdl", "prospero", "http", "imap", "https", "shttp", "rtsp", "rtspu", "sip", "sips", "mms", "sftp", "tel"] +const non_hierarchical = ["gopher", "hdl", "mailto", "news", "telnet", "wais", "imap", "snews", "sip", "sips"] +const uses_query = ["http", "wais", "imap", "https", "shttp", "mms", "gopher", "rtsp", "rtspu", "sip", "sips", "ldap"] +const uses_fragment = ["hdfs", "ftp", "hdl", "http", "gopher", "news", "nntp", "wais", "https", "shttp", "snews", "file", "prospero"] + +function isvalid(uri::URI) + scheme = uri.scheme + isempty(scheme) && error("Can not validate relative URI") + slash = findfirst(isequal('/'), uri.path) + hasslash = slash !== nothing && slash > 1 # For 0.6 compatibility + if ((scheme in non_hierarchical) && hasslash) || # path hierarchy not allowed + (!(scheme in uses_query) && !isempty(uri.query)) || # query component not allowed + (!(scheme in uses_fragment) && !isempty(uri.fragment)) || # fragment identifier component not allowed + (!(scheme in uses_authority) && (!isempty(uri.host) || (0 != uri.port) || !isempty(uri.userinfo))) # authority component not allowed + return false + end + true +end diff --git a/packages/URIParser/test/runtests.jl b/packages/URIParser/test/runtests.jl new file mode 100755 index 0000000..19eeff0 --- /dev/null +++ b/packages/URIParser/test/runtests.jl @@ -0,0 +1,78 @@ +using URIParser +using Test + +urls = ["hdfs://user:password@hdfshost:9000/root/folder/file.csv#frag", + "https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag", + "https://user:password@httphost:9000/path1/path2?q=a&p=r#frag", + "https://user:password@httphost:9000/path1/path2;paramstring#frag", + "https://user:password@httphost:9000/path1/path2#frag", + "file:///path/to/file/with%3fshould%3dwork%23fine", + "ftp://ftp.is.co.za/rfc/rfc1808.txt", "http://www.ietf.org/rfc/rfc2396.txt", + "ldap://[2001:db8::7]/c=GB?objectClass?one", "mailto:John.Doe@example.com", + "news:comp.infosystems.www.servers.unix", "tel:+1-816-555-1212", "telnet://192.0.2.16:80/", + "urn:oasis:names:specification:docbook:dtd:xml:4.1.2"] + +failed = 0 +for url in urls + global failed + u = URI(url) + if !(string(u) == url) || !isvalid(u) + failed += 1 + println("Test failed for ",url) + end +end +if failed != 0 + exit(failed) +end + +@test URI("hdfs://user:password@hdfshost:9000/root/folder/file.csv") == URI("hdfs","hdfshost",9000,"/root/folder/file.csv","","","user:password") +@test URI("google.com","/some/path") == URI("http://google.com:80/some/path") +g = URI("google.com","/some/path") +@test URI(g,port=160) == URI("http://google.com:160/some/path") + +@test escape("abcdef αβ 1234-=~!@#\$()_+{}|[]a;") == "abcdef%20%CE%B1%CE%B2%201234-%3D~%21%40%23%24%28%29_%2B%7B%7D%7C%5B%5Da%3B" +@test unescape(escape("abcdef 1234-=~!@#\$()_+{}|[]a;")) == "abcdef 1234-=~!@#\$()_+{}|[]a;" +@test unescape(escape("👽")) == "👽" + +@test escape_form("abcdef 1234-=~!@#\$()_+{}|[]a;") == "abcdef+1234-%3D~%21%40%23%24%28%29_%2B%7B%7D%7C%5B%5Da%3B" +@test unescape_form(escape_form("abcdef 1234-=~!@#\$()_+{}|[]a;")) == "abcdef 1234-=~!@#\$()_+{}|[]a;" + +@test ("user", "password") == userinfo(URI("https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag")) +@test URI("https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r") == defrag(URI("https://user:password@httphost:9000/path1/path2;paramstring?q=a&p=r#frag")) + +@test ["dc","example","dc","com"] == path_params(URI("ldap://ldap.example.com/dc=example,dc=com"))[1] +@test ["servlet","jsessionid","OI24B9ASD7BSSD"] == path_params(URI("http://www.mysite.com/servlet;jsessionid=OI24B9ASD7BSSD"))[1] + +@test Dict("q"=>"a","p"=>"r") == query_params(URI("https://httphost/path1/path2;paramstring?q=a&p=r#frag")) +@test Dict("q"=>"a","malformed"=>"") == query_params(URI("https://foo.net/?q=a&malformed")) + +@test false == isvalid(URI("file:///path/to/file/with?should=work#fine")) +@test true == isvalid(URI("file:///path/to/file/with%3fshould%3dwork%23fine")) + +@test URI("s3://bucket/key") == URI("s3","bucket",0,"/key") + +@test sprint(show, URI("http://google.com")) == "URI(http://google.com/)" + +# Error paths +# Non-ASCII characters +@test_throws ErrorException URI("http://🍕.com") +# Unexpected start of URL +@test_throws ErrorException URI(".google.com") +# Unexpected character after scheme +@test_throws ErrorException URI("ht!tp://google.com") + +# Issue #27 +@test URIParser.escape("t est\n") == "t%20est%0A" + +# Issue #2 +@test sprint(show, MIME("text/html"), URI("http://google.com")) == + """http://google.com/""" + +@test URI("file://wsl%24/Ubuntu-18.04/foo/bar") == URI("file", "wsl%24", 0, "/Ubuntu-18.04/foo/bar") + +@test URI("file://wsl%24more/Ubuntu-18.04/foo/bar") == URI("file", "wsl%24more", 0, "/Ubuntu-18.04/foo/bar") + +@test URI("gitlens://%28deleted%29/foo/bar") == URI("gitlens", "%28deleted%29", 0, "/foo/bar") + +# Test that an invalid escape sequence throws an error +@test_throws ErrorException URI("file://wsl%2more/Ubuntu-18.04/foo/bar") diff --git a/src/TestItemRunner2.jl b/src/TestItemRunner2.jl new file mode 100644 index 0000000..671a627 --- /dev/null +++ b/src/TestItemRunner2.jl @@ -0,0 +1,345 @@ +module TestItemRunner2 + +export run_tests, kill_test_processes + +# For easier dev, switch these two lines +const pkg_root = "../packages" +# const pkg_root = joinpath(homedir(), ".julia", "dev") + +import JSON, JSONRPC, ProgressMeter, TOML, UUIDs, Sockets, JuliaWorkspaces + +using JSONRPC: @dict_readable +using JuliaWorkspaces: JuliaWorkspace, get_text +using JuliaWorkspaces.URIs2: URI, filepath2uri, uri2filepath + +include("vendored_code.jl") + +include(joinpath(pkg_root, "TestItemServer", "src", "testserver_protocol.jl")) + +mutable struct TestProcess + key + process + connection + current_testitem + log_out + log_err +end + +const TEST_PROCESSES = Dict{NamedTuple{(:project_uri,:package_uri,:package_name),Tuple{Union{URI,Nothing},URI,String}},Vector{TestProcess}}() +const SOME_TESTITEM_FINISHED = Base.Event(true) + +function get_key_from_testitem(testitem) + return ( + project_uri = testitem.detail.project_uri, + package_uri = testitem.detail.package_uri, + package_name = testitem.detail.package_name + ) +end + +function launch_new_process(testitem) + key = get_key_from_testitem(testitem) + + pipe_name = generate_pipe_name("tir", UUIDs.uuid4()) + + server = Sockets.listen(pipe_name) + + testserver_script = joinpath(@__DIR__, "testserver_main.jl") + + buffer_out = IOBuffer() + buffer_err = IOBuffer() + + jl_process = open( + pipeline( + Cmd(`$(Base.julia_cmd()) --startup-file=no --history-file=no --depwarn=no $testserver_script $pipe_name $(key.project_uri===nothing ? "" : uri2filepath(key.project_uri)) $(uri2filepath(key.package_uri)) $(key.package_name)`), + stdout = buffer_out, + stderr = buffer_err + ) + ) + + socket = Sockets.accept(server) + + connection = JSONRPC.JSONRPCEndpoint(socket, socket) + + run(connection) + + test_process = TestProcess(key, jl_process, connection, nothing, buffer_out, buffer_err) + + if !haskey(TEST_PROCESSES, key) + TEST_PROCESSES[key] = TestProcess[] + end + + test_processes = TEST_PROCESSES[key] + + push!(test_processes, test_process) + + @async try + wait(jl_process) + + i = findfirst(isequal(test_process), test_processes) + + popat!(test_processes, i) + catch err + Base.display_error(err, catch_backtrace()) + end + + return test_process +end + +function isconnected(testprocess) + # TODO Properly implement + true +end + +function isbusy(testprocess) + return testprocess.current_testitem!==nothing +end + +function run_revise(testprocess) + return JSONRPC.send(testprocess.connection,testserver_revise_request_type, nothing) +end + +function get_free_testprocess(testitem, max_num_processes) + key = get_key_from_testitem(testitem) + + if !haskey(TEST_PROCESSES, key) + return launch_new_process(testitem) + else + test_processes = TEST_PROCESSES[key] + + # TODO add some way to cancel + while true + + # First lets just see whether we have an idle test process we can use + for test_process in test_processes + if !isbusy(test_process) + needs_new_process = false + + if !isconnected(test_process) + needs_new_process = true + else + status = run_revise(test_process) + + if status != "success" + terminate(test_process) + + needs_new_process = true + end + end + + if needs_new_process + test_process = launch_new_process(testitem) + end + + return test_process + end + end + + if length(test_processes) < max_num_processes + return launch_new_process(testitem) + else + wait(SOME_TESTITEM_FINISHED) + end + end + end +end + +function execute_test(test_process, testitem, testsetups, timeout) + + test_process.current_testitem = testitem + + return_value = Channel(1) + + finished = false + + timer = timeout>0 ? Timer(timeout) do i + if !finished + kill(test_process.process) + end + end : nothing + + @async try + JSONRPC.send( + test_process.connection, + testserver_update_testsetups_type, + TestserverUpdateTestsetupsRequestParams([TestsetupDetails( + string(k), + string(v.detail.uri), + # TODO use proper location info here + 1, + 1, + v.code + ) for (k,v) in testsetups]) + ) + + result = JSONRPC.send( + test_process.connection, + testserver_run_testitem_request_type, + TestserverRunTestitemRequestParams( + string(testitem.detail.uri), + testitem.detail.name, + testitem.detail.package_name, + testitem.detail.option_default_imports, + convert(Vector{String}, string.(testitem.detail.option_setup)), + # TODO use proper location info here + 1, #pos.line, + 1, #pos.column, + testitem.code + ) + ) + + out_log = String(take!(test_process.log_out)) + err_log = String(take!(test_process.log_err)) + + finished = true + + timer === nothing || close(timer) + + test_process.current_testitem = nothing + + notify(SOME_TESTITEM_FINISHED) + + push!(return_value, (status = result.status, message = result.message, duration = result.duration, log_out = out_log, log_err = err_log)) + catch err + if err isa InvalidStateException + + try + out_log = String(take!(test_process.log_out)) + err_log = String(take!(test_process.log_err)) + + notify(SOME_TESTITEM_FINISHED) + + push!(return_value, (status="timeout", message="The test timed out", log_out = out_log, log_err = err_log)) + catch err2 + Base.display_error(err2, catch_backtrace()) + end + else + Base.display_error(err, catch_backtrace()) + end + end + + return return_value +end + +function run_tests(path; filter=nothing, verbose=false, max_workers::Int=Sys.CPU_THREADS, timeout=60*5, return_results=false, print_failed_results=true) + jw = JuliaWorkspace(Set([filepath2uri(path)])) + + if count(i -> true, Iterators.flatten(values(jw._testerrors))) > 0 + println("There are errors in your test definitions, we are aborting.") + + for te in Iterators.flatten(values(jw._testerrors)) + pos = JuliaWorkspaces.get_position_from_offset(jw._text_documents[te.uri], te.range[1]) + println() + println("File: $(uri2filepath(te.uri)):$(pos[1]+1)") + println() + println(te.message) + println() + end + + return nothing + end + + # testsetups maps @testsetup PACKAGE => NAME => TESTSETUPdetail + testsetups = Dict{JuliaWorkspaces.URIs2.URI,Dict{Symbol,Any}}() + for i in Iterators.flatten(values(jw._testsetups)) + testsetups_in_package = get!(() -> Dict{Symbol,Any}(), testsetups, i.package_uri) + + haskey(testsetups_in_package, i.name) && error("The name '$(i.name)' is used for more than one test setup.") + + testsetups_in_package[i.name] = (detail=i, code=get_text(jw._text_documents[i.uri])[i.code_range]) + end + + # Flat list of @testitems + testitems = [(detail=i, code=get_text(jw._text_documents[i.uri])[i.code_range]) for i in Iterators.flatten(values(jw._testitems))] + + # Filter @testitems + if filter !== nothing + filter!(i->filter((filename=uri2filepath(i.detail.uri), name=i.detail.name, tags=i.detail.option_tags, package_name=i.detail.package_name)), testitems) + end + + executed_testitems = [] + + p = ProgressMeter.Progress(length(testitems), barlen=50) + + count_success = 0 + count_timeout = 0 + count_fail = 0 + + # Loop over all test items that should be executed + for testitem in testitems + test_process = get_free_testprocess(testitem, max_workers) + + result_channel = execute_test(test_process, testitem, get(()->Dict{Symbol,Any}(), testsetups, testitem.detail.package_uri), timeout) + + progress_reported_channel = Channel(1) + + @async try + res = fetch(result_channel) + + if res.status=="passed" + count_success += 1 + elseif res.status=="timeout" + count_timeout += 1 + elseif res.status == "failed" + count_fail += 1 + end + + ProgressMeter.next!( + p, + showvalues = [ + (Symbol("Successful tests"), count_success), + (Symbol("Failed tests"), count_fail), + (Symbol("Timed out tests"), count_timeout), + ((Symbol("Number of processes for package '$(i.first.package_name)'"), length(i.second)) for i in TEST_PROCESSES)... + ] + ) + push!(progress_reported_channel, true) + catch err + Base.display_error(err, catch_backtrace()) + end + + push!(executed_testitems, (testitem=testitem, result=result_channel, progress_reported_channel=progress_reported_channel)) + end + + yield() + + for i in executed_testitems + wait(i.result) + end + + responses = [(testitem=i.testitem, result=take!(i.result)) for i in executed_testitems] + + if print_failed_results + for i in responses + if i.result.status == "failed" && i.result.message!==missing + println() + println("Errors for test $(i.testitem.detail.name)") + for j in i.result.message + println(j.message) + end + println() + end + end + end + + for i in executed_testitems + wait(i.progress_reported_channel) + end + + println("$(length(responses)) tests ran, $(count_success) passed, $(count_fail) failed, $(count_timeout) timed out.") + + if return_results + return responses + else + return nothing + end +end + +function kill_test_processes() + for i in values(TEST_PROCESSES) + for j in i + kill(j.process) + end + end +end + +end diff --git a/src/testserver_main.jl b/src/testserver_main.jl new file mode 100644 index 0000000..e781ff0 --- /dev/null +++ b/src/testserver_main.jl @@ -0,0 +1,14 @@ +pushfirst!(LOAD_PATH, joinpath(@__DIR__, "..", "packages")) +using TestItemServer +popfirst!(LOAD_PATH) + +import Sockets + +try + conn = Sockets.connect(ARGS[1]) + + TestItemServer.serve(conn, ARGS[2], ARGS[3], ARGS[4]) + +catch err + Base.display(err) +end diff --git a/src/vendored_code.jl b/src/vendored_code.jl new file mode 100644 index 0000000..3a69d7c --- /dev/null +++ b/src/vendored_code.jl @@ -0,0 +1,31 @@ +function withpath(f, path) + tls = task_local_storage() + hassource = haskey(tls, :SOURCE_PATH) + hassource && (path′ = tls[:SOURCE_PATH]) + tls[:SOURCE_PATH] = path + try + return f() + finally + hassource ? (tls[:SOURCE_PATH] = path′) : delete!(tls, :SOURCE_PATH) + end +end + +function generate_pipe_name(part1, part2) + if Sys.iswindows() + return "\\\\.\\pipe\\$part1-$part2" + end + # Pipe names on unix may only be 92 chars (JuliaLang/julia#43281), and since + # tempdir can be arbitrary long (in particular on macos) we try to keep the name + # within bounds here. + pipename = joinpath(tempdir(), "$part1-$part2") + if length(pipename) >= 92 + # Try to use /tmp and if that fails, hope the long pipe name works anyway + maybe = "/tmp/$part1-$part2" + try + touch(maybe); rm(maybe) # Check permissions on this path + pipename = maybe + catch + end + end + return pipename +end