From 906838ce6835948138b1e86e3f0c59262840227f Mon Sep 17 00:00:00 2001 From: Sergio Alejandro Vargas Date: Mon, 1 Aug 2022 11:24:31 -0500 Subject: [PATCH 01/39] Format WorkspaceManager - Prefer quote blocks to colon quote expressions - Prefer if blocks to ternary operators - Remove single pipes - Minor changes to docstrings and comments --- src/evaluation/WorkspaceManager.jl | 232 +++++++++++++++++------------ 1 file changed, 134 insertions(+), 98 deletions(-) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index c556e97e15..bcd95db01c 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -8,7 +8,11 @@ import ..Pluto.ExpressionExplorer: FunctionName import ..PlutoRunner import Distributed -"Contains the Julia process (in the sense of `Distributed.addprocs`) to evaluate code in. Each notebook gets at most one `Workspace` at any time, but it can also have no `Workspace` (it cannot `eval` code in this case)." +""" +Contains the Julia process (in the sense of `Distributed.addprocs`) to evaluate code in. +Each notebook gets at most one `Workspace` at any time, but it can also have no `Workspace` +(it cannot `eval` code in this case). +""" Base.@kwdef mutable struct Workspace pid::Integer notebook_id::UUID @@ -22,21 +26,26 @@ Base.@kwdef mutable struct Workspace original_ACTIVE_PROJECT::Union{Nothing,String}=nothing end +const SN = Tuple{ServerSession, Notebook} + "These expressions get evaluated whenever a new `Workspace` process is created." -process_preamble() = quote +const process_preamble = quote ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0) include($(project_relative_path(joinpath("src", "runner"), "Loader.jl"))) ENV["GKSwstype"] = "nul" ENV["JULIA_REVISE_WORKER_ONLY"] = "1" end +const Distributed_expr = quote + Base.loaded_modules[Base.PkgId(Base.UUID("8ba89e20-285c-5b6f-9357-94700520ee1b"), "Distributed")] +end + const workspaces = Dict{UUID,Task}() + "Set of notebook IDs that we will never make a process for again." const discarded_workspaces = Set{UUID}() -const SN = Tuple{ServerSession,Notebook} - -"""Create a workspace for the notebook, optionally in the main process.""" +"Create a workspace for the notebook, optionally in the main process." function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false)::Workspace is_offline_renderer || (notebook.process_status = ProcessStatus.starting) @@ -62,18 +71,18 @@ function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false Distributed.remotecall_eval(Main, [pid], session.options.evaluation.workspace_custom_startup_expr) - Distributed.remotecall_eval(Main, [pid], :(PlutoRunner.notebook_id[] = $(notebook.notebook_id))) + Distributed.remotecall_eval(Main, [pid], quote + PlutoRunner.notebook_id[] = $(notebook.notebook_id) + end) remote_log_channel = Core.eval(Main, quote $(Distributed).RemoteChannel(() -> eval(quote - channel = Channel{Any}(10) Main.PlutoRunner.setup_plutologger( - $($(notebook.notebook_id)), - channel; + $($(notebook.notebook_id)), + channel; make_global=$($(use_distributed)) ) - channel end), $pid) end) @@ -81,6 +90,7 @@ function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false run_channel = Core.eval(Main, quote $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.run_channel)), $pid) end) + module_name = create_emptyworkspacemodule(pid) original_LOAD_PATH, original_ACTIVE_PROJECT = Distributed.remotecall_eval(Main, pid, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) @@ -108,14 +118,10 @@ end function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) enabled = notebook.nbpkg_ctx !== nothing - if workspace.nbpkg_was_active == enabled - return - end + workspace.nbpkg_was_active == enabled && return workspace = workspace !== nothing ? workspace : get_workspace((session, notebook)) - if workspace.discarded - return - end + workspace.discarded && return workspace.nbpkg_was_active = enabled if workspace.pid != Distributed.myid() @@ -127,7 +133,7 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) Base.ACTIVE_PROJECT[] = $(new_AP) end) else - # uhmmmmmm TODO + # TODO end end @@ -159,26 +165,24 @@ function start_relaying_logs((session, notebook)::SN, log_channel::Distributed.R fn = next_log["file"] match = findfirst("#==#", fn) - # We always show the log at the currently running cell, which is given by + # Show the log at the currently running cell, which is given by running_cell_id = next_log["cell_id"]::UUID # Some logs originate from outside of the running code, through function calls. Some code here to deal with that: begin source_cell_id = if match !== nothing - # the log originated from within the notebook - + # The log originated from within the notebook UUID(fn[match[end]+1:end]) else - # the log originated from a function call defined outside of the notebook - - # we will show the log at the currently running cell, at "line -1", i.e. without line info. + # The log originated from a function call defined outside of the notebook. + # Show the log at the currently running cell, at "line -1", i.e. without line info. next_log["line"] = -1 running_cell_id end if running_cell_id != source_cell_id - # the log originated from a function in another cell of the notebook - # we will show the log at the currently running cell, at "line -1", i.e. without line info. + # The log originated from a function in another cell of the notebook + # Show the log at the currently running cell, at "line -1", i.e. without line info. next_log["line"] = -1 end end @@ -226,7 +230,7 @@ end "Call `cd(\$path)` inside the workspace. This is done when creating a workspace, and whenever the notebook changes path." function cd_workspace(workspace, path::AbstractString) eval_in_workspace(workspace, quote - cd($(path |> dirname)) + cd(dirname($path)) end) end @@ -241,21 +245,18 @@ end function possible_bond_values(session_notebook::SN, n::Symbol; get_length::Bool=false) workspace = get_workspace(session_notebook) - pid = workspace.pid - Distributed.remotecall_eval(Main, pid, quote + Distributed.remotecall_eval(Main, workspace.pid, quote PlutoRunner.possible_bond_values($(QuoteNode(n)); get_length=$(get_length)) end) end function create_emptyworkspacemodule(pid::Integer)::Symbol - Distributed.remotecall_eval(Main, pid, :(PlutoRunner.increment_current_module())) + Distributed.remotecall_eval(Main, pid, quote + PlutoRunner.increment_current_module() + end) end -const Distributed_expr = :( - Base.loaded_modules[Base.PkgId(Base.UUID("8ba89e20-285c-5b6f-9357-94700520ee1b"), "Distributed")] -) - # NOTE: this function only start a worker process using given # compiler options, it does not resolve paths for notebooks # compiler configurations passed to it should be resolved before this @@ -266,15 +267,16 @@ function create_workspaceprocess(; compiler_options=CompilerOptions())::Integer $(Distributed_expr).addprocs(1; exeflags=$(_convert_to_flags(compiler_options))) |> first end) - Distributed.remotecall_eval(Main, [pid], process_preamble()) + Distributed.remotecall_eval(Main, [pid], process_preamble) # so that we NEVER break the workspace with an interrupt 🤕 - @async Distributed.remotecall_eval(Main, [pid], - :(while true + @async Distributed.remotecall_eval(Main, [pid], quote + while true try wait() catch end - end)) + end + end) pid end @@ -291,14 +293,15 @@ function get_workspace(session_notebook::SN; allow_creation::Bool=true)::Union{N error("Cannot run code in this notebook: it has already shut down.") end - task = !allow_creation ? - get(workspaces, notebook.notebook_id, nothing) : - get!(workspaces, notebook.notebook_id) do - Task(() -> make_workspace(session_notebook)) + task = if !allow_creation + get(workspaces, notebook.notebook_id, nothing) + else + get!(workspaces, notebook.notebook_id) do + Task(() -> make_workspace(session_notebook)) + end end isnothing(task) && return nothing - istaskstarted(task) || schedule(task) fetch(task) end @@ -345,15 +348,12 @@ function distributed_exception_result(ex::Base.IOError, workspace::Workspace) ) end - - function distributed_exception_result(exs::CompositeException, workspace::Workspace) - ex = exs.exceptions |> first + ex = first(exs.exceptions) if ex isa Distributed.RemoteException && - ex.pid == workspace.pid && - ex.captured.ex isa InterruptException - + ex.pid == workspace.pid && + ex.captured.ex isa InterruptException ( output_formatted=PlutoRunner.format_output(CapturedException(InterruptException(), [])), errored=true, @@ -388,9 +388,12 @@ function distributed_exception_result(exs::CompositeException, workspace::Worksp end -"Evaluate expression inside the workspace - output is fetched and formatted, errors are caught and formatted. Returns formatted output and error flags. +""" +Evaluate expression inside the workspace - output is fetched and formatted, +errors are caught and formatted. Returns formatted output and error flags. -`expr` has to satisfy `ExpressionExplorer.is_toplevel_expr`." +`expr` has to satisfy `ExpressionExplorer.is_toplevel_expr`. +""" function eval_format_fetch_in_workspace( session_notebook::Union{SN,Workspace}, expr::Expr, @@ -415,40 +418,46 @@ function eval_format_fetch_in_workspace( use_nbpkg_environment(session_notebook, workspace) end - # run the code 🏃‍♀️ + # Run the code 🏃 - # a try block (on this process) to catch an InterruptException + # A try block (on this process) to catch an InterruptException take!(workspace.dowork_token) early_result = try - # we use [pid] instead of pid to prevent fetching output - Distributed.remotecall_eval(Main, [workspace.pid], :(PlutoRunner.run_expression( - getfield(Main, $(QuoteNode(workspace.module_name))), - $(QuoteNode(expr)), - $(workspace.notebook_id), - $cell_id, - $function_wrapped_info, - $forced_expr_id; - user_requested_run=$user_requested_run, - capture_stdout=$(capture_stdout && !is_on_this_process), - ))) + # Use [pid] instead of pid to prevent fetching output + Distributed.remotecall_eval(Main, [workspace.pid], quote + PlutoRunner.run_expression( + getfield(Main, $(QuoteNode(workspace.module_name))), + $(QuoteNode(expr)), + $(workspace.notebook_id), + $cell_id, + $function_wrapped_info, + $forced_expr_id; + user_requested_run=$user_requested_run, + capture_stdout=$(capture_stdout && !is_on_this_process), + ) + end) put!(workspace.dowork_token) nothing - catch exs - # We don't use a `finally` because the token needs to be back asap for the interrupting code to pick it up. + catch e + # Don't use a `finally` because the token needs to be back asap for the interrupting code to pick it up. put!(workspace.dowork_token) - distributed_exception_result(exs, workspace) + distributed_exception_result(e, workspace) end - early_result === nothing ? - format_fetch_in_workspace(workspace, cell_id, ends_with_semicolon, known_published_objects) : - early_result + if early_result === nothing + format_fetch_in_workspace(workspace, cell_id, ends_with_semicolon, known_published_objects) + else + early_result + end end "Evaluate expression inside the workspace - output is not fetched, errors are rethrown. For internal use." function eval_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, [workspace.pid], :(Core.eval($(workspace.module_name), $(expr |> QuoteNode)))) + Distributed.remotecall_eval(Main, [workspace.pid], quote + Core.eval($(workspace.module_name), $(QuoteNode(expr))) + end) nothing end @@ -461,42 +470,42 @@ function format_fetch_in_workspace( )::PlutoRunner.FormattedCellResult workspace = get_workspace(session_notebook) - # instead of fetching the output value (which might not make sense in our context, since the user can define structs, types, functions, etc), we format the cell output on the worker, and fetch the formatted output. + # Instead of fetching the output value (which might not make sense in our context, + # since the user can define structs, types, functions, etc), + # we format the cell output on the worker, and fetch the formatted output. withtoken(workspace.dowork_token) do try - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.formatted_result_of( - $(workspace.notebook_id), - $cell_id, - $ends_with_semicolon, - $known_published_objects, - $showmore_id, - getfield(Main, $(QuoteNode(workspace.module_name))), - ))) - catch ex - distributed_exception_result(CompositeException([ex]), workspace) + Distributed.remotecall_eval(Main, workspace.pid, quote + PlutoRunner.formatted_result_of( + $(workspace.notebook_id), + $cell_id, + $ends_with_semicolon, + $known_published_objects, + $showmore_id, + getfield(Main, $(QuoteNode(workspace.module_name))), + ) + end) + catch e + distributed_exception_result(CompositeException([e]), workspace) end end end function collect_soft_definitions(session_notebook::SN, modules::Set{Expr}) workspace = get_workspace(session_notebook) - module_name = workspace.module_name - - ex = quote - PlutoRunner.collect_soft_definitions($module_name, $modules) - end - Distributed.remotecall_eval(Main, workspace.pid, ex) + Distributed.remotecall_eval(Main, workspace.pid, quote + PlutoRunner.collect_soft_definitions($(workspace.module_name), $modules) + end) end - function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macrocall, cell_id, module_name=nothing)::Tuple{Bool,Any} workspace = get_workspace(session_notebook) module_name = module_name === nothing ? workspace.module_name : module_name Distributed.remotecall_eval(Main, workspace.pid, quote try - (true, PlutoRunner.try_macroexpand($(module_name), $(workspace.notebook_id), $(cell_id), $(macrocall |> QuoteNode))) + (true, PlutoRunner.try_macroexpand($module_name, $(workspace.notebook_id), $cell_id, $(QuoteNode(macrocall)))) catch error # We have to be careful here, for example a thrown `MethodError()` will contain the called method and arguments. # which normally would be very useful for debugging, but we can't serialize it! @@ -514,21 +523,46 @@ end function eval_fetch_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, :(Core.eval($(workspace.module_name), $(expr |> QuoteNode)))) + Distributed.remotecall_eval(Main, workspace.pid, quote + Core.eval($(workspace.module_name), $(QuoteNode(expr))) + end) end function do_reimports(session_notebook::Union{SN,Workspace}, module_imports_to_move::Set{Expr}) workspace = get_workspace(session_notebook) - workspace_name = workspace.module_name - Distributed.remotecall_eval(Main, [workspace.pid], :(PlutoRunner.do_reimports($(workspace_name), $module_imports_to_move))) + + Distributed.remotecall_eval(Main, [workspace.pid], quote + PlutoRunner.do_reimports($(workspace.module_name), $module_imports_to_move) + end) end -"Move variables to a new module. A given set of variables to be 'deleted' will not be moved to the new module, making them unavailable. " -function move_vars(session_notebook::Union{SN,Workspace}, old_workspace_name::Symbol, new_workspace_name::Union{Nothing,Symbol}, to_delete::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,FunctionName}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}; kwargs...) +""" +Move variables to a new module. A given set of variables to be 'deleted' will +not be moved to the new module, making them unavailable. +""" +function move_vars( + session_notebook::Union{SN,Workspace}, + old_workspace_name::Symbol, + new_workspace_name::Union{Nothing,Symbol}, + to_delete::Set{Symbol}, + methods_to_delete::Set{Tuple{UUID,FunctionName}}, + module_imports_to_move::Set{Expr}, + invalidated_cell_uuids::Set{UUID}; + kwargs...) + workspace = get_workspace(session_notebook) new_workspace_name = something(new_workspace_name, workspace.module_name) - Distributed.remotecall_eval(Main, [workspace.pid], :(PlutoRunner.move_vars($(old_workspace_name |> QuoteNode), $(new_workspace_name |> QuoteNode), $to_delete, $methods_to_delete, $module_imports_to_move, $invalidated_cell_uuids))) + Distributed.remotecall_eval(Main, [workspace.pid], quote + PlutoRunner.move_vars( + $(QuoteNode(old_workspace_name)), + $(QuoteNode(new_workspace_name)), + $to_delete, + $methods_to_delete, + $module_imports_to_move, + $invalidated_cell_uuids, + ) + end) end move_vars(session_notebook::Union{SN,Workspace}, to_delete::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,FunctionName}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}; kwargs...) = @@ -605,10 +639,12 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true https://docs.microsoft.com/en-us/windows/wsl" return false end + if workspace.pid == Distributed.myid() verbose && @warn """Cells in this workspace can't be stopped, because it is not running in a separate workspace. Use `ENV["PLUTO_WORKSPACE_USE_DISTRIBUTED"]` to control whether future workspaces are generated in a separate process.""" return false end + if isready(workspace.dowork_token) verbose && @info "Tried to stop idle workspace - ignoring." return true @@ -642,10 +678,10 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true verbose && println() verbose && println("Cell interrupted!") true - catch ex - if !(ex isa KeyError) + catch e + if !(e isa KeyError) @warn "Interrupt failed for unknown reason" - showerror(ex, stacktrace(catch_backtrace())) + showerror(e, stacktrace(catch_backtrace())) end false end From b1463ddb2fc41ae00f6ecb0a6f5bfaabae18fb22 Mon Sep 17 00:00:00 2001 From: Sergio Alejandro Vargas Date: Sat, 3 Sep 2022 02:38:49 -0500 Subject: [PATCH 02/39] Replace Distributed with Malt - Remove `workspace_use_distributed` flag. Pluto will now always use separate processes. - Replace calls to `remotecall_eval` with the appropiate Malt calls. - Replace Distributed with Malt in tests to remove it from dependencies. - Add `unmake_workspace` calls in `MacroAnalysis` to prevent creating too many processes at the same time. - Replace Distributed exceptions with Malt exceptions. - Replace `pid::Integer` with `worker::Worker` in WorkspaceManager. - Rename `workspaces` to `active_workspaces`. - Rename `distributed_exception_result` to `workspace_exception_result`. - Rename `make_distributed_serializable` to `make_serializable`. --- Project.toml | 3 + src/Configuration.jl | 5 - src/evaluation/WorkspaceManager.jl | 185 ++++++++++------------------- src/runner/PlutoRunner.jl | 29 +++-- src/webserver/REPLTools.jl | 23 ++-- test/Bonds.jl | 19 ++- test/Configuration.jl | 8 +- test/DependencyCache.jl | 1 - test/Dynamic.jl | 1 - test/Events.jl | 3 +- test/Logging.jl | 1 - test/MacroAnalysis.jl | 68 +++++++++-- test/Notebook.jl | 3 - test/React.jl | 26 +--- test/ReloadFromFile.jl | 4 +- test/RichOutput.jl | 8 +- test/WorkspaceManager.jl | 13 +- test/cell_disabling.jl | 1 - test/compiletimes.jl | 3 +- test/helpers.jl | 16 +-- test/packages/Basic.jl | 39 +++--- test/runtests.jl | 15 --- test/webserver.jl | 5 +- 23 files changed, 208 insertions(+), 271 deletions(-) diff --git a/Project.toml b/Project.toml index 7736236318..771750ce4a 100644 --- a/Project.toml +++ b/Project.toml @@ -16,6 +16,7 @@ HypertextLiteral = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MIMEs = "6c6e2e6c-3030-632d-7369-2d6c69616d65" +Malt = "36869731-bdee-424d-aa32-cab38c994e3b" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" MsgPack = "99f44e22-a591-53d1-9472-aa23ef4bd671" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -23,6 +24,7 @@ PrecompileSignatures = "91cefc8d-f054-46dc-8f8c-26e11d7c5411" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" RegistryInstances = "2792f1a3-b283-48e8-9a74-f99dce5104f3" RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" @@ -35,6 +37,7 @@ FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.0.2" HypertextLiteral = "0.7, 0.8, 0.9" MIMEs = "0.1" +Malt = "^0.7.0" MsgPack = "1.1" PrecompileSignatures = "3" RegistryInstances = "0.1" diff --git a/src/Configuration.jl b/src/Configuration.jl index 920782d25d..70974a83f7 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -128,7 +128,6 @@ Note that Pluto is quickly evolving software, maintained by designers, educators end const RUN_NOTEBOOK_ON_LOAD_DEFAULT = true -const WORKSPACE_USE_DISTRIBUTED_DEFAULT = true const LAZY_WORKSPACE_CREATION_DEFAULT = false const CAPTURE_STDOUT_DEFAULT = true const WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT = nothing @@ -140,14 +139,12 @@ Options to change Pluto's evaluation behaviour during internal testing and by do These options are not intended to be changed during normal use. - `run_notebook_on_load::Bool = $RUN_NOTEBOOK_ON_LOAD_DEFAULT` Whether to evaluate a notebook on load. -- `workspace_use_distributed::Bool = $WORKSPACE_USE_DISTRIBUTED_DEFAULT` Whether to start notebooks in a separate process. - `lazy_workspace_creation::Bool = $LAZY_WORKSPACE_CREATION_DEFAULT` - `capture_stdout::Bool = $CAPTURE_STDOUT_DEFAULT` - `workspace_custom_startup_expr::Union{Nothing,Expr} = $WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT` An expression to be evaluated in the workspace process before running notebook code. """ @option mutable struct EvaluationOptions run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT - workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT @@ -254,7 +251,6 @@ function from_flat_kwargs(; require_secret_for_open_links::Bool = REQUIRE_SECRET_FOR_OPEN_LINKS_DEFAULT, require_secret_for_access::Bool = REQUIRE_SECRET_FOR_ACCESS_DEFAULT, run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT, - workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT, lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT, capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT, workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT, @@ -294,7 +290,6 @@ function from_flat_kwargs(; ) evaluation = EvaluationOptions(; run_notebook_on_load, - workspace_use_distributed, lazy_workspace_creation, capture_stdout, workspace_custom_startup_expr, diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index bcd95db01c..bf85814c0b 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -6,18 +6,18 @@ import ..Pluto.PkgCompat import ..Configuration: CompilerOptions, _merge_notebook_compiler_options, _convert_to_flags import ..Pluto.ExpressionExplorer: FunctionName import ..PlutoRunner -import Distributed +import Malt """ -Contains the Julia process (in the sense of `Distributed.addprocs`) to evaluate code in. +Contains the Julia process to evaluate code in. Each notebook gets at most one `Workspace` at any time, but it can also have no `Workspace` (it cannot `eval` code in this case). """ Base.@kwdef mutable struct Workspace - pid::Integer + worker::Malt.Worker notebook_id::UUID discarded::Bool=false - remote_log_channel::Distributed.RemoteChannel + remote_log_channel::Channel module_name::Symbol dowork_token::Token=Token() nbpkg_was_active::Bool=false @@ -36,11 +36,7 @@ const process_preamble = quote ENV["JULIA_REVISE_WORKER_ONLY"] = "1" end -const Distributed_expr = quote - Base.loaded_modules[Base.PkgId(Base.UUID("8ba89e20-285c-5b6f-9357-94700520ee1b"), "Distributed")] -end - -const workspaces = Dict{UUID,Task}() +const active_workspaces = Dict{UUID,Task}() "Set of notebook IDs that we will never make a process for again." const discarded_workspaces = Set{UUID}() @@ -49,54 +45,33 @@ const discarded_workspaces = Set{UUID}() function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false)::Workspace is_offline_renderer || (notebook.process_status = ProcessStatus.starting) - use_distributed = if is_offline_renderer - false - else - session.options.evaluation.workspace_use_distributed - end - - pid = if use_distributed - @debug "Creating workspace process" notebook.path length(notebook.cells) - create_workspaceprocess(; compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) - else - pid = Distributed.myid() - if !(isdefined(Main, :PlutoRunner) && Main.PlutoRunner isa Module) - # we make PlutoRunner available in Main, right now it's only defined inside this Pluto module. - @eval Main begin - PlutoRunner = $(PlutoRunner) - end - end - pid - end + @debug "Creating workspace process" notebook.path length(notebook.cells) + worker = create_workspaceprocess(;compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) - Distributed.remotecall_eval(Main, [pid], session.options.evaluation.workspace_custom_startup_expr) + Malt.remote_eval_wait(Main, worker, session.options.evaluation.workspace_custom_startup_expr) - Distributed.remotecall_eval(Main, [pid], quote + Malt.remote_eval_wait(Main, worker, quote PlutoRunner.notebook_id[] = $(notebook.notebook_id) end) - remote_log_channel = Core.eval(Main, quote - $(Distributed).RemoteChannel(() -> eval(quote - channel = Channel{Any}(10) - Main.PlutoRunner.setup_plutologger( - $($(notebook.notebook_id)), - channel; - make_global=$($(use_distributed)) - ) - channel - end), $pid) + remote_log_channel = Malt.worker_channel(worker, quote + channel = Channel{Any}(10) + Main.PlutoRunner.setup_plutologger( + $(notebook.notebook_id), + channel; + make_global=false # TODO(savq): Is this still necessary? + ) + channel end) - run_channel = Core.eval(Main, quote - $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.run_channel)), $pid) - end) + run_channel = Malt.worker_channel(worker, :(Main.PlutoRunner.run_channel)) - module_name = create_emptyworkspacemodule(pid) + module_name = create_emptyworkspacemodule(worker) - original_LOAD_PATH, original_ACTIVE_PROJECT = Distributed.remotecall_eval(Main, pid, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) + original_LOAD_PATH, original_ACTIVE_PROJECT = Malt.remote_eval_fetch(Main, worker, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) workspace = Workspace(; - pid, + worker, notebook_id=notebook.notebook_id, remote_log_channel, module_name, @@ -124,20 +99,16 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) workspace.discarded && return workspace.nbpkg_was_active = enabled - if workspace.pid != Distributed.myid() - new_LP = enabled ? ["@", "@stdlib"] : workspace.original_LOAD_PATH - new_AP = enabled ? PkgCompat.env_dir(notebook.nbpkg_ctx) : workspace.original_ACTIVE_PROJECT + new_LP = enabled ? ["@", "@stdlib"] : workspace.original_LOAD_PATH + new_AP = enabled ? PkgCompat.env_dir(notebook.nbpkg_ctx) : workspace.original_ACTIVE_PROJECT - Distributed.remotecall_eval(Main, [workspace.pid], quote - copy!(LOAD_PATH, $(new_LP)) - Base.ACTIVE_PROJECT[] = $(new_AP) - end) - else - # TODO - end + Malt.remote_eval_wait(Main, workspace.worker, quote + copy!(LOAD_PATH, $(new_LP)) + Base.ACTIVE_PROJECT[] = $(new_AP) + end) end -function start_relaying_self_updates((session, notebook)::SN, run_channel::Distributed.RemoteChannel) +function start_relaying_self_updates((session, notebook)::SN, run_channel::Channel) while true try next_run_uuid = take!(run_channel) @@ -153,7 +124,7 @@ function start_relaying_self_updates((session, notebook)::SN, run_channel::Distr end end -function start_relaying_logs((session, notebook)::SN, log_channel::Distributed.RemoteChannel) +function start_relaying_logs((session, notebook)::SN, log_channel::Channel) update_throttled, flush_throttled = Pluto.throttled(0.1) do Pluto.send_notebook_changes!(Pluto.ClientRequest(session=session, notebook=notebook)) end @@ -238,7 +209,7 @@ end function bump_workspace_module(session_notebook::SN) workspace = get_workspace(session_notebook) old_name = workspace.module_name - new_name = workspace.module_name = create_emptyworkspacemodule(workspace.pid) + new_name = workspace.module_name = create_emptyworkspacemodule(workspace.worker) old_name, new_name end @@ -246,13 +217,13 @@ end function possible_bond_values(session_notebook::SN, n::Symbol; get_length::Bool=false) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote PlutoRunner.possible_bond_values($(QuoteNode(n)); get_length=$(get_length)) end) end -function create_emptyworkspacemodule(pid::Integer)::Symbol - Distributed.remotecall_eval(Main, pid, quote +function create_emptyworkspacemodule(worker::Malt.Worker)::Symbol + Malt.remote_eval_fetch(Main, worker, quote PlutoRunner.increment_current_module() end) end @@ -260,17 +231,12 @@ end # NOTE: this function only start a worker process using given # compiler options, it does not resolve paths for notebooks # compiler configurations passed to it should be resolved before this -function create_workspaceprocess(; compiler_options=CompilerOptions())::Integer - # run on proc 1 in case Pluto is being used inside a notebook process - # Workaround for "only process 1 can add/remove workers" - pid = Distributed.remotecall_eval(Main, 1, quote - $(Distributed_expr).addprocs(1; exeflags=$(_convert_to_flags(compiler_options))) |> first - end) - - Distributed.remotecall_eval(Main, [pid], process_preamble) +function create_workspaceprocess(;compiler_options=CompilerOptions())::Malt.Worker + worker = Malt.Worker(;exeflags=_convert_to_flags(compiler_options)) + Malt.remote_eval_wait(Main, worker, process_preamble) # so that we NEVER break the workspace with an interrupt 🤕 - @async Distributed.remotecall_eval(Main, [pid], quote + Malt.remote_eval(Main, worker, quote while true try wait() @@ -278,7 +244,7 @@ function create_workspaceprocess(; compiler_options=CompilerOptions())::Integer end end) - pid + worker end """ @@ -294,9 +260,9 @@ function get_workspace(session_notebook::SN; allow_creation::Bool=true)::Union{N end task = if !allow_creation - get(workspaces, notebook.notebook_id, nothing) + get(active_workspaces, notebook.notebook_id, nothing) else - get!(workspaces, notebook.notebook_id) do + get!(active_workspaces, notebook.notebook_id) do Task(() -> make_workspace(session_notebook)) end end @@ -315,28 +281,16 @@ function unmake_workspace(session_notebook::SN; async::Bool=false, verbose::Bool workspace.discarded = true allow_restart || push!(discarded_workspaces, notebook.notebook_id) - if workspace.pid != Distributed.myid() - filter!(p -> fetch(p.second).pid != workspace.pid, workspaces) - t = @async begin - interrupt_workspace(workspace; verbose=false) - # run on proc 1 in case Pluto is being used inside a notebook process - # Workaround for "only process 1 can add/remove workers" - Distributed.remotecall_eval(Main, 1, quote - $(Distributed_expr).rmprocs($(workspace.pid)) - end) - end - async || wait(t) - else - if !isready(workspace.dowork_token) - @error "Cannot unmake a workspace running inside the same process: the notebook is still running." - elseif verbose - @warn "Cannot unmake a workspace running inside the same process: the notebook might still be running. If you are sure that your code is not running the notebook async, then you can use the `verbose=false` keyword argument to disable this message." - end + filter!(p -> fetch(p.second).worker != workspace.worker, active_workspaces) + t = @async begin + interrupt_workspace(workspace; verbose=false) + Malt.stop(workspace.worker) end + async || wait(t) nothing end -function distributed_exception_result(ex::Base.IOError, workspace::Workspace) +function workspace_exception_result(ex::Base.IOError, workspace::Workspace) ( output_formatted=PlutoRunner.format_output(CapturedException(ex, [])), errored=true, @@ -348,12 +302,10 @@ function distributed_exception_result(ex::Base.IOError, workspace::Workspace) ) end -function distributed_exception_result(exs::CompositeException, workspace::Workspace) +function workspace_exception_result(exs::CompositeException, workspace::Workspace) ex = first(exs.exceptions) - if ex isa Distributed.RemoteException && - ex.pid == workspace.pid && - ex.captured.ex isa InterruptException + if ex.worker == workspace.worker && ex.captured.ex isa InterruptException ( output_formatted=PlutoRunner.format_output(CapturedException(InterruptException(), [])), errored=true, @@ -363,7 +315,7 @@ function distributed_exception_result(exs::CompositeException, workspace::Worksp published_objects=Dict{String,Any}(), has_pluto_hook_features=false, ) - elseif ex isa Distributed.ProcessExitedException + elseif ex isa Malt.TerminatedWorkerException ( output_formatted=PlutoRunner.format_output(CapturedException(exs, [])), errored=true, @@ -408,13 +360,8 @@ function eval_format_fetch_in_workspace( workspace = get_workspace(session_notebook) - is_on_this_process = workspace.pid == Distributed.myid() - # if multiple notebooks run on the same process, then we need to `cd` between the different notebook paths if session_notebook isa Tuple - if is_on_this_process - cd_workspace(workspace, session_notebook[2].path) - end use_nbpkg_environment(session_notebook, workspace) end @@ -423,8 +370,7 @@ function eval_format_fetch_in_workspace( # A try block (on this process) to catch an InterruptException take!(workspace.dowork_token) early_result = try - # Use [pid] instead of pid to prevent fetching output - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(Main, workspace.worker, quote PlutoRunner.run_expression( getfield(Main, $(QuoteNode(workspace.module_name))), $(QuoteNode(expr)), @@ -433,7 +379,7 @@ function eval_format_fetch_in_workspace( $function_wrapped_info, $forced_expr_id; user_requested_run=$user_requested_run, - capture_stdout=$(capture_stdout && !is_on_this_process), + capture_stdout=$capture_stdout, ) end) put!(workspace.dowork_token) @@ -441,7 +387,7 @@ function eval_format_fetch_in_workspace( catch e # Don't use a `finally` because the token needs to be back asap for the interrupting code to pick it up. put!(workspace.dowork_token) - distributed_exception_result(e, workspace) + workspace_exception_result(e, workspace) end if early_result === nothing @@ -455,7 +401,7 @@ end function eval_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(Main, workspace.worker, quote Core.eval($(workspace.module_name), $(QuoteNode(expr))) end) nothing @@ -475,7 +421,7 @@ function format_fetch_in_workspace( # we format the cell output on the worker, and fetch the formatted output. withtoken(workspace.dowork_token) do try - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote PlutoRunner.formatted_result_of( $(workspace.notebook_id), $cell_id, @@ -486,7 +432,7 @@ function format_fetch_in_workspace( ) end) catch e - distributed_exception_result(CompositeException([e]), workspace) + workspace_exception_result(CompositeException([e]), workspace) end end end @@ -494,7 +440,7 @@ end function collect_soft_definitions(session_notebook::SN, modules::Set{Expr}) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote PlutoRunner.collect_soft_definitions($(workspace.module_name), $modules) end) end @@ -503,7 +449,7 @@ function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macroca workspace = get_workspace(session_notebook) module_name = module_name === nothing ? workspace.module_name : module_name - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote try (true, PlutoRunner.try_macroexpand($module_name, $(workspace.notebook_id), $cell_id, $(QuoteNode(macrocall)))) catch error @@ -523,7 +469,7 @@ end function eval_fetch_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote Core.eval($(workspace.module_name), $(QuoteNode(expr))) end) end @@ -531,7 +477,7 @@ end function do_reimports(session_notebook::Union{SN,Workspace}, module_imports_to_move::Set{Expr}) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(Main, workspace.worker, quote PlutoRunner.do_reimports($(workspace.module_name), $module_imports_to_move) end) end @@ -553,7 +499,7 @@ function move_vars( workspace = get_workspace(session_notebook) new_workspace_name = something(new_workspace_name, workspace.module_name) - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(Main, workspace.worker, quote PlutoRunner.move_vars( $(QuoteNode(old_workspace_name)), $(QuoteNode(new_workspace_name)), @@ -640,11 +586,6 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true return false end - if workspace.pid == Distributed.myid() - verbose && @warn """Cells in this workspace can't be stopped, because it is not running in a separate workspace. Use `ENV["PLUTO_WORKSPACE_USE_DISTRIBUTED"]` to control whether future workspaces are generated in a separate process.""" - return false - end - if isready(workspace.dowork_token) verbose && @info "Tried to stop idle workspace - ignoring." return true @@ -655,8 +596,8 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true # TODO: this will also kill "pending" evaluations, and any evaluations started within 100ms of the kill. A global "evaluation count" would fix this. # TODO: listen for the final words of the remote process on stdout/stderr: "Force throwing a SIGINT" try - verbose && @info "Sending interrupt to process $(workspace.pid)" - Distributed.interrupt(workspace.pid) + verbose && @info "Sending interrupt to process $(workspace.worker)" + Malt.interrupt(workspace.worker) if poll(() -> isready(workspace.dowork_token), 5.0, 5/100) verbose && println("Cell interrupted!") @@ -667,7 +608,7 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true while !isready(workspace.dowork_token) for _ in 1:5 verbose && print(" 🔥 ") - Distributed.interrupt(workspace.pid) + Malt.interrupt(workspace.worker) sleep(0.18) if isready(workspace.dowork_token) break diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 30e957a112..7983450a9f 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -1,17 +1,21 @@ # Will be evaluated _inside_ the workspace process. -# Pluto does most things on process 1 (the server), and it uses little workspace processes to evaluate notebook code in. -# These baby processes don't import Pluto, they only import this module. Functions from this module are called by WorkspaceManager.jl via Distributed +# Pluto does most things on the server, but it uses worker processes to evaluate notebook code in. +# These processes don't import Pluto, they only import this module. +# Functions from this module are called by WorkspaceManager.jl via Malt. -# So when reading this file, pretend that you are living in process 2, and you are communicating with Pluto's server, who lives in process 1. +# When reading this file, pretend that you are living in a worker process, +# and you are communicating with Pluto's server, who lives in the main process. # The package environment that this file is loaded with is the NotebookProcessProject.toml file in this directory. # SOME EXTRA NOTES # 1. The entire PlutoRunner should be a single file. -# 2. We restrict the communication between this PlutoRunner and the Pluto server to only use *Base Julia types*, like `String`, `Dict`, `NamedTuple`, etc. +# 2. Restrict the communication between this PlutoRunner and the Pluto server to only use *Base Julia types*, like `String`, `Dict`, `NamedTuple`, etc. -# These restriction are there to allow flexibility in the way that this file is loaded on a runner process, which is something that we might want to change in the future, like when we make the transition to our own Distributed. +# These restriction are there to allow flexibility in the way that this file is +# loaded on a runner process, which is something that we might want to change +# in the future. module PlutoRunner @@ -21,7 +25,6 @@ import InteractiveUtils using Markdown import Markdown: html, htmlinline, LaTeX, withtag, htmlesc -import Distributed import Base64 import FuzzyCompletions: Completion, BslashCompletion, ModuleCompletion, PropertyCompletion, FieldCompletion, PathCompletion, DictCompletion, completions, completion_text, score import Base: show, istextmime @@ -32,7 +35,7 @@ import REPL export @bind -# This is not a struct to make it easier to pass these objects between distributed processes. +# This is not a struct to make it easier to pass these objects between processes. const MimedOutput = Tuple{Union{String,Vector{UInt8},Dict{Symbol,Any}},MIME} const ObjectID = typeof(objectid("hello computer")) @@ -846,7 +849,7 @@ const table_column_display_limit_increase = 30 const tree_display_extra_items = Dict{UUID,Dict{ObjectDimPair,Int64}}() -# This is not a struct to make it easier to pass these objects between distributed processes. +# This is not a struct to make it easier to pass these objects between processes. const FormattedCellResult = NamedTuple{(:output_formatted, :errored, :interrupted, :process_exited, :runtime, :published_objects, :has_pluto_hook_features),Tuple{PlutoRunner.MimedOutput,Bool,Bool,Bool,Union{UInt64,Nothing},Dict{String,Any},Bool}} function formatted_result_of( @@ -1888,15 +1891,15 @@ function possible_bond_values(s::Symbol; get_length::Bool=false) try length(possible_values) catch - length(make_distributed_serializable(possible_values)) + length(make_serializable(possible_values)) end : - make_distributed_serializable(possible_values) + make_serializable(possible_values) end end -make_distributed_serializable(x::Any) = x -make_distributed_serializable(x::Union{AbstractVector,AbstractSet,Base.Generator}) = collect(x) -make_distributed_serializable(x::Union{Vector,Set,OrdinalRange}) = x +make_serializable(x::Any) = x +make_serializable(x::Union{AbstractVector,AbstractSet,Base.Generator}) = collect(x) +make_serializable(x::Union{Vector,Set,OrdinalRange}) = x """ diff --git a/src/webserver/REPLTools.jl b/src/webserver/REPLTools.jl index d6a09b7c93..857b8752e1 100644 --- a/src/webserver/REPLTools.jl +++ b/src/webserver/REPLTools.jl @@ -1,5 +1,5 @@ import FuzzyCompletions: complete_path, completion_text, score -import Distributed +import Malt import .PkgCompat: package_completions using Markdown import REPL @@ -83,10 +83,13 @@ responses[:complete] = function response_complete(🙋::ClientRequest) if will_run_code(🙋.notebook) && workspace isa WorkspaceManager.Workspace && isready(workspace.dowork_token) # we don't use eval_format_fetch_in_workspace because we don't want the output to be string-formatted. # This works in this particular case, because the return object, a `Completion`, exists in this scope too. - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.completion_fetcher( - $query, $pos, - getfield(Main, $(QuoteNode(workspace.module_name))), - ))) + Malt.remote_eval_fetch(workspace.worker, quote + PlutoRunner.completion_fetcher( + $query, + $pos, + getfield(Main, $(QuoteNode(workspace.module_name))), + ) + end) else # We can at least autocomplete general julia things: PlutoRunner.completion_fetcher(query, pos, Main) @@ -125,10 +128,12 @@ responses[:docs] = function response_docs(🙋::ClientRequest) workspace = WorkspaceManager.get_workspace((🙋.session, 🙋.notebook); allow_creation=false) if will_run_code(🙋.notebook) && workspace isa WorkspaceManager.Workspace && isready(workspace.dowork_token) - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.doc_fetcher( - $query, - getfield(Main, $(QuoteNode(workspace.module_name))), - ))) + Malt.remote_eval_fetch(workspace.worker, quote + PlutoRunner.doc_fetcher( + $query, + getfield(Main, $(QuoteNode(workspace.module_name))), + ) + end) else (nothing, :⌛) end diff --git a/test/Bonds.jl b/test/Bonds.jl index 21ab86d598..1ed679aca4 100644 --- a/test/Bonds.jl +++ b/test/Bonds.jl @@ -1,15 +1,13 @@ using Test import Pluto import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell -import Distributed +import Malt @testset "Bonds" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false - + @testset "AbstractPlutoDingetjes.jl" begin - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ # 1 Cell(""" @@ -245,7 +243,7 @@ import Distributed @test Pluto.possible_bond_values(🍭, notebook, :x_new) == [1,2,3] - @test_throws Exception Pluto.possible_bond_values(🍭, notebook, :asdfasdfx_new) + @test Pluto.possible_bond_values(🍭, notebook, :asdfasdfx_new) == KeyError(:asdfasdfx_new) @test Pluto.possible_bond_values(🍭, notebook, :pv1) == :NotGiven @test Pluto.possible_bond_values(🍭, notebook, :pv2) == :InfinitePossibilities @test Pluto.possible_bond_values(🍭, notebook, :pv3) == [1,2,3] @@ -299,14 +297,13 @@ import Distributed WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false # test that the notebook file is runnable: - test_proc = Distributed.addprocs(1)[1] + test_worker = Malt.Worker() - Distributed.remotecall_eval(Main, test_proc, quote + Malt.remote_eval_fetch(Main, test_worker, quote import Pkg try Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true @@ -314,16 +311,16 @@ import Distributed Pkg.activate(mktempdir()) Pkg.add("AbstractPlutoDingetjes") end) - @test Distributed.remotecall_eval(Main, test_proc, quote + @test Malt.remote_eval_fetch(Main, test_worker, quote include($(notebook.path)) true end) - Distributed.rmprocs(test_proc) + Malt.stop(test_worker) end @testset "Dependent Bound Variables" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true + notebook = Notebook([ Cell(raw"""@bind x HTML("")"""), Cell(raw"""@bind y HTML("")"""), diff --git a/test/Configuration.jl b/test/Configuration.jl index 0b6b131618..355da2eb63 100644 --- a/test/Configuration.jl +++ b/test/Configuration.jl @@ -59,7 +59,7 @@ end @testset "Authentication" begin port = 1238 - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false) 🍭 = Pluto.ServerSession(; options) host = 🍭.options.server.host secret = 🍭.secret @@ -140,21 +140,21 @@ end server_running() = HTTP.get(local_url("favicon.ico")).status == 200 && HTTP.get(local_url("edit")).status == 200 # without notebook at startup - server_task = @async Pluto.run(port=port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + server_task = @async Pluto.run(port=port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) @test poll(5) do server_running() end @async schedule(server_task, InterruptException(); error=true) # with a single notebook at startup - server_task = @async Pluto.run(notebook=first(nbnames), port=port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + server_task = @async Pluto.run(notebook=first(nbnames), port=port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) @test poll(5) do server_running() end @async schedule(server_task, InterruptException(); error=true) # with multiple notebooks at startup - server_task = @async Pluto.run(notebook=nbnames, port=port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + server_task = @async Pluto.run(notebook=nbnames, port=port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) @test poll(5) do server_running() end diff --git a/test/DependencyCache.jl b/test/DependencyCache.jl index 77b2c0b660..2de1884771 100644 --- a/test/DependencyCache.jl +++ b/test/DependencyCache.jl @@ -5,7 +5,6 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @testset "CellDepencencyVisualization" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("x = 1"), # prerequisite of test cell diff --git a/test/Dynamic.jl b/test/Dynamic.jl index fcdcadc984..9a431f6f92 100644 --- a/test/Dynamic.jl +++ b/test/Dynamic.jl @@ -178,7 +178,6 @@ end @testset "PlutoRunner API" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ Cell("PlutoRunner.notebook_id[] |> Text"), diff --git a/test/Events.jl b/test/Events.jl index 1dbfdaee03..b336ff61ef 100644 --- a/test/Events.jl +++ b/test/Events.jl @@ -14,7 +14,6 @@ import UUIDs: UUID end 🍭 = ServerSession() 🍭.options.server.on_event = test_listener - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("[1,1,[1]]"), @@ -35,4 +34,4 @@ import UUIDs: UUID # Pluto.save_notebook(io::IOBuffer, notebook): saves notebook to IO # Pluto.ServerSession(;options, event_listener) -end \ No newline at end of file +end diff --git a/test/Logging.jl b/test/Logging.jl index 125f048f2f..cc3e9dde55 100644 --- a/test/Logging.jl +++ b/test/Logging.jl @@ -5,7 +5,6 @@ using Pluto.WorkspaceManager: poll @testset "Logging" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook(Cell.([ "println(123)", diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index 59e79157c3..79d8151ffb 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -5,7 +5,6 @@ import Memoize: @memoize @testset "Macro analysis" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false @testset "Base macro call" begin notebook = Notebook([ @@ -27,6 +26,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test :Fruit ∈ notebook.topology.nodes[cell(3)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 1" begin @@ -48,6 +49,8 @@ import Memoize: @memoize # Works on second time because of old workspace @test :x ∈ notebook.topology.nodes[cell(2)].definitions @test Symbol("@my_assign") ∈ notebook.topology.nodes[cell(2)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 2" begin @@ -77,6 +80,8 @@ import Memoize: @memoize @test cell(1) |> noerror @test cell(2) |> noerror @test cell(3) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 3" begin @@ -100,6 +105,8 @@ import Memoize: @memoize update_run!(🍭, notebook, cell(1)) @test cell(2) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 4" begin @@ -114,6 +121,8 @@ import Memoize: @memoize update_run!(🍭, notebook, notebook.cells) @test Symbol("@my_assign") ∈ notebook.topology.nodes[cell(2)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 5" begin @@ -130,6 +139,8 @@ import Memoize: @memoize @test :a ∉ references(2) @test :b ∉ references(2) @test :c ∉ references(2) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 6" begin @@ -150,6 +161,8 @@ import Memoize: @memoize @test [Symbol("@my_macro"), :x, :y] ⊆ notebook.topology.nodes[cell(2)].references @test cell(3).output.body == "3" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Function docs" begin @@ -166,6 +179,8 @@ import Memoize: @memoize @test :f ∈ notebook.topology.nodes[cell(1)].funcdefs_without_signatures @test :f ∈ notebook.topology.nodes[cell(2)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Expr sanitization" begin @@ -211,6 +226,8 @@ import Memoize: @memoize @test cell(2).output.body == "true" @test all(noerror, notebook.cells) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Reverse order" begin @@ -238,6 +255,8 @@ import Memoize: @memoize @test cell(2) |> noerror @test cell(3) |> noerror @test cell(1).output.body == "\"yay\"" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "@a defines @b" begin @@ -272,6 +291,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(4) |> noerror @test cell(1).output.body == "42" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Removing macros undefvar errors dependent cells" begin @@ -293,6 +314,8 @@ import Memoize: @memoize @test notebook.cells[end].errored @test occursinerror("UndefVarError: @m", notebook.cells[end]) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Redefines macro with new SymbolsState" begin @@ -342,6 +365,8 @@ import Memoize: @memoize # See Run.jl#resolve_topology. @test cell(4).output.body == "42" @test cell(3).errored == true + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Reactive macro update does not invalidate the macro calls" begin @@ -378,6 +403,8 @@ import Memoize: @memoize @test cell(4).output.body != "42" @test cell(4).errored == true @test cell(5) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Explicitely running macrocalls updates the reactive node" begin @@ -409,6 +436,8 @@ import Memoize: @memoize @test cell(4).errored == true @test cell(5) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Implicitely running macrocalls updates the reactive node" begin @@ -447,6 +476,8 @@ import Memoize: @memoize # an explicit run of @b() must be done. @test cell(4).output.body == output_1 @test cell(5).errored == true + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Weird behavior" begin @@ -474,6 +505,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(3).output.body == "1234" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @@ -490,6 +523,8 @@ import Memoize: @memoize # x ("@b(x)") was run. Should it? Maybe set a higher precedence to cells that define # macros inside the notebook. @test_broken noerror(notebook.cells[1]; verbose=false) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "@a defines @b initial loading" begin @@ -513,6 +548,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(4) |> noerror @test cell(1).output.body == "42" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macro with long compile time gets function wrapped" begin @@ -559,11 +596,11 @@ import Memoize: @memoize @test cell(1) |> noerror @test output_3 != cell(1).output.body + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macro Prefix" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook(Cell.([ "@sprintf \"answer = %d\" x", "x = y+1", @@ -592,9 +629,9 @@ import Memoize: @memoize @test cell(1) |> noerror WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end + @testset "Package macro 1" begin notebook = Notebook([ Cell("using Dates"), @@ -620,11 +657,11 @@ import Memoize: @memoize @test cell(1) |> noerror @test cell(2) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Package macro 2" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook([ Cell("z = x^2 + y"), Cell("@variables x y"), @@ -674,8 +711,6 @@ import Memoize: @memoize @test cell(2) |> noerror WorkspaceManager.unmake_workspace((🍭, notebook)) - - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Previous workspace for unknowns" begin @@ -699,6 +734,8 @@ import Memoize: @memoize module_from_cell3 = cell(3).output.body @test module_from_cell2 == module_from_cell3 + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Definitions" begin @@ -725,6 +762,8 @@ import Memoize: @memoize @test ":world" == cell(3).output.body @test ":world" == cell(4).output.body + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Is just text macros" begin @@ -739,6 +778,8 @@ import Memoize: @memoize update_run!(🍭, notebook, notebook.cells) @test isempty(notebook.topology.unresolved_cells) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macros using import" begin @@ -757,6 +798,8 @@ import Memoize: @memoize @test :option_type ∈ notebook.topology.nodes[cell(1)].references @test cell(1) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "GlobalRefs in macros should be respected" begin @@ -781,6 +824,8 @@ import Memoize: @memoize @test all(cell.([1,2,3]) .|> noerror) @test cell(3).output.body == "20" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "GlobalRefs shouldn't break unreached undefined references" begin @@ -806,6 +851,8 @@ import Memoize: @memoize @test all(cell.([1,2]) .|> noerror) @test cell(2).output.body == ":this_should_be_returned" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Doc strings" begin @@ -878,11 +925,12 @@ import Memoize: @memoize update_run!(🍭, notebook, bool) @test !occursin("An empty conjugate", bool.output.body) @test occursin("complex conjugate", bool.output.body) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Delete methods from macros" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("using Memoize"), @@ -922,5 +970,7 @@ import Memoize: @memoize @test occursinerror("UndefVarError: custom_func", cell(4)) @test :memoized_func ∉ notebook.topology.nodes[cell(5)].funcdefs_without_signatures @test occursinerror("UndefVarError: memoized_func", cell(6)) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end end diff --git a/test/Notebook.jl b/test/Notebook.jl index 7e7d5480ec..089792c084 100644 --- a/test/Notebook.jl +++ b/test/Notebook.jl @@ -203,7 +203,6 @@ end @testset "Cell Metadata" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false @testset "Disabling & Metadata" begin nb = cell_metadata_notebook() @@ -238,7 +237,6 @@ end @testset "Notebook Metadata" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false nb = notebook_metadata_notebook() update_run!(🍭, nb, nb.cells) @@ -263,7 +261,6 @@ end @testset "Skip as script" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false nb = skip_as_script_notebook() update_run!(🍭, nb, nb.cells) diff --git a/test/React.jl b/test/React.jl index eb1b5e0d7a..bcb4600b21 100644 --- a/test/React.jl +++ b/test/React.jl @@ -1,15 +1,12 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager import Pluto.Configuration: Options, EvaluationOptions -import Distributed +import Malt @testset "Reactivity" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false - @testset "Basic $(parallel ? "distributed" : "single-process")" for parallel in [false, true] - 🍭.options.evaluation.workspace_use_distributed = parallel - + @testset "Basic" begin notebook = Notebook([ Cell("x = 1"), Cell("y = x"), @@ -26,7 +23,7 @@ import Distributed Cell("Distributed.myid()"), ]) - @test !haskey(WorkspaceManager.workspaces, notebook.notebook_id) + @test !haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) update_run!(🍭, notebook, notebook.cells[1:2]) @test notebook.cells[1].output.body == notebook.cells[2].output.body @@ -70,18 +67,12 @@ import Distributed @test notebook.cells[6].output.body == "3" update_run!(🍭, notebook, notebook.cells[7:8]) - @test if parallel - notebook.cells[8].output.body != string(Distributed.myid()) - else - notebook.cells[8].output.body == string(Distributed.myid()) - end + notebook.cells[8].output.body == "1" WorkspaceManager.unmake_workspace((🍭, notebook); verbose=false) end - 🍭.options.evaluation.workspace_use_distributed = false - @testset "Mutliple assignments" begin notebook = Notebook([ Cell("x = 1"), @@ -207,8 +198,6 @@ import Distributed # PlutoTest.jl is only working on Julia version >= 1.6 @testset "Test Firebasey" begin - 🍭.options.evaluation.workspace_use_distributed = true - file = tempname() write(file, read(normpath(Pluto.project_relative_path("src", "webserver", "Firebasey.jl")))) @@ -222,7 +211,6 @@ import Distributed @test all(noerror, notebook.cells) WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Pkg topology workarounds" begin @@ -371,8 +359,6 @@ import Distributed end @testset "Reactive usings 4" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook([ Cell("@sprintf \"double_december = %d\" double_december"), Cell("double_december = 2December"), @@ -397,7 +383,6 @@ import Distributed @test notebook.cells[1].output.body == "\"double_december = 24\"" WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Reactive usings 5" begin @@ -429,8 +414,6 @@ import Distributed end @testset "Function dependencies" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook(Cell.([ "a'b", "import LinearAlgebra", @@ -446,7 +429,6 @@ import Distributed @test notebook.cells[1].output.body == "200" WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Function use inv in its def but also has a method on inv" begin diff --git a/test/ReloadFromFile.jl b/test/ReloadFromFile.jl index 01bf9358ca..98a9091c2f 100644 --- a/test/ReloadFromFile.jl +++ b/test/ReloadFromFile.jl @@ -1,7 +1,6 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager, SessionActions, save_notebook import Pluto.Configuration: Options, EvaluationOptions -import Distributed using Pluto.WorkspaceManager: poll import Pkg @@ -23,7 +22,6 @@ end retry(3) do 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false 🍭.options.server.auto_reload_from_file = true @@ -160,4 +158,4 @@ end @assert notebook.nbpkg_restart_required_msg !== nothing end @test true -end \ No newline at end of file +end diff --git a/test/RichOutput.jl b/test/RichOutput.jl index 6675aa4008..de631263fb 100644 --- a/test/RichOutput.jl +++ b/test/RichOutput.jl @@ -6,8 +6,7 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @testset "Rich output" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false - + @testset "Tree viewer" begin @testset "Basics" begin notebook = Notebook([ @@ -134,8 +133,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @testset "Special arrays" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook([ Cell("using OffsetArrays"), Cell("OffsetArray(zeros(3), 20:22)"), @@ -190,7 +187,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @test occursin("103", s) WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Circular references" begin @@ -230,7 +226,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb end @testset "Table viewer" begin - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ Cell("using DataFrames, Tables"), Cell("DataFrame()"), @@ -308,7 +303,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb # TODO: test lazy loading more rows/cols WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end begin diff --git a/test/WorkspaceManager.jl b/test/WorkspaceManager.jl index 92975be90a..1d952d5019 100644 --- a/test/WorkspaceManager.jl +++ b/test/WorkspaceManager.jl @@ -2,14 +2,13 @@ using Test using Pluto.Configuration: CompilerOptions using Pluto.WorkspaceManager: _merge_notebook_compiler_options import Pluto: update_save_run!, update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell, project_relative_path -import Distributed +import Malt @testset "Workspace manager" begin # basic functionality is already tested by the reactivity tests @testset "Multiple notebooks" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true notebookA = Notebook([ Cell("x = 3") @@ -34,7 +33,6 @@ import Distributed end @testset "Variables with secret names" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("result = 1"), @@ -54,7 +52,7 @@ import Distributed Sys.iswindows() || @testset "Pluto inside Pluto" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true + 🍭.options.evaluation.capture_stdout = false notebook = Notebook([ Cell("""begin @@ -86,17 +84,10 @@ import Distributed update_run!(🍭, notebook, notebook.cells[5]) @test notebook.cells[5] |> noerror - - desired_nprocs = Distributed.nprocs() - 1 setcode!(notebook.cells[5], "Pluto.SessionActions.shutdown(s, nb)") update_run!(🍭, notebook, notebook.cells[5]) @test noerror(notebook.cells[5]) - while Distributed.nprocs() != desired_nprocs - sleep(.1) - end - sleep(.1) - WorkspaceManager.unmake_workspace((🍭, notebook)) end end diff --git a/test/cell_disabling.jl b/test/cell_disabling.jl index f5f9575dd3..420389de5d 100644 --- a/test/cell_disabling.jl +++ b/test/cell_disabling.jl @@ -8,7 +8,6 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook, set_disa @testset "Cell Disabling" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("const a = 1") diff --git a/test/compiletimes.jl b/test/compiletimes.jl index c1eef7d165..9f43543aad 100644 --- a/test/compiletimes.jl +++ b/test/compiletimes.jl @@ -30,7 +30,6 @@ end 🍭 = Pluto.ServerSession() 🍭.options.server.disable_writing_notebook_files = true -🍭.options.evaluation.workspace_use_distributed = false path = joinpath(pkgdir(Pluto), "sample", "Basic.jl") @@ -45,7 +44,7 @@ HTTP.get("http://github.com") @timeit TOUT "Pluto.run" server_task = @eval let port = 13435 - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) 🍭 = Pluto.ServerSession(; options) server_task = @async Pluto.run(🍭) diff --git a/test/helpers.jl b/test/helpers.jl index c254a748f2..718166a581 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -227,14 +227,14 @@ has_embedded_pkgfiles(contents::AbstractString) = has_embedded_pkgfiles(nb::Pluto.Notebook) = read(nb.path, String) |> has_embedded_pkgfiles -""" -Log an error message if there are any running processes created by Distrubted, that were not shut down. -""" -function verify_no_running_processes() - if length(Distributed.procs()) != 1 - @error "Not all notebook processes were closed during tests!" Distributed.procs() - end -end +# """ +# Log an error message if there are any running processes created by Distrubted, that were not shut down. +# """ +# function verify_no_running_processes() +# if length(Distributed.procs()) != 1 +# @error "Not all notebook processes were closed during tests!" Distributed.procs() +# end +# end # We have our own registry for these test! Take a look at https://github.com/JuliaPluto/PlutoPkgTestRegistry#readme for more info about the test packages and their dependencies. diff --git a/test/packages/Basic.jl b/test/packages/Basic.jl index 5df3a3135b..09010eb05f 100644 --- a/test/packages/Basic.jl +++ b/test/packages/Basic.jl @@ -6,7 +6,7 @@ using Pluto.WorkspaceManager: _merge_notebook_compiler_options, poll import Pluto: update_save_run!, update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell, project_relative_path, SessionActions, load_notebook import Pluto.PkgUtils import Pluto.PkgCompat -import Distributed +import Malt @testset "Built-in Pkg" begin @@ -370,8 +370,7 @@ import Distributed end @testset "DrWatson cell" begin - 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false + 🍭 = ServerSession() notebook = Notebook([ Cell("using Plots"), @@ -421,12 +420,13 @@ import Distributed end @testset "File format -- Forwards compat" begin - # Using Distributed, we will create a new Julia process in which we install Pluto 0.14.7 (before PlutoPkg). We run the new notebook file on the old Pluto. - p = Distributed.addprocs(1) |> first + # Using Malt, create a Julia process in which we install Pluto 0.14.7 (before PlutoPkg). + # Run the new notebook file on the old Pluto. + test_worker = Malt.Worker() @test post_pkg_notebook isa String - Distributed.remotecall_eval(Main, p, quote + Malt.remote_eval_wait(Main, test_worker, quote path = tempname() write(path, $(post_pkg_notebook)) import Pkg @@ -435,35 +435,38 @@ import Distributed Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true end - Pkg.activate(mktempdir()) + Pkg.activate(;temp=true) Pkg.add(Pkg.PackageSpec(;name="Pluto",version=v"0.14.7")) + # Distributed is required for old Pluto to work! + Pkg.add("Distributed") + import Pluto + @info Pluto.PLUTO_VERSION @assert Pluto.PLUTO_VERSION == v"0.14.7" + end) + @test Malt.remote_eval_fetch(Main, test_worker, quote s = Pluto.ServerSession() - s.options.evaluation.workspace_use_distributed = false - nb = Pluto.SessionActions.open(s, path; run_async=false) - - nothing + nb.cells[2].errored == false end) # Cells that use Example will error because the package is not installed. - # @test Distributed.remotecall_eval(Main, p, quote + # @test Malt.remote_eval_fetch(Main, test_worker, quote # nb.cells[1].errored == false # end) - @test Distributed.remotecall_eval(Main, p, quote - nb.cells[2].errored == false - end) - # @test Distributed.remotecall_eval(Main, p, quote + # @test Malt.remote_eval_fetch(Main, test_worker, quote + # nb.cells[2].errored == false + # end) + # @test Malt.remote_eval_fetch(Main, test_worker, quote # nb.cells[3].errored == false # end) - # @test Distributed.remotecall_eval(Main, p, quote + # @test Malt.remote_eval_fetch(Main, test_worker, quote # nb.cells[3].output.body == "25" # end) - Distributed.rmprocs([p]) + Malt.stop(test_worker) end @testset "PkgUtils -- reset" begin diff --git a/test/runtests.jl b/test/runtests.jl index 3b3ce21a51..af377ef689 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,35 +3,22 @@ include("helpers.jl") # tests that start new processes: @timeit_include("compiletimes.jl") -verify_no_running_processes() if get(ENV, "PLUTO_TEST_ONLY_COMPILETIMES", nothing) == "true" print_timeroutput() exit(0) end @timeit_include("Events.jl") -verify_no_running_processes() @timeit_include("WorkspaceManager.jl") -verify_no_running_processes() @timeit_include("packages/Basic.jl") -verify_no_running_processes() @timeit_include("Bonds.jl") -verify_no_running_processes() @timeit_include("RichOutput.jl") -verify_no_running_processes() @timeit_include("React.jl") -verify_no_running_processes() @timeit_include("Dynamic.jl") -verify_no_running_processes() @timeit_include("MacroAnalysis.jl") -verify_no_running_processes() @timeit_include("Logging.jl") -verify_no_running_processes() @timeit_include("webserver.jl") -verify_no_running_processes() @timeit_include("Notebook.jl") -verify_no_running_processes() @timeit_include("Configuration.jl") -verify_no_running_processes() # tests that don't start new processes: @timeit_include("ReloadFromFile.jl") @@ -46,8 +33,6 @@ verify_no_running_processes() @timeit_include("Throttled.jl") @timeit_include("cell_disabling.jl") -verify_no_running_processes() - print_timeroutput() # TODO: test PlutoRunner functions like: diff --git a/test/webserver.jl b/test/webserver.jl index 9afbd3bdf1..1a643347be 100644 --- a/test/webserver.jl +++ b/test/webserver.jl @@ -26,7 +26,6 @@ using Pluto.WorkspaceManager: WorkspaceManager, poll options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, - workspace_use_distributed=true, require_secret_for_access=false, require_secret_for_open_links=false, base_url, @@ -69,7 +68,7 @@ end # without notebook at startup - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=true, require_secret_for_access=false, require_secret_for_open_links=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) 🍭 = Pluto.ServerSession(; options) server_task = @async Pluto.run(🍭) @@ -91,7 +90,7 @@ end # right now, the notebook was only added to the session and assigned an ID. Let's wait for it to get a process: @test poll(60) do - haskey(WorkspaceManager.workspaces, notebook.notebook_id) + haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) end sleep(1) From ec8d9897275945d7d03058150fb4dc5b0cc42c14 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 09:58:45 +0200 Subject: [PATCH 03/39] Discard changes to test/Bonds.jl --- test/Bonds.jl | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/test/Bonds.jl b/test/Bonds.jl index 1ed679aca4..21ab86d598 100644 --- a/test/Bonds.jl +++ b/test/Bonds.jl @@ -1,13 +1,15 @@ using Test import Pluto import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell -import Malt +import Distributed @testset "Bonds" begin 🍭 = ServerSession() - + 🍭.options.evaluation.workspace_use_distributed = false + @testset "AbstractPlutoDingetjes.jl" begin + 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ # 1 Cell(""" @@ -243,7 +245,7 @@ import Malt @test Pluto.possible_bond_values(🍭, notebook, :x_new) == [1,2,3] - @test Pluto.possible_bond_values(🍭, notebook, :asdfasdfx_new) == KeyError(:asdfasdfx_new) + @test_throws Exception Pluto.possible_bond_values(🍭, notebook, :asdfasdfx_new) @test Pluto.possible_bond_values(🍭, notebook, :pv1) == :NotGiven @test Pluto.possible_bond_values(🍭, notebook, :pv2) == :InfinitePossibilities @test Pluto.possible_bond_values(🍭, notebook, :pv3) == [1,2,3] @@ -297,13 +299,14 @@ import Malt WorkspaceManager.unmake_workspace((🍭, notebook)) + 🍭.options.evaluation.workspace_use_distributed = false # test that the notebook file is runnable: - test_worker = Malt.Worker() + test_proc = Distributed.addprocs(1)[1] - Malt.remote_eval_fetch(Main, test_worker, quote + Distributed.remotecall_eval(Main, test_proc, quote import Pkg try Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true @@ -311,16 +314,16 @@ import Malt Pkg.activate(mktempdir()) Pkg.add("AbstractPlutoDingetjes") end) - @test Malt.remote_eval_fetch(Main, test_worker, quote + @test Distributed.remotecall_eval(Main, test_proc, quote include($(notebook.path)) true end) - Malt.stop(test_worker) + Distributed.rmprocs(test_proc) end @testset "Dependent Bound Variables" begin 🍭 = ServerSession() - + 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ Cell(raw"""@bind x HTML("")"""), Cell(raw"""@bind y HTML("")"""), From a3ca342a83737057e4fa1864deaca8b2bfdb8683 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 09:58:54 +0200 Subject: [PATCH 04/39] Discard changes to test/Logging.jl --- test/Logging.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Logging.jl b/test/Logging.jl index cc3e9dde55..125f048f2f 100644 --- a/test/Logging.jl +++ b/test/Logging.jl @@ -5,6 +5,7 @@ using Pluto.WorkspaceManager: poll @testset "Logging" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook(Cell.([ "println(123)", From 88cb7b474822f70e823421251341fc9b58082059 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 09:59:05 +0200 Subject: [PATCH 05/39] Discard changes to test/ReloadFromFile.jl --- test/ReloadFromFile.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/ReloadFromFile.jl b/test/ReloadFromFile.jl index 98a9091c2f..01bf9358ca 100644 --- a/test/ReloadFromFile.jl +++ b/test/ReloadFromFile.jl @@ -1,6 +1,7 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager, SessionActions, save_notebook import Pluto.Configuration: Options, EvaluationOptions +import Distributed using Pluto.WorkspaceManager: poll import Pkg @@ -22,6 +23,7 @@ end retry(3) do 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false 🍭.options.server.auto_reload_from_file = true @@ -158,4 +160,4 @@ end @assert notebook.nbpkg_restart_required_msg !== nothing end @test true -end +end \ No newline at end of file From 1d34ff6d54a40d64559c4f27b0a376764ad9634a Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 09:59:27 +0200 Subject: [PATCH 06/39] Discard changes to test/React.jl --- test/React.jl | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/test/React.jl b/test/React.jl index bcb4600b21..eb1b5e0d7a 100644 --- a/test/React.jl +++ b/test/React.jl @@ -1,12 +1,15 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager import Pluto.Configuration: Options, EvaluationOptions -import Malt +import Distributed @testset "Reactivity" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false - @testset "Basic" begin + @testset "Basic $(parallel ? "distributed" : "single-process")" for parallel in [false, true] + 🍭.options.evaluation.workspace_use_distributed = parallel + notebook = Notebook([ Cell("x = 1"), Cell("y = x"), @@ -23,7 +26,7 @@ import Malt Cell("Distributed.myid()"), ]) - @test !haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) + @test !haskey(WorkspaceManager.workspaces, notebook.notebook_id) update_run!(🍭, notebook, notebook.cells[1:2]) @test notebook.cells[1].output.body == notebook.cells[2].output.body @@ -67,12 +70,18 @@ import Malt @test notebook.cells[6].output.body == "3" update_run!(🍭, notebook, notebook.cells[7:8]) - notebook.cells[8].output.body == "1" + @test if parallel + notebook.cells[8].output.body != string(Distributed.myid()) + else + notebook.cells[8].output.body == string(Distributed.myid()) + end WorkspaceManager.unmake_workspace((🍭, notebook); verbose=false) end + 🍭.options.evaluation.workspace_use_distributed = false + @testset "Mutliple assignments" begin notebook = Notebook([ Cell("x = 1"), @@ -198,6 +207,8 @@ import Malt # PlutoTest.jl is only working on Julia version >= 1.6 @testset "Test Firebasey" begin + 🍭.options.evaluation.workspace_use_distributed = true + file = tempname() write(file, read(normpath(Pluto.project_relative_path("src", "webserver", "Firebasey.jl")))) @@ -211,6 +222,7 @@ import Malt @test all(noerror, notebook.cells) WorkspaceManager.unmake_workspace((🍭, notebook)) + 🍭.options.evaluation.workspace_use_distributed = false end @testset "Pkg topology workarounds" begin @@ -359,6 +371,8 @@ import Malt end @testset "Reactive usings 4" begin + 🍭.options.evaluation.workspace_use_distributed = true + notebook = Notebook([ Cell("@sprintf \"double_december = %d\" double_december"), Cell("double_december = 2December"), @@ -383,6 +397,7 @@ import Malt @test notebook.cells[1].output.body == "\"double_december = 24\"" WorkspaceManager.unmake_workspace((🍭, notebook)) + 🍭.options.evaluation.workspace_use_distributed = false end @testset "Reactive usings 5" begin @@ -414,6 +429,8 @@ import Malt end @testset "Function dependencies" begin + 🍭.options.evaluation.workspace_use_distributed = true + notebook = Notebook(Cell.([ "a'b", "import LinearAlgebra", @@ -429,6 +446,7 @@ import Malt @test notebook.cells[1].output.body == "200" WorkspaceManager.unmake_workspace((🍭, notebook)) + 🍭.options.evaluation.workspace_use_distributed = false end @testset "Function use inv in its def but also has a method on inv" begin From 8b1179b329ae58d175a3ce5b64e93f64e976a6ae Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 09:59:37 +0200 Subject: [PATCH 07/39] Discard changes to test/RichOutput.jl --- test/RichOutput.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/RichOutput.jl b/test/RichOutput.jl index de631263fb..6675aa4008 100644 --- a/test/RichOutput.jl +++ b/test/RichOutput.jl @@ -6,7 +6,8 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @testset "Rich output" begin 🍭 = ServerSession() - + 🍭.options.evaluation.workspace_use_distributed = false + @testset "Tree viewer" begin @testset "Basics" begin notebook = Notebook([ @@ -133,6 +134,8 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @testset "Special arrays" begin + 🍭.options.evaluation.workspace_use_distributed = true + notebook = Notebook([ Cell("using OffsetArrays"), Cell("OffsetArray(zeros(3), 20:22)"), @@ -187,6 +190,7 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @test occursin("103", s) WorkspaceManager.unmake_workspace((🍭, notebook)) + 🍭.options.evaluation.workspace_use_distributed = false end @testset "Circular references" begin @@ -226,6 +230,7 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb end @testset "Table viewer" begin + 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ Cell("using DataFrames, Tables"), Cell("DataFrame()"), @@ -303,6 +308,7 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb # TODO: test lazy loading more rows/cols WorkspaceManager.unmake_workspace((🍭, notebook)) + 🍭.options.evaluation.workspace_use_distributed = false end begin From 31e98aca2f747b1b7769f109f37234e76a6363e8 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 09:59:44 +0200 Subject: [PATCH 08/39] Discard changes to test/Notebook.jl --- test/Notebook.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/Notebook.jl b/test/Notebook.jl index 089792c084..7e7d5480ec 100644 --- a/test/Notebook.jl +++ b/test/Notebook.jl @@ -203,6 +203,7 @@ end @testset "Cell Metadata" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false @testset "Disabling & Metadata" begin nb = cell_metadata_notebook() @@ -237,6 +238,7 @@ end @testset "Notebook Metadata" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false nb = notebook_metadata_notebook() update_run!(🍭, nb, nb.cells) @@ -261,6 +263,7 @@ end @testset "Skip as script" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false nb = skip_as_script_notebook() update_run!(🍭, nb, nb.cells) From 3fb2def5d3001c30fc8cdae8bd497132af58c90b Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 10:00:03 +0200 Subject: [PATCH 09/39] Discard changes to test/MacroAnalysis.jl --- test/MacroAnalysis.jl | 68 ++++++------------------------------------- 1 file changed, 9 insertions(+), 59 deletions(-) diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index 79d8151ffb..59e79157c3 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -5,6 +5,7 @@ import Memoize: @memoize @testset "Macro analysis" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false @testset "Base macro call" begin notebook = Notebook([ @@ -26,8 +27,6 @@ import Memoize: @memoize @test cell(3) |> noerror @test :Fruit ∈ notebook.topology.nodes[cell(3)].references - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 1" begin @@ -49,8 +48,6 @@ import Memoize: @memoize # Works on second time because of old workspace @test :x ∈ notebook.topology.nodes[cell(2)].definitions @test Symbol("@my_assign") ∈ notebook.topology.nodes[cell(2)].references - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 2" begin @@ -80,8 +77,6 @@ import Memoize: @memoize @test cell(1) |> noerror @test cell(2) |> noerror @test cell(3) |> noerror - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 3" begin @@ -105,8 +100,6 @@ import Memoize: @memoize update_run!(🍭, notebook, cell(1)) @test cell(2) |> noerror - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 4" begin @@ -121,8 +114,6 @@ import Memoize: @memoize update_run!(🍭, notebook, notebook.cells) @test Symbol("@my_assign") ∈ notebook.topology.nodes[cell(2)].references - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 5" begin @@ -139,8 +130,6 @@ import Memoize: @memoize @test :a ∉ references(2) @test :b ∉ references(2) @test :c ∉ references(2) - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 6" begin @@ -161,8 +150,6 @@ import Memoize: @memoize @test [Symbol("@my_macro"), :x, :y] ⊆ notebook.topology.nodes[cell(2)].references @test cell(3).output.body == "3" - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Function docs" begin @@ -179,8 +166,6 @@ import Memoize: @memoize @test :f ∈ notebook.topology.nodes[cell(1)].funcdefs_without_signatures @test :f ∈ notebook.topology.nodes[cell(2)].references - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Expr sanitization" begin @@ -226,8 +211,6 @@ import Memoize: @memoize @test cell(2).output.body == "true" @test all(noerror, notebook.cells) - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Reverse order" begin @@ -255,8 +238,6 @@ import Memoize: @memoize @test cell(2) |> noerror @test cell(3) |> noerror @test cell(1).output.body == "\"yay\"" - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "@a defines @b" begin @@ -291,8 +272,6 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(4) |> noerror @test cell(1).output.body == "42" - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Removing macros undefvar errors dependent cells" begin @@ -314,8 +293,6 @@ import Memoize: @memoize @test notebook.cells[end].errored @test occursinerror("UndefVarError: @m", notebook.cells[end]) - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Redefines macro with new SymbolsState" begin @@ -365,8 +342,6 @@ import Memoize: @memoize # See Run.jl#resolve_topology. @test cell(4).output.body == "42" @test cell(3).errored == true - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Reactive macro update does not invalidate the macro calls" begin @@ -403,8 +378,6 @@ import Memoize: @memoize @test cell(4).output.body != "42" @test cell(4).errored == true @test cell(5) |> noerror - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Explicitely running macrocalls updates the reactive node" begin @@ -436,8 +409,6 @@ import Memoize: @memoize @test cell(4).errored == true @test cell(5) |> noerror - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Implicitely running macrocalls updates the reactive node" begin @@ -476,8 +447,6 @@ import Memoize: @memoize # an explicit run of @b() must be done. @test cell(4).output.body == output_1 @test cell(5).errored == true - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Weird behavior" begin @@ -505,8 +474,6 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(3).output.body == "1234" - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @@ -523,8 +490,6 @@ import Memoize: @memoize # x ("@b(x)") was run. Should it? Maybe set a higher precedence to cells that define # macros inside the notebook. @test_broken noerror(notebook.cells[1]; verbose=false) - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "@a defines @b initial loading" begin @@ -548,8 +513,6 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(4) |> noerror @test cell(1).output.body == "42" - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macro with long compile time gets function wrapped" begin @@ -596,11 +559,11 @@ import Memoize: @memoize @test cell(1) |> noerror @test output_3 != cell(1).output.body - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macro Prefix" begin + 🍭.options.evaluation.workspace_use_distributed = true + notebook = Notebook(Cell.([ "@sprintf \"answer = %d\" x", "x = y+1", @@ -629,9 +592,9 @@ import Memoize: @memoize @test cell(1) |> noerror WorkspaceManager.unmake_workspace((🍭, notebook)) + 🍭.options.evaluation.workspace_use_distributed = false end - @testset "Package macro 1" begin notebook = Notebook([ Cell("using Dates"), @@ -657,11 +620,11 @@ import Memoize: @memoize @test cell(1) |> noerror @test cell(2) |> noerror - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Package macro 2" begin + 🍭.options.evaluation.workspace_use_distributed = true + notebook = Notebook([ Cell("z = x^2 + y"), Cell("@variables x y"), @@ -711,6 +674,8 @@ import Memoize: @memoize @test cell(2) |> noerror WorkspaceManager.unmake_workspace((🍭, notebook)) + + 🍭.options.evaluation.workspace_use_distributed = false end @testset "Previous workspace for unknowns" begin @@ -734,8 +699,6 @@ import Memoize: @memoize module_from_cell3 = cell(3).output.body @test module_from_cell2 == module_from_cell3 - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Definitions" begin @@ -762,8 +725,6 @@ import Memoize: @memoize @test ":world" == cell(3).output.body @test ":world" == cell(4).output.body - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Is just text macros" begin @@ -778,8 +739,6 @@ import Memoize: @memoize update_run!(🍭, notebook, notebook.cells) @test isempty(notebook.topology.unresolved_cells) - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macros using import" begin @@ -798,8 +757,6 @@ import Memoize: @memoize @test :option_type ∈ notebook.topology.nodes[cell(1)].references @test cell(1) |> noerror - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "GlobalRefs in macros should be respected" begin @@ -824,8 +781,6 @@ import Memoize: @memoize @test all(cell.([1,2,3]) .|> noerror) @test cell(3).output.body == "20" - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "GlobalRefs shouldn't break unreached undefined references" begin @@ -851,8 +806,6 @@ import Memoize: @memoize @test all(cell.([1,2]) .|> noerror) @test cell(2).output.body == ":this_should_be_returned" - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Doc strings" begin @@ -925,12 +878,11 @@ import Memoize: @memoize update_run!(🍭, notebook, bool) @test !occursin("An empty conjugate", bool.output.body) @test occursin("complex conjugate", bool.output.body) - - WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Delete methods from macros" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("using Memoize"), @@ -970,7 +922,5 @@ import Memoize: @memoize @test occursinerror("UndefVarError: custom_func", cell(4)) @test :memoized_func ∉ notebook.topology.nodes[cell(5)].funcdefs_without_signatures @test occursinerror("UndefVarError: memoized_func", cell(6)) - - WorkspaceManager.unmake_workspace((🍭, notebook)) end end From 080bd40fa49907c43eea225032fe609b6311bdb0 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 10:00:17 +0200 Subject: [PATCH 10/39] Discard changes to test/cell_disabling.jl --- test/cell_disabling.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cell_disabling.jl b/test/cell_disabling.jl index 420389de5d..f5f9575dd3 100644 --- a/test/cell_disabling.jl +++ b/test/cell_disabling.jl @@ -8,6 +8,7 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook, set_disa @testset "Cell Disabling" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("const a = 1") From 00cc36016d464dd33d4ad1fe1521484232dead71 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 10:00:24 +0200 Subject: [PATCH 11/39] Discard changes to test/compiletimes.jl --- test/compiletimes.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/compiletimes.jl b/test/compiletimes.jl index 9f43543aad..c1eef7d165 100644 --- a/test/compiletimes.jl +++ b/test/compiletimes.jl @@ -30,6 +30,7 @@ end 🍭 = Pluto.ServerSession() 🍭.options.server.disable_writing_notebook_files = true +🍭.options.evaluation.workspace_use_distributed = false path = joinpath(pkgdir(Pluto), "sample", "Basic.jl") @@ -44,7 +45,7 @@ HTTP.get("http://github.com") @timeit TOUT "Pluto.run" server_task = @eval let port = 13435 - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) 🍭 = Pluto.ServerSession(; options) server_task = @async Pluto.run(🍭) From bca2d58fcf6ba3dc20aa6b8ac3b1da6efc5fc22a Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 10:01:26 +0200 Subject: [PATCH 12/39] Discard changes to test/webserver.jl --- test/webserver.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/webserver.jl b/test/webserver.jl index 1a643347be..9afbd3bdf1 100644 --- a/test/webserver.jl +++ b/test/webserver.jl @@ -26,6 +26,7 @@ using Pluto.WorkspaceManager: WorkspaceManager, poll options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, + workspace_use_distributed=true, require_secret_for_access=false, require_secret_for_open_links=false, base_url, @@ -68,7 +69,7 @@ end # without notebook at startup - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=true, require_secret_for_access=false, require_secret_for_open_links=false) 🍭 = Pluto.ServerSession(; options) server_task = @async Pluto.run(🍭) @@ -90,7 +91,7 @@ end # right now, the notebook was only added to the session and assigned an ID. Let's wait for it to get a process: @test poll(60) do - haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) + haskey(WorkspaceManager.workspaces, notebook.notebook_id) end sleep(1) From 2afc06a13ac5f3fe032eb3d5a03c47346a478992 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 10:01:40 +0200 Subject: [PATCH 13/39] Discard changes to src/Configuration.jl --- src/Configuration.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Configuration.jl b/src/Configuration.jl index 70974a83f7..920782d25d 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -128,6 +128,7 @@ Note that Pluto is quickly evolving software, maintained by designers, educators end const RUN_NOTEBOOK_ON_LOAD_DEFAULT = true +const WORKSPACE_USE_DISTRIBUTED_DEFAULT = true const LAZY_WORKSPACE_CREATION_DEFAULT = false const CAPTURE_STDOUT_DEFAULT = true const WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT = nothing @@ -139,12 +140,14 @@ Options to change Pluto's evaluation behaviour during internal testing and by do These options are not intended to be changed during normal use. - `run_notebook_on_load::Bool = $RUN_NOTEBOOK_ON_LOAD_DEFAULT` Whether to evaluate a notebook on load. +- `workspace_use_distributed::Bool = $WORKSPACE_USE_DISTRIBUTED_DEFAULT` Whether to start notebooks in a separate process. - `lazy_workspace_creation::Bool = $LAZY_WORKSPACE_CREATION_DEFAULT` - `capture_stdout::Bool = $CAPTURE_STDOUT_DEFAULT` - `workspace_custom_startup_expr::Union{Nothing,Expr} = $WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT` An expression to be evaluated in the workspace process before running notebook code. """ @option mutable struct EvaluationOptions run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT + workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT @@ -251,6 +254,7 @@ function from_flat_kwargs(; require_secret_for_open_links::Bool = REQUIRE_SECRET_FOR_OPEN_LINKS_DEFAULT, require_secret_for_access::Bool = REQUIRE_SECRET_FOR_ACCESS_DEFAULT, run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT, + workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT, lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT, capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT, workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT, @@ -290,6 +294,7 @@ function from_flat_kwargs(; ) evaluation = EvaluationOptions(; run_notebook_on_load, + workspace_use_distributed, lazy_workspace_creation, capture_stdout, workspace_custom_startup_expr, From 3d0e8929f40cbb6a2c02b819f52bc3b799d8b386 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 10:07:07 +0200 Subject: [PATCH 14/39] small updates --- Project.toml | 3 +-- src/evaluation/WorkspaceManager.jl | 34 +++++++++++++++--------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/Project.toml b/Project.toml index 771750ce4a..d0dc97377d 100644 --- a/Project.toml +++ b/Project.toml @@ -37,11 +37,10 @@ FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.0.2" HypertextLiteral = "0.7, 0.8, 0.9" MIMEs = "0.1" -Malt = "^0.7.0" MsgPack = "1.1" PrecompileSignatures = "3" RegistryInstances = "0.1" -RelocatableFolders = "0.1, 0.2, 0.3" +RelocatableFolders = "0.1, 0.2, 0.3, 1" Tables = "1" URIs = "1.3" julia = "^1.6" diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index bf85814c0b..affb8baf24 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -14,7 +14,7 @@ Each notebook gets at most one `Workspace` at any time, but it can also have no (it cannot `eval` code in this case). """ Base.@kwdef mutable struct Workspace - worker::Malt.Worker + worker::Malt.AbstractWorker notebook_id::UUID discarded::Bool=false remote_log_channel::Channel @@ -48,9 +48,9 @@ function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false @debug "Creating workspace process" notebook.path length(notebook.cells) worker = create_workspaceprocess(;compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) - Malt.remote_eval_wait(Main, worker, session.options.evaluation.workspace_custom_startup_expr) + Malt.remote_eval_wait(worker, session.options.evaluation.workspace_custom_startup_expr) - Malt.remote_eval_wait(Main, worker, quote + Malt.remote_eval_wait(worker, quote PlutoRunner.notebook_id[] = $(notebook.notebook_id) end) @@ -68,7 +68,7 @@ function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false module_name = create_emptyworkspacemodule(worker) - original_LOAD_PATH, original_ACTIVE_PROJECT = Malt.remote_eval_fetch(Main, worker, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) + original_LOAD_PATH, original_ACTIVE_PROJECT = Malt.remote_eval_fetch(worker, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) workspace = Workspace(; worker, @@ -102,7 +102,7 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) new_LP = enabled ? ["@", "@stdlib"] : workspace.original_LOAD_PATH new_AP = enabled ? PkgCompat.env_dir(notebook.nbpkg_ctx) : workspace.original_ACTIVE_PROJECT - Malt.remote_eval_wait(Main, workspace.worker, quote + Malt.remote_eval_wait(workspace.worker, quote copy!(LOAD_PATH, $(new_LP)) Base.ACTIVE_PROJECT[] = $(new_AP) end) @@ -217,13 +217,13 @@ end function possible_bond_values(session_notebook::SN, n::Symbol; get_length::Bool=false) workspace = get_workspace(session_notebook) - Malt.remote_eval_fetch(Main, workspace.worker, quote + Malt.remote_eval_fetch(workspace.worker, quote PlutoRunner.possible_bond_values($(QuoteNode(n)); get_length=$(get_length)) end) end function create_emptyworkspacemodule(worker::Malt.Worker)::Symbol - Malt.remote_eval_fetch(Main, worker, quote + Malt.remote_eval_fetch(worker, quote PlutoRunner.increment_current_module() end) end @@ -233,10 +233,10 @@ end # compiler configurations passed to it should be resolved before this function create_workspaceprocess(;compiler_options=CompilerOptions())::Malt.Worker worker = Malt.Worker(;exeflags=_convert_to_flags(compiler_options)) - Malt.remote_eval_wait(Main, worker, process_preamble) + Malt.remote_eval_wait(worker, process_preamble) # so that we NEVER break the workspace with an interrupt 🤕 - Malt.remote_eval(Main, worker, quote + Malt.remote_eval(worker, quote while true try wait() @@ -370,7 +370,7 @@ function eval_format_fetch_in_workspace( # A try block (on this process) to catch an InterruptException take!(workspace.dowork_token) early_result = try - Malt.remote_eval_wait(Main, workspace.worker, quote + Malt.remote_eval_wait(workspace.worker, quote PlutoRunner.run_expression( getfield(Main, $(QuoteNode(workspace.module_name))), $(QuoteNode(expr)), @@ -401,7 +401,7 @@ end function eval_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Malt.remote_eval_wait(Main, workspace.worker, quote + Malt.remote_eval_wait(workspace.worker, quote Core.eval($(workspace.module_name), $(QuoteNode(expr))) end) nothing @@ -421,7 +421,7 @@ function format_fetch_in_workspace( # we format the cell output on the worker, and fetch the formatted output. withtoken(workspace.dowork_token) do try - Malt.remote_eval_fetch(Main, workspace.worker, quote + Malt.remote_eval_fetch(workspace.worker, quote PlutoRunner.formatted_result_of( $(workspace.notebook_id), $cell_id, @@ -440,7 +440,7 @@ end function collect_soft_definitions(session_notebook::SN, modules::Set{Expr}) workspace = get_workspace(session_notebook) - Malt.remote_eval_fetch(Main, workspace.worker, quote + Malt.remote_eval_fetch(workspace.worker, quote PlutoRunner.collect_soft_definitions($(workspace.module_name), $modules) end) end @@ -449,7 +449,7 @@ function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macroca workspace = get_workspace(session_notebook) module_name = module_name === nothing ? workspace.module_name : module_name - Malt.remote_eval_fetch(Main, workspace.worker, quote + Malt.remote_eval_fetch(workspace.worker, quote try (true, PlutoRunner.try_macroexpand($module_name, $(workspace.notebook_id), $cell_id, $(QuoteNode(macrocall)))) catch error @@ -469,7 +469,7 @@ end function eval_fetch_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Malt.remote_eval_fetch(Main, workspace.worker, quote + Malt.remote_eval_fetch(workspace.worker, quote Core.eval($(workspace.module_name), $(QuoteNode(expr))) end) end @@ -477,7 +477,7 @@ end function do_reimports(session_notebook::Union{SN,Workspace}, module_imports_to_move::Set{Expr}) workspace = get_workspace(session_notebook) - Malt.remote_eval_wait(Main, workspace.worker, quote + Malt.remote_eval_wait(workspace.worker, quote PlutoRunner.do_reimports($(workspace.module_name), $module_imports_to_move) end) end @@ -499,7 +499,7 @@ function move_vars( workspace = get_workspace(session_notebook) new_workspace_name = something(new_workspace_name, workspace.module_name) - Malt.remote_eval_wait(Main, workspace.worker, quote + Malt.remote_eval_wait(workspace.worker, quote PlutoRunner.move_vars( $(QuoteNode(old_workspace_name)), $(QuoteNode(new_workspace_name)), From 2195960ad9de5754bd8146cb5265689336346baf Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 10:39:24 +0200 Subject: [PATCH 15/39] bring back workspace_use_distributed and workspace_use_distributed_stdlib --- src/Configuration.jl | 5 +++ src/evaluation/WorkspaceManager.jl | 59 +++++++++++++++++++++--------- test/React.jl | 2 +- test/webserver.jl | 2 +- 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/src/Configuration.jl b/src/Configuration.jl index 920782d25d..ea36fa5c16 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -129,6 +129,7 @@ end const RUN_NOTEBOOK_ON_LOAD_DEFAULT = true const WORKSPACE_USE_DISTRIBUTED_DEFAULT = true +const WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT = true const LAZY_WORKSPACE_CREATION_DEFAULT = false const CAPTURE_STDOUT_DEFAULT = true const WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT = nothing @@ -141,6 +142,7 @@ These options are not intended to be changed during normal use. - `run_notebook_on_load::Bool = $RUN_NOTEBOOK_ON_LOAD_DEFAULT` Whether to evaluate a notebook on load. - `workspace_use_distributed::Bool = $WORKSPACE_USE_DISTRIBUTED_DEFAULT` Whether to start notebooks in a separate process. +- `workspace_use_distributed_stdlib::Bool = $WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT` Should we use the Distributed stdlib to run processes? This has been replaced by Malt.jl, but you can use this option to get the old behaviour. Option be removed in the future. - `lazy_workspace_creation::Bool = $LAZY_WORKSPACE_CREATION_DEFAULT` - `capture_stdout::Bool = $CAPTURE_STDOUT_DEFAULT` - `workspace_custom_startup_expr::Union{Nothing,Expr} = $WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT` An expression to be evaluated in the workspace process before running notebook code. @@ -148,6 +150,7 @@ These options are not intended to be changed during normal use. @option mutable struct EvaluationOptions run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT + workspace_use_distributed_stdlib::Bool = WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT @@ -255,6 +258,7 @@ function from_flat_kwargs(; require_secret_for_access::Bool = REQUIRE_SECRET_FOR_ACCESS_DEFAULT, run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT, workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT, + workspace_use_distributed_stdlib::Bool = WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT, lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT, capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT, workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT, @@ -295,6 +299,7 @@ function from_flat_kwargs(; evaluation = EvaluationOptions(; run_notebook_on_load, workspace_use_distributed, + workspace_use_distributed_stdlib, lazy_workspace_creation, capture_stdout, workspace_custom_startup_expr, diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index affb8baf24..b292c70d9b 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -7,6 +7,7 @@ import ..Configuration: CompilerOptions, _merge_notebook_compiler_options, _conv import ..Pluto.ExpressionExplorer: FunctionName import ..PlutoRunner import Malt +import Malt.Distributed """ Contains the Julia process to evaluate code in. @@ -17,7 +18,7 @@ Base.@kwdef mutable struct Workspace worker::Malt.AbstractWorker notebook_id::UUID discarded::Bool=false - remote_log_channel::Channel + remote_log_channel::Union{Distributed.RemoteChannel,Channel} module_name::Symbol dowork_token::Token=Token() nbpkg_was_active::Bool=false @@ -29,7 +30,7 @@ end const SN = Tuple{ServerSession, Notebook} "These expressions get evaluated whenever a new `Workspace` process is created." -const process_preamble = quote +process_preamble() = quote ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0) include($(project_relative_path(joinpath("src", "runner"), "Loader.jl"))) ENV["GKSwstype"] = "nul" @@ -44,9 +45,17 @@ const discarded_workspaces = Set{UUID}() "Create a workspace for the notebook, optionally in the main process." function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false)::Workspace is_offline_renderer || (notebook.process_status = ProcessStatus.starting) - + + WorkerType = if is_offline_renderer || !session.options.evaluation.workspace_use_distributed + Malt.InProcessWorker + elseif session.options.evaluation.workspace_use_distributed_stdlib + Malt.DistributedStdlibWorker + else + Malt.Worker + end + @debug "Creating workspace process" notebook.path length(notebook.cells) - worker = create_workspaceprocess(;compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) + worker = create_workspaceprocess(WorkerType; compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) Malt.remote_eval_wait(worker, session.options.evaluation.workspace_custom_startup_expr) @@ -222,7 +231,7 @@ function possible_bond_values(session_notebook::SN, n::Symbol; get_length::Bool= end) end -function create_emptyworkspacemodule(worker::Malt.Worker)::Symbol +function create_emptyworkspacemodule(worker::Malt.AbstractWorker)::Symbol Malt.remote_eval_fetch(worker, quote PlutoRunner.increment_current_module() end) @@ -231,20 +240,34 @@ end # NOTE: this function only start a worker process using given # compiler options, it does not resolve paths for notebooks # compiler configurations passed to it should be resolved before this -function create_workspaceprocess(;compiler_options=CompilerOptions())::Malt.Worker - worker = Malt.Worker(;exeflags=_convert_to_flags(compiler_options)) - Malt.remote_eval_wait(worker, process_preamble) - - # so that we NEVER break the workspace with an interrupt 🤕 - Malt.remote_eval(worker, quote - while true - try - wait() - catch end +function create_workspaceprocess(WorkerType; compiler_options=CompilerOptions())::Malt.AbstractWorker + if WorkerType === Malt.InProcessWorker + worker = WorkerType() + + if !(isdefined(Main, :PlutoRunner) && Main.PlutoRunner isa Module) + # we make PlutoRunner available in Main, right now it's only defined inside this Pluto module. + Malt.remote_eval_wait(Main, worker, quote + PlutoRunner = $(PlutoRunner) + end) end - end) - - worker + + worker + else + worker = WorkerType(; exeflags=_convert_to_flags(compiler_options)) + + Malt.remote_eval_wait(worker, process_preamble()) + + # so that we NEVER break the workspace with an interrupt 🤕 + Malt.remote_eval(worker, quote + while true + try + wait() + catch end + end + end) + + worker + end end """ diff --git a/test/React.jl b/test/React.jl index eb1b5e0d7a..eff57fc685 100644 --- a/test/React.jl +++ b/test/React.jl @@ -26,7 +26,7 @@ import Distributed Cell("Distributed.myid()"), ]) - @test !haskey(WorkspaceManager.workspaces, notebook.notebook_id) + @test !haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) update_run!(🍭, notebook, notebook.cells[1:2]) @test notebook.cells[1].output.body == notebook.cells[2].output.body diff --git a/test/webserver.jl b/test/webserver.jl index 9afbd3bdf1..0d94e1887c 100644 --- a/test/webserver.jl +++ b/test/webserver.jl @@ -91,7 +91,7 @@ end # right now, the notebook was only added to the session and assigned an ID. Let's wait for it to get a process: @test poll(60) do - haskey(WorkspaceManager.workspaces, notebook.notebook_id) + haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) end sleep(1) From b2bb80baf2516d56735d527b87b3effe925a4ceb Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sun, 9 Jul 2023 11:31:24 +0200 Subject: [PATCH 16/39] fixiefix its broken but close --- src/evaluation/Run.jl | 6 ++++++ src/evaluation/WorkspaceManager.jl | 8 ++++++-- test/packages/Basic.jl | 23 +++++++++++++++++++---- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index ff106ef5e7..d50142f874 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -339,8 +339,10 @@ function update_save_run!( old = notebook.topology new = notebook.topology = updated_topology(old, notebook, cells) # macros are not yet resolved + @info 80923 update_dependency_cache!(notebook) save && save_notebook(session, notebook) + @info 91832098732 # _assume `prerender_text == false` if you want to skip some details_ to_run_online = if !prerender_text @@ -391,8 +393,12 @@ function update_save_run!( to_run_online = to_run_online ∩ setup_cells end + @info 87389173 maybe_async(run_async) do + @info 67236723678 withtoken(notebook.executetoken) do + @info 32167236723678 + sync_nbpkg(session, notebook, old, new; save=(save && !session.options.server.disable_writing_notebook_files), take_token=false) if !(isempty(to_run_online) && session.options.evaluation.lazy_workspace_creation) && will_run_code(notebook) # not async because that would be double async diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index b292c70d9b..1fafbf7315 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -108,6 +108,10 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) workspace.discarded && return workspace.nbpkg_was_active = enabled + if workspace.worker isa Malt.InProcessWorker + # Not supported + return + end new_LP = enabled ? ["@", "@stdlib"] : workspace.original_LOAD_PATH new_AP = enabled ? PkgCompat.env_dir(notebook.nbpkg_ctx) : workspace.original_ACTIVE_PROJECT @@ -117,7 +121,7 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) end) end -function start_relaying_self_updates((session, notebook)::SN, run_channel::Channel) +function start_relaying_self_updates((session, notebook)::SN, run_channel) while true try next_run_uuid = take!(run_channel) @@ -133,7 +137,7 @@ function start_relaying_self_updates((session, notebook)::SN, run_channel::Chann end end -function start_relaying_logs((session, notebook)::SN, log_channel::Channel) +function start_relaying_logs((session, notebook)::SN, log_channel) update_throttled, flush_throttled = Pluto.throttled(0.1) do Pluto.send_notebook_changes!(Pluto.ClientRequest(session=session, notebook=notebook)) end diff --git a/test/packages/Basic.jl b/test/packages/Basic.jl index 09010eb05f..06773366fa 100644 --- a/test/packages/Basic.jl +++ b/test/packages/Basic.jl @@ -14,11 +14,12 @@ import Malt # We have our own registry for these test! Take a look at https://github.com/JuliaPluto/PlutoPkgTestRegistry#readme for more info about the test packages and their dependencies. Pkg.Registry.add(pluto_test_registry_spec) - @testset "Basic" begin + @testset "Basic $(use_distributed_stdlib ? "Distributed" : "Malt")" for use_distributed_stdlib in (false, true) 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed_stdlib = use_distributed_stdlib # See https://github.com/JuliaPluto/PlutoPkgTestRegistry - +@info 1 notebook = Notebook([ Cell("import PlutoPkgTestA"), # cell 1 Cell("PlutoPkgTestA.MY_VERSION |> Text"), @@ -36,8 +37,10 @@ import Malt @test !notebook.nbpkg_ctx_instantiated - update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D - @test noerror(notebook.cells[1]) +@info 2 +update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D +@info 3 +@test noerror(notebook.cells[1]) @test noerror(notebook.cells[2]) @test noerror(notebook.cells[7]) @test noerror(notebook.cells[8]) @@ -51,6 +54,7 @@ import Malt last_install_time = notebook.nbpkg_install_time_ns terminals = notebook.nbpkg_terminal_outputs + @info 4 @test haskey(terminals, "PlutoPkgTestA") @test haskey(terminals, "PlutoPkgTestD") @@ -64,6 +68,7 @@ import Malt @test PkgCompat.get_manifest_version(notebook.nbpkg_ctx, "PlutoPkgTestA") == v"0.3.1" @test PkgCompat.get_manifest_version(notebook.nbpkg_ctx, "PlutoPkgTestD") == v"0.1.0" + @info 5 old_A_terminal = deepcopy(terminals["PlutoPkgTestA"]) # @show old_A_terminal @@ -80,6 +85,7 @@ import Malt @test notebook.nbpkg_install_time_ns > last_install_time @test notebook.nbpkg_busy_packages |> isempty last_install_time = notebook.nbpkg_install_time_ns + @info 6 @test haskey(terminals, "PlutoPkgTestB") @test terminals["PlutoPkgTestA"] == terminals["PlutoPkgTestD"] == old_A_terminal @@ -91,6 +97,7 @@ import Malt # running the 5th cell will import PlutoPkgTestC, putting a 0.2 compatibility bound on PlutoPkgTestA. This means that a notebook restart is required, since PlutoPkgTestA was already loaded at version 0.3.1. update_save_run!(🍭, notebook, notebook.cells[[5, 6]]) + @info 7 @test noerror(notebook.cells[5]) @test noerror(notebook.cells[6]) @@ -105,6 +112,7 @@ import Malt # running cells again should persist the restart message update_save_run!(🍭, notebook, notebook.cells[1:8]) + @info 8 @test notebook.nbpkg_restart_required_msg !== nothing Pluto.response_restart_process(Pluto.ClientRequest( @@ -112,6 +120,7 @@ import Malt notebook=notebook, ); run_async=false) + @info 9 # @test_nowarn SessionActions.shutdown(🍭, notebook; keep_in_session=true, async=true) # @test_nowarn update_save_run!(🍭, notebook, notebook.cells[1:8]; , save=true) @@ -137,6 +146,7 @@ import Malt update_save_run!(🍭, notebook, notebook.cells[9]) + @info 10 @test noerror(notebook.cells[9]) @test notebook.nbpkg_ctx !== nothing @@ -149,10 +159,12 @@ import Malt @test notebook.cells[10].errored == true + @info 11 ptoml_contents() = PkgCompat.read_project_file(notebook) mtoml_contents() = PkgCompat.read_manifest_file(notebook) nb_contents() = read(notebook.path, String) + @info 12 @testset "Project & Manifest stored in notebook" begin @@ -182,6 +194,7 @@ import Malt ## remove `import Dates` setcode!(notebook.cells[9], "") update_save_run!(🍭, notebook, notebook.cells[9]) + @info 13 # removing a stdlib does not require a restart @test noerror(notebook.cells[9]) @@ -204,8 +217,10 @@ import Malt @test count("PlutoPkgTestD", ptoml_contents()) == 0 + @info 14 WorkspaceManager.unmake_workspace((🍭, notebook)) + @info 15 end simple_import_path = joinpath(@__DIR__, "simple_import.jl") From 0e8f3b4cef8aee9c01d2a9560e891353382e6590 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sat, 15 Jul 2023 12:53:09 +0200 Subject: [PATCH 17/39] tiny fix --- Project.toml | 1 + src/evaluation/Run.jl | 5 ----- src/evaluation/WorkspaceManager.jl | 2 +- test/packages/Basic.jl | 23 +++++------------------ 4 files changed, 7 insertions(+), 24 deletions(-) diff --git a/Project.toml b/Project.toml index d0dc97377d..90411a6dc1 100644 --- a/Project.toml +++ b/Project.toml @@ -37,6 +37,7 @@ FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.0.2" HypertextLiteral = "0.7, 0.8, 0.9" MIMEs = "0.1" +Malt = "0.8.2" MsgPack = "1.1" PrecompileSignatures = "3" RegistryInstances = "0.1" diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index d50142f874..65e4bd01b2 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -339,10 +339,8 @@ function update_save_run!( old = notebook.topology new = notebook.topology = updated_topology(old, notebook, cells) # macros are not yet resolved - @info 80923 update_dependency_cache!(notebook) save && save_notebook(session, notebook) - @info 91832098732 # _assume `prerender_text == false` if you want to skip some details_ to_run_online = if !prerender_text @@ -393,11 +391,8 @@ function update_save_run!( to_run_online = to_run_online ∩ setup_cells end - @info 87389173 maybe_async(run_async) do - @info 67236723678 withtoken(notebook.executetoken) do - @info 32167236723678 sync_nbpkg(session, notebook, old, new; save=(save && !session.options.server.disable_writing_notebook_files), take_token=false) if !(isempty(to_run_online) && session.options.evaluation.lazy_workspace_creation) && will_run_code(notebook) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 1fafbf7315..6e8f39a643 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -18,7 +18,7 @@ Base.@kwdef mutable struct Workspace worker::Malt.AbstractWorker notebook_id::UUID discarded::Bool=false - remote_log_channel::Union{Distributed.RemoteChannel,Channel} + remote_log_channel::Union{Distributed.RemoteChannel,AbstractChannel} module_name::Symbol dowork_token::Token=Token() nbpkg_was_active::Bool=false diff --git a/test/packages/Basic.jl b/test/packages/Basic.jl index 06773366fa..3016dda032 100644 --- a/test/packages/Basic.jl +++ b/test/packages/Basic.jl @@ -19,7 +19,6 @@ import Malt 🍭.options.evaluation.workspace_use_distributed_stdlib = use_distributed_stdlib # See https://github.com/JuliaPluto/PlutoPkgTestRegistry -@info 1 notebook = Notebook([ Cell("import PlutoPkgTestA"), # cell 1 Cell("PlutoPkgTestA.MY_VERSION |> Text"), @@ -37,10 +36,10 @@ import Malt @test !notebook.nbpkg_ctx_instantiated -@info 2 -update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D -@info 3 -@test noerror(notebook.cells[1]) + + update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D + + @test noerror(notebook.cells[1]) @test noerror(notebook.cells[2]) @test noerror(notebook.cells[7]) @test noerror(notebook.cells[8]) @@ -54,7 +53,6 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D last_install_time = notebook.nbpkg_install_time_ns terminals = notebook.nbpkg_terminal_outputs - @info 4 @test haskey(terminals, "PlutoPkgTestA") @test haskey(terminals, "PlutoPkgTestD") @@ -68,8 +66,7 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D @test PkgCompat.get_manifest_version(notebook.nbpkg_ctx, "PlutoPkgTestA") == v"0.3.1" @test PkgCompat.get_manifest_version(notebook.nbpkg_ctx, "PlutoPkgTestD") == v"0.1.0" - @info 5 - + old_A_terminal = deepcopy(terminals["PlutoPkgTestA"]) # @show old_A_terminal @@ -85,7 +82,6 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D @test notebook.nbpkg_install_time_ns > last_install_time @test notebook.nbpkg_busy_packages |> isempty last_install_time = notebook.nbpkg_install_time_ns - @info 6 @test haskey(terminals, "PlutoPkgTestB") @test terminals["PlutoPkgTestA"] == terminals["PlutoPkgTestD"] == old_A_terminal @@ -97,7 +93,6 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D # running the 5th cell will import PlutoPkgTestC, putting a 0.2 compatibility bound on PlutoPkgTestA. This means that a notebook restart is required, since PlutoPkgTestA was already loaded at version 0.3.1. update_save_run!(🍭, notebook, notebook.cells[[5, 6]]) - @info 7 @test noerror(notebook.cells[5]) @test noerror(notebook.cells[6]) @@ -112,7 +107,6 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D # running cells again should persist the restart message update_save_run!(🍭, notebook, notebook.cells[1:8]) - @info 8 @test notebook.nbpkg_restart_required_msg !== nothing Pluto.response_restart_process(Pluto.ClientRequest( @@ -120,7 +114,6 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D notebook=notebook, ); run_async=false) - @info 9 # @test_nowarn SessionActions.shutdown(🍭, notebook; keep_in_session=true, async=true) # @test_nowarn update_save_run!(🍭, notebook, notebook.cells[1:8]; , save=true) @@ -146,7 +139,6 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D update_save_run!(🍭, notebook, notebook.cells[9]) - @info 10 @test noerror(notebook.cells[9]) @test notebook.nbpkg_ctx !== nothing @@ -159,12 +151,10 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D @test notebook.cells[10].errored == true - @info 11 ptoml_contents() = PkgCompat.read_project_file(notebook) mtoml_contents() = PkgCompat.read_manifest_file(notebook) nb_contents() = read(notebook.path, String) - @info 12 @testset "Project & Manifest stored in notebook" begin @@ -194,7 +184,6 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D ## remove `import Dates` setcode!(notebook.cells[9], "") update_save_run!(🍭, notebook, notebook.cells[9]) - @info 13 # removing a stdlib does not require a restart @test noerror(notebook.cells[9]) @@ -217,10 +206,8 @@ update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D @test count("PlutoPkgTestD", ptoml_contents()) == 0 - @info 14 WorkspaceManager.unmake_workspace((🍭, notebook)) - @info 15 end simple_import_path = joinpath(@__DIR__, "simple_import.jl") From e3294685f256af19f68e6f34ab39cde6a68b0499 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Sat, 15 Jul 2023 16:08:48 +0200 Subject: [PATCH 18/39] hmmmmm --- test/WorkspaceManager.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/WorkspaceManager.jl b/test/WorkspaceManager.jl index 1d952d5019..72d027f954 100644 --- a/test/WorkspaceManager.jl +++ b/test/WorkspaceManager.jl @@ -53,6 +53,7 @@ import Malt Sys.iswindows() || @testset "Pluto inside Pluto" begin 🍭 = ServerSession() 🍭.options.evaluation.capture_stdout = false + 🍭.options.evaluation.workspace_use_distributed_stdlib = false notebook = Notebook([ Cell("""begin @@ -63,7 +64,10 @@ import Malt import Pluto end"""), Cell(""" - s = Pluto.ServerSession() + begin + s = Pluto.ServerSession() + s.options.evaluation.workspace_use_distributed_stdlib = false + end """), Cell(""" nb = Pluto.SessionActions.open(s, Pluto.project_relative_path("sample", "Tower of Hanoi.jl"); run_async=false, as_sample=true) From 42793521324cb83021dac7e27a5fae5b9a4876f1 Mon Sep 17 00:00:00 2001 From: Panagiotis Georgakopoulos Date: Wed, 9 Aug 2023 12:03:41 +0300 Subject: [PATCH 19/39] fix: ref: https://github.com/esm-dev/esm.sh/issues/703 --- frontend/components/CellInput/lezer_template.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/components/CellInput/lezer_template.js b/frontend/components/CellInput/lezer_template.js index 9be4f144d0..98310fa0b0 100644 --- a/frontend/components/CellInput/lezer_template.js +++ b/frontend/components/CellInput/lezer_template.js @@ -2,7 +2,7 @@ import { julia_andrey, NodeProp, syntaxTree, Text } from "../../imports/Codemirr import lodash from "../../imports/lodash.js" // @ts-ignore -import ManyKeysWeakMap from "https://cdn.esm.sh/v64/many-keys-weakmap@1.0.0/es2021/many-keys-weakmap.js" +import ManyKeysWeakMap from "https://cdn.esm.sh/v64/many-keys-weakmap@1.0.0/es2021/many-keys-weakmap.mjs" /** * @param {string} julia_code From 3ee93ac8fdb60f749f52932c48ecbdfb23d96a08 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 13:08:14 +0200 Subject: [PATCH 20/39] small fixes --- frontend/components/CellInput/lezer_template.js | 2 +- src/evaluation/WorkspaceManager.jl | 4 ++-- test/Events.jl | 3 ++- test/WorkspaceManager.jl | 2 ++ test/packages/Basic.jl | 7 +++---- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/frontend/components/CellInput/lezer_template.js b/frontend/components/CellInput/lezer_template.js index 98310fa0b0..11a96ec1ed 100644 --- a/frontend/components/CellInput/lezer_template.js +++ b/frontend/components/CellInput/lezer_template.js @@ -2,7 +2,7 @@ import { julia_andrey, NodeProp, syntaxTree, Text } from "../../imports/Codemirr import lodash from "../../imports/lodash.js" // @ts-ignore -import ManyKeysWeakMap from "https://cdn.esm.sh/v64/many-keys-weakmap@1.0.0/es2021/many-keys-weakmap.mjs" +import ManyKeysWeakMap from "https://esm.sh/many-keys-weakmap@1.0.0?pin=v113&target=es202" /** * @param {string} julia_code diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 3aef7306cf..49db6ec96c 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -268,7 +268,7 @@ end # NOTE: this function only start a worker process using given # compiler options, it does not resolve paths for notebooks # compiler configurations passed to it should be resolved before this -function create_workspaceprocess(WorkerType; compiler_options=CompilerOptions(), status::Status.Business=Business())::Malt.AbstractWorker +function create_workspaceprocess(WorkerType; compiler_options=CompilerOptions(), status::Status.Business=Status.Business())::Malt.AbstractWorker if WorkerType === Malt.InProcessWorker worker = WorkerType() @@ -417,7 +417,7 @@ function eval_format_fetch_in_workspace( )::PlutoRunner.FormattedCellResult workspace = get_workspace(session_notebook) - is_on_this_process = worker.worker isa Malt.InProcessWorker + is_on_this_process = workspace.worker isa Malt.InProcessWorker # if multiple notebooks run on the same process, then we need to `cd` between the different notebook paths if session_notebook isa Tuple diff --git a/test/Events.jl b/test/Events.jl index b336ff61ef..72fd4d4c67 100644 --- a/test/Events.jl +++ b/test/Events.jl @@ -14,6 +14,7 @@ import UUIDs: UUID end 🍭 = ServerSession() 🍭.options.server.on_event = test_listener + # 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("[1,1,[1]]"), @@ -34,4 +35,4 @@ import UUIDs: UUID # Pluto.save_notebook(io::IOBuffer, notebook): saves notebook to IO # Pluto.ServerSession(;options, event_listener) -end +end \ No newline at end of file diff --git a/test/WorkspaceManager.jl b/test/WorkspaceManager.jl index 72d027f954..fc7138e4a9 100644 --- a/test/WorkspaceManager.jl +++ b/test/WorkspaceManager.jl @@ -9,6 +9,7 @@ import Malt @testset "Multiple notebooks" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = true notebookA = Notebook([ Cell("x = 3") @@ -33,6 +34,7 @@ import Malt end @testset "Variables with secret names" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("result = 1"), diff --git a/test/packages/Basic.jl b/test/packages/Basic.jl index 599603e920..a45ca7c4ca 100644 --- a/test/packages/Basic.jl +++ b/test/packages/Basic.jl @@ -18,6 +18,7 @@ import Malt 🍭.options.evaluation.workspace_use_distributed_stdlib = use_distributed_stdlib # See https://github.com/JuliaPluto/PlutoPkgTestRegistry + notebook = Notebook([ Cell("import PlutoPkgTestA"), # cell 1 Cell("PlutoPkgTestA.MY_VERSION |> Text"), @@ -35,9 +36,7 @@ import Malt @test !notebook.nbpkg_ctx_instantiated - update_save_run!(🍭, notebook, notebook.cells[[1, 2, 7, 8]]) # import A and D - @test noerror(notebook.cells[1]) @test noerror(notebook.cells[2]) @test noerror(notebook.cells[7]) @@ -66,7 +65,7 @@ import Malt @test PkgCompat.get_manifest_version(notebook.nbpkg_ctx, "PlutoPkgTestA") == v"0.3.1" @test PkgCompat.get_manifest_version(notebook.nbpkg_ctx, "PlutoPkgTestD") == v"0.1.0" - + old_A_terminal = deepcopy(terminals["PlutoPkgTestA"]) # @show old_A_terminal @@ -495,6 +494,7 @@ import Malt # end) Malt.stop(test_worker) + end end @testset "PkgUtils -- reset" begin @@ -767,4 +767,3 @@ end # LibGit2.checkout!(repo, "aef26d37e1d0e8f8387c011ccb7c4a38398a18f6") - From 6cc8a5460b3b76f9b98b062618885a3d4597a977 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 13:16:30 +0200 Subject: [PATCH 21/39] fixiefix --- src/evaluation/WorkspaceManager.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 49db6ec96c..577a6ea4df 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -246,8 +246,8 @@ end function get_bond_names(session_notebook::SN, cell_id) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote - PlutoRunner.get_bond_names($cell_id) + Malt.remote_eval_fetch(workspace.worker, quote + PlutoRunner.get_bond_names($cell_id) end) end From 8dba3898c73637059c8d322da4085849be05154e Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 20:35:54 +0200 Subject: [PATCH 22/39] Malt 1.0 compat --- Project.toml | 2 +- src/evaluation/WorkspaceManager.jl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 03369afcd6..30cf9d4904 100644 --- a/Project.toml +++ b/Project.toml @@ -40,7 +40,7 @@ HTTP = "^1.5.2" HypertextLiteral = "0.7, 0.8, 0.9" LoggingExtras = "0.4, 1" MIMEs = "0.1" -Malt = "0.8.2" +Malt = "1.0.1" MsgPack = "1.1" PrecompileSignatures = "3" PrecompileTools = "1" diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 577a6ea4df..3a8dcfe208 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -363,7 +363,8 @@ end function workspace_exception_result(exs::CompositeException, workspace::Workspace) ex = first(exs.exceptions) - if ex.worker == workspace.worker && ex.captured.ex isa InterruptException + if ex isa InterruptException || (ex isa Malt.RemoteException && occursin("InterruptException", ex.message)) + @info "Found an interrupt!" ex ( output_formatted=PlutoRunner.format_output(CapturedException(InterruptException(), [])), errored=true, From c082c7858b1a94ee2f0c3eee51b05e26974b036e Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 20:36:07 +0200 Subject: [PATCH 23/39] verify_no_running_processes --- test/helpers.jl | 17 +++++++++-------- test/runtests.jl | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/test/helpers.jl b/test/helpers.jl index dbe7cba735..5337e58885 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -18,6 +18,7 @@ using Test using HTTP import Distributed import Pkg +import Malt function Base.show(io::IO, s::SymbolsState) print(io, "SymbolsState([") @@ -239,14 +240,14 @@ has_embedded_pkgfiles(contents::AbstractString) = has_embedded_pkgfiles(nb::Pluto.Notebook) = read(nb.path, String) |> has_embedded_pkgfiles -# """ -# Log an error message if there are any running processes created by Distrubted, that were not shut down. -# """ -# function verify_no_running_processes() -# if length(Distributed.procs()) != 1 -# @error "Not all notebook processes were closed during tests!" Distributed.procs() -# end -# end +""" +Log an error message if there are any running processes created by Distrubted, that were not shut down. +""" +function verify_no_running_processes() + if length(Distributed.procs()) != 1 || !isempty(Malt.__iNtErNaL_get_running_procs()) + @error "Not all notebook processes were closed during tests!" Distributed.procs() Malt.__iNtErNaL_get_running_procs() + end +end # We have our own registry for these test! Take a look at https://github.com/JuliaPluto/PlutoPkgTestRegistry#readme for more info about the test packages and their dependencies. diff --git a/test/runtests.jl b/test/runtests.jl index af377ef689..3b3ce21a51 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,22 +3,35 @@ include("helpers.jl") # tests that start new processes: @timeit_include("compiletimes.jl") +verify_no_running_processes() if get(ENV, "PLUTO_TEST_ONLY_COMPILETIMES", nothing) == "true" print_timeroutput() exit(0) end @timeit_include("Events.jl") +verify_no_running_processes() @timeit_include("WorkspaceManager.jl") +verify_no_running_processes() @timeit_include("packages/Basic.jl") +verify_no_running_processes() @timeit_include("Bonds.jl") +verify_no_running_processes() @timeit_include("RichOutput.jl") +verify_no_running_processes() @timeit_include("React.jl") +verify_no_running_processes() @timeit_include("Dynamic.jl") +verify_no_running_processes() @timeit_include("MacroAnalysis.jl") +verify_no_running_processes() @timeit_include("Logging.jl") +verify_no_running_processes() @timeit_include("webserver.jl") +verify_no_running_processes() @timeit_include("Notebook.jl") +verify_no_running_processes() @timeit_include("Configuration.jl") +verify_no_running_processes() # tests that don't start new processes: @timeit_include("ReloadFromFile.jl") @@ -33,6 +46,8 @@ end @timeit_include("Throttled.jl") @timeit_include("cell_disabling.jl") +verify_no_running_processes() + print_timeroutput() # TODO: test PlutoRunner functions like: From 9fdebd62f89a59868be85a9e338cf516e1635f82 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 20:36:36 +0200 Subject: [PATCH 24/39] enable Malt by default for testing this PR --- src/Configuration.jl | 2 +- test/React.jl | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Configuration.jl b/src/Configuration.jl index 4e862e5143..403a060d84 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -140,7 +140,7 @@ end const RUN_NOTEBOOK_ON_LOAD_DEFAULT = true const WORKSPACE_USE_DISTRIBUTED_DEFAULT = true -const WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT = true +const WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT = false const LAZY_WORKSPACE_CREATION_DEFAULT = false const CAPTURE_STDOUT_DEFAULT = true const WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT = nothing diff --git a/test/React.jl b/test/React.jl index 0921a4ce0d..71ad229a9e 100644 --- a/test/React.jl +++ b/test/React.jl @@ -7,8 +7,10 @@ import Distributed 🍭 = ServerSession() 🍭.options.evaluation.workspace_use_distributed = false - @testset "Basic $(parallel ? "distributed" : "single-process")" for parallel in [false, true] - 🍭.options.evaluation.workspace_use_distributed = parallel + @testset "Basic $workertype" for workertype in [:Malt, :Distributed, :InProcess] + 🍭.options.evaluation.workspace_use_distributed = workertype !== :InProcess + 🍭.options.evaluation.workspace_use_distributed_stdlib = workertype === :Distributed + notebook = Notebook([ Cell("x = 1"), @@ -70,9 +72,11 @@ import Distributed @test notebook.cells[6].output.body == "3" update_run!(🍭, notebook, notebook.cells[7:8]) - @test if parallel - notebook.cells[8].output.body != string(Distributed.myid()) - else + @test if workertype === :Distributed + notebook.cells[8].output.body ∉ ("1", string(Distributed.myid())) + elseif workertype === :Malt + notebook.cells[8].output.body == "1" + elseif workertype === :InProcess notebook.cells[8].output.body == string(Distributed.myid()) end From aeeb191306bcc5ccf7991f05277239b928a295b4 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 21:24:22 +0200 Subject: [PATCH 25/39] nothing --- Project.toml | 2 -- frontend/components/CellInput/lezer_template.js | 2 +- test/Bonds.jl | 10 +++++----- test/cell_disabling.jl | 4 +++- test/helpers.jl | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Project.toml b/Project.toml index 30cf9d4904..7f19b5b83c 100644 --- a/Project.toml +++ b/Project.toml @@ -8,7 +8,6 @@ version = "0.19.27" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" FuzzyCompletions = "fb4132e2-a121-4a70-b8a1-d5b831dcdcc2" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" @@ -26,7 +25,6 @@ PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" RegistryInstances = "2792f1a3-b283-48e8-9a74-f99dce5104f3" RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00" -Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" diff --git a/frontend/components/CellInput/lezer_template.js b/frontend/components/CellInput/lezer_template.js index 11a96ec1ed..c034642452 100644 --- a/frontend/components/CellInput/lezer_template.js +++ b/frontend/components/CellInput/lezer_template.js @@ -2,7 +2,7 @@ import { julia_andrey, NodeProp, syntaxTree, Text } from "../../imports/Codemirr import lodash from "../../imports/lodash.js" // @ts-ignore -import ManyKeysWeakMap from "https://esm.sh/many-keys-weakmap@1.0.0?pin=v113&target=es202" +import ManyKeysWeakMap from "https://esm.sh/many-keys-weakmap@1.0.0?pin=v113&target=es2020" /** * @param {string} julia_code diff --git a/test/Bonds.jl b/test/Bonds.jl index 3fe9e559c4..d7f1c6976a 100644 --- a/test/Bonds.jl +++ b/test/Bonds.jl @@ -2,6 +2,7 @@ using Test import Pluto import Pluto: update_run!, update_save_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell import Distributed +import Malt @testset "Bonds" begin @@ -358,9 +359,8 @@ import Distributed # test that the notebook file is runnable: - test_proc = Distributed.addprocs(1)[1] - - Distributed.remotecall_eval(Main, test_proc, quote + test_proc = Malt.Worker() + Malt.remote_eval_wait(test_proc, quote import Pkg try Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true @@ -368,11 +368,11 @@ import Distributed Pkg.activate(mktempdir()) Pkg.add("AbstractPlutoDingetjes") end) - @test Distributed.remotecall_eval(Main, test_proc, quote + @test Malt.remote_eval_fetch(test_proc, quote include($(notebook.path)) true end) - Distributed.rmprocs(test_proc) + Malt.stop(test_proc) end @testset "Dependent Bound Variables" begin diff --git a/test/cell_disabling.jl b/test/cell_disabling.jl index f5f9575dd3..3c8e8f498b 100644 --- a/test/cell_disabling.jl +++ b/test/cell_disabling.jl @@ -1,6 +1,6 @@ using Test using Pluto -using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook, set_disabled, is_disabled +using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook, set_disabled, is_disabled, WorkspaceManager @@ -236,6 +236,7 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook, set_disa update_run!(🍭, notebook, c([12])) @test c(14).output.body == "3" + WorkspaceManager.unmake_workspace((🍭, notebook)) end @@ -342,4 +343,5 @@ end update_run!(🍭, notebook, notebook.cells) @test get_disabled_cells(notebook) == [] + WorkspaceManager.unmake_workspace((🍭, notebook)) end diff --git a/test/helpers.jl b/test/helpers.jl index 5337e58885..3b745c25fc 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -16,9 +16,9 @@ import Pluto.ExpressionExplorer: SymbolsState, compute_symbolreferences, Functio using Sockets using Test using HTTP -import Distributed import Pkg import Malt +import Malt.Distributed function Base.show(io::IO, s::SymbolsState) print(io, "SymbolsState([") From 8d5a0e73836e1162d500fa4076912879fa084aea Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 21:24:27 +0200 Subject: [PATCH 26/39] huh --- test/runtests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 3b3ce21a51..88901d4ae3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,7 +10,7 @@ if get(ENV, "PLUTO_TEST_ONLY_COMPILETIMES", nothing) == "true" end @timeit_include("Events.jl") verify_no_running_processes() -@timeit_include("WorkspaceManager.jl") +@timeit_include("Configuration.jl") verify_no_running_processes() @timeit_include("packages/Basic.jl") verify_no_running_processes() @@ -30,7 +30,7 @@ verify_no_running_processes() verify_no_running_processes() @timeit_include("Notebook.jl") verify_no_running_processes() -@timeit_include("Configuration.jl") +@timeit_include("WorkspaceManager.jl") verify_no_running_processes() # tests that don't start new processes: From eb161b063444745b47be8afb9f16a4c60257f6a5 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 21:40:59 +0200 Subject: [PATCH 27/39] asdf --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 7f19b5b83c..44845a354a 100644 --- a/Project.toml +++ b/Project.toml @@ -25,6 +25,7 @@ PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" RegistryInstances = "2792f1a3-b283-48e8-9a74-f99dce5104f3" RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" From 69569b95a26611b4a79231340108b3c2f1bebab9 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 21:47:01 +0200 Subject: [PATCH 28/39] asdf --- test/Bonds.jl | 1 - test/React.jl | 1 - test/ReloadFromFile.jl | 1 - 3 files changed, 3 deletions(-) diff --git a/test/Bonds.jl b/test/Bonds.jl index d7f1c6976a..bf1b028e09 100644 --- a/test/Bonds.jl +++ b/test/Bonds.jl @@ -1,7 +1,6 @@ using Test import Pluto import Pluto: update_run!, update_save_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell -import Distributed import Malt @testset "Bonds" begin diff --git a/test/React.jl b/test/React.jl index 71ad229a9e..d28f2d9cf7 100644 --- a/test/React.jl +++ b/test/React.jl @@ -1,7 +1,6 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager import Pluto.Configuration: Options, EvaluationOptions -import Distributed @testset "Reactivity" begin 🍭 = ServerSession() diff --git a/test/ReloadFromFile.jl b/test/ReloadFromFile.jl index 7cc6b9aac4..ec602fee3c 100644 --- a/test/ReloadFromFile.jl +++ b/test/ReloadFromFile.jl @@ -1,7 +1,6 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager, SessionActions, save_notebook import Pluto.Configuration: Options, EvaluationOptions -import Distributed using Pluto.WorkspaceManager: poll import Pkg From abb0a79f09a8286829fcd59d83e53a8584d46bd4 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 22:22:28 +0200 Subject: [PATCH 29/39] Malt 1.0.2 --- Project.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 44845a354a..472cb790ec 100644 --- a/Project.toml +++ b/Project.toml @@ -25,7 +25,6 @@ PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" RegistryInstances = "2792f1a3-b283-48e8-9a74-f99dce5104f3" RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00" -Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" @@ -39,7 +38,7 @@ HTTP = "^1.5.2" HypertextLiteral = "0.7, 0.8, 0.9" LoggingExtras = "0.4, 1" MIMEs = "0.1" -Malt = "1.0.1" +Malt = "1.0.2" MsgPack = "1.1" PrecompileSignatures = "3" PrecompileTools = "1" From c4bb0e2cfc7b701d466ead2e7cb20275a846d7a7 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 12 Sep 2023 23:21:30 +0200 Subject: [PATCH 30/39] fix test --- test/React.jl | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/test/React.jl b/test/React.jl index d28f2d9cf7..bb249032c9 100644 --- a/test/React.jl +++ b/test/React.jl @@ -23,7 +23,13 @@ import Pluto.Configuration: Options, EvaluationOptions end"""), Cell("g(6) + g(6,6)"), - Cell("import Distributed"), + Cell(""" + begin + pushfirst!(LOAD_PATH, "@stdlib") + import Distributed + popfirst!(LOAD_PATH) + end + """), Cell("Distributed.myid()"), ]) @@ -71,12 +77,14 @@ import Pluto.Configuration: Options, EvaluationOptions @test notebook.cells[6].output.body == "3" update_run!(🍭, notebook, notebook.cells[7:8]) - @test if workertype === :Distributed - notebook.cells[8].output.body ∉ ("1", string(Distributed.myid())) + if workertype === :Distributed + @test notebook.cells[8].output.body ∉ ("1", string(Distributed.myid())) elseif workertype === :Malt - notebook.cells[8].output.body == "1" + @test notebook.cells[8].output.body == "1" elseif workertype === :InProcess - notebook.cells[8].output.body == string(Distributed.myid()) + @test notebook.cells[8].output.body == string(Distributed.myid()) + else + error() end WorkspaceManager.unmake_workspace((🍭, notebook); verbose=false) From 45e6e1b873859538556b48b2cc1688c6e379193d Mon Sep 17 00:00:00 2001 From: Paul Berg Date: Wed, 13 Sep 2023 13:23:32 +0200 Subject: [PATCH 31/39] Test setting OPENBLAS_NUM_THREADS=1 on windows for Malt.Worker --- src/evaluation/WorkspaceManager.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 3a8dcfe208..3c31e474b3 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -284,8 +284,12 @@ function create_workspaceprocess(WorkerType; compiler_options=CompilerOptions(), Status.report_business_started!(status, Symbol(1)) Status.report_business_planned!(status, Symbol(2)) - worker = WorkerType(; exeflags=_convert_to_flags(compiler_options)) - + # NOTE: Test to reduce the required memory on windows + # .... see https://github.com/JuliaLang/julia/pull/47803 + # .... the same fix can maybe be applied to other os? + env = Sys.iswindows() && WorkerType == Malt.Worker ? String["OPENBLAS_NUM_THREADS=1"] : String[] + worker = WorkerType(; exeflags=_convert_to_flags(compiler_options), env) + Status.report_business_finished!(status, Symbol(1)) Status.report_business_started!(status, Symbol(2)) From 82de248d1c8682ac717bd1a6ca469a0d6d68d49c Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Thu, 14 Sep 2023 13:08:37 +0200 Subject: [PATCH 32/39] trying JULIA_NUM_PRECOMPILE_TASKS=1 --- src/evaluation/WorkspaceManager.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 3c31e474b3..7a1ecb31ab 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -284,10 +284,16 @@ function create_workspaceprocess(WorkerType; compiler_options=CompilerOptions(), Status.report_business_started!(status, Symbol(1)) Status.report_business_planned!(status, Symbol(2)) + env = String[] # NOTE: Test to reduce the required memory on windows # .... see https://github.com/JuliaLang/julia/pull/47803 # .... the same fix can maybe be applied to other os? - env = Sys.iswindows() && WorkerType == Malt.Worker ? String["OPENBLAS_NUM_THREADS=1"] : String[] + if Sys.iswindows() && WorkerType == Malt.Worker + push!(env, "OPENBLAS_NUM_THREADS=1") + if VERSION < v"1.7.0" + push!(env, "JULIA_NUM_PRECOMPILE_TASKS=1") + end + end worker = WorkerType(; exeflags=_convert_to_flags(compiler_options), env) Status.report_business_finished!(status, Symbol(1)) From d542e4cce8318e5bab0cbdf2f2be98b739f8f56d Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 18 Sep 2023 11:28:07 +0200 Subject: [PATCH 33/39] Revert "trying JULIA_NUM_PRECOMPILE_TASKS=1" This reverts commit 82de248d1c8682ac717bd1a6ca469a0d6d68d49c. --- src/evaluation/WorkspaceManager.jl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 7a1ecb31ab..3c31e474b3 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -284,16 +284,10 @@ function create_workspaceprocess(WorkerType; compiler_options=CompilerOptions(), Status.report_business_started!(status, Symbol(1)) Status.report_business_planned!(status, Symbol(2)) - env = String[] # NOTE: Test to reduce the required memory on windows # .... see https://github.com/JuliaLang/julia/pull/47803 # .... the same fix can maybe be applied to other os? - if Sys.iswindows() && WorkerType == Malt.Worker - push!(env, "OPENBLAS_NUM_THREADS=1") - if VERSION < v"1.7.0" - push!(env, "JULIA_NUM_PRECOMPILE_TASKS=1") - end - end + env = Sys.iswindows() && WorkerType == Malt.Worker ? String["OPENBLAS_NUM_THREADS=1"] : String[] worker = WorkerType(; exeflags=_convert_to_flags(compiler_options), env) Status.report_business_finished!(status, Symbol(1)) From 07101de744cd3ff2c71ae0d692b963ed7e22dd61 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 18 Sep 2023 11:48:24 +0200 Subject: [PATCH 34/39] Revert "Test setting OPENBLAS_NUM_THREADS=1 on windows for Malt.Worker" This reverts commit 45e6e1b873859538556b48b2cc1688c6e379193d. --- src/evaluation/WorkspaceManager.jl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 3c31e474b3..3a8dcfe208 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -284,12 +284,8 @@ function create_workspaceprocess(WorkerType; compiler_options=CompilerOptions(), Status.report_business_started!(status, Symbol(1)) Status.report_business_planned!(status, Symbol(2)) - # NOTE: Test to reduce the required memory on windows - # .... see https://github.com/JuliaLang/julia/pull/47803 - # .... the same fix can maybe be applied to other os? - env = Sys.iswindows() && WorkerType == Malt.Worker ? String["OPENBLAS_NUM_THREADS=1"] : String[] - worker = WorkerType(; exeflags=_convert_to_flags(compiler_options), env) - + worker = WorkerType(; exeflags=_convert_to_flags(compiler_options)) + Status.report_business_finished!(status, Symbol(1)) Status.report_business_started!(status, Symbol(2)) From 27307e303d86072703ed088792c404a3ea370a9d Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 18 Sep 2023 11:54:55 +0200 Subject: [PATCH 35/39] Disable Malt by default --- src/Configuration.jl | 8 ++++---- src/evaluation/WorkspaceManager.jl | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Configuration.jl b/src/Configuration.jl index 403a060d84..2b7eed6ae9 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -140,7 +140,7 @@ end const RUN_NOTEBOOK_ON_LOAD_DEFAULT = true const WORKSPACE_USE_DISTRIBUTED_DEFAULT = true -const WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT = false +const WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT = nothing const LAZY_WORKSPACE_CREATION_DEFAULT = false const CAPTURE_STDOUT_DEFAULT = true const WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT = nothing @@ -153,7 +153,7 @@ These options are not intended to be changed during normal use. - `run_notebook_on_load::Bool = $RUN_NOTEBOOK_ON_LOAD_DEFAULT` Whether to evaluate a notebook on load. - `workspace_use_distributed::Bool = $WORKSPACE_USE_DISTRIBUTED_DEFAULT` Whether to start notebooks in a separate process. -- `workspace_use_distributed_stdlib::Bool = $WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT` Should we use the Distributed stdlib to run processes? This has been replaced by Malt.jl, but you can use this option to get the old behaviour. Option be removed in the future. +- `workspace_use_distributed_stdlib::Bool? = $WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT` Should we use the Distributed stdlib to run processes? Distributed will be replaced by Malt.jl, you can use this option to already get the old behaviour. `nothing` means: determine automatically (which is currently the same as `true`). - `lazy_workspace_creation::Bool = $LAZY_WORKSPACE_CREATION_DEFAULT` - `capture_stdout::Bool = $CAPTURE_STDOUT_DEFAULT` - `workspace_custom_startup_expr::Union{Nothing,Expr} = $WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT` An expression to be evaluated in the workspace process before running notebook code. @@ -161,7 +161,7 @@ These options are not intended to be changed during normal use. @option mutable struct EvaluationOptions run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT - workspace_use_distributed_stdlib::Bool = WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT + workspace_use_distributed_stdlib::Union{Bool,Nothing} = WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT @@ -295,7 +295,7 @@ function from_flat_kwargs(; run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT, workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT, - workspace_use_distributed_stdlib::Bool = WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT, + workspace_use_distributed_stdlib::Union{Bool,Nothing} = WORKSPACE_USE_DISTRIBUTED_STDLIB_DEFAULT, lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT, capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT, workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT, diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 3a8dcfe208..0d952a7d55 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -53,7 +53,11 @@ function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false WorkerType = if is_offline_renderer || !session.options.evaluation.workspace_use_distributed Malt.InProcessWorker - elseif session.options.evaluation.workspace_use_distributed_stdlib + elseif something( + session.options.evaluation.workspace_use_distributed_stdlib, + true + # VERSION < v"1.8.0-0" + ) Malt.DistributedStdlibWorker else Malt.Worker From 7b09e1dc91901c8be2d8c6f967bd1ec2e52d950e Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 18 Sep 2023 12:13:43 +0200 Subject: [PATCH 36/39] Put back some workspace_use_distributed=false in tests --- Project.toml | 2 +- test/Configuration.jl | 2 +- test/DependencyCache.jl | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 472cb790ec..1744e0881d 100644 --- a/Project.toml +++ b/Project.toml @@ -38,7 +38,7 @@ HTTP = "^1.5.2" HypertextLiteral = "0.7, 0.8, 0.9" LoggingExtras = "0.4, 1" MIMEs = "0.1" -Malt = "1.0.2" +Malt = "1.0.3" MsgPack = "1.1" PrecompileSignatures = "3" PrecompileTools = "1" diff --git a/test/Configuration.jl b/test/Configuration.jl index 7a6a3a0f08..4d2d6c2f54 100644 --- a/test/Configuration.jl +++ b/test/Configuration.jl @@ -59,7 +59,7 @@ end @testset "Authentication" begin port = 1238 - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=false) 🍭 = Pluto.ServerSession(; options) host = 🍭.options.server.host secret = 🍭.secret diff --git a/test/DependencyCache.jl b/test/DependencyCache.jl index 2de1884771..77b2c0b660 100644 --- a/test/DependencyCache.jl +++ b/test/DependencyCache.jl @@ -5,6 +5,7 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @testset "CellDepencencyVisualization" begin 🍭 = ServerSession() + 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("x = 1"), # prerequisite of test cell From a37fd31b6aafeeea25c3452f61f7c11c971cc967 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 18 Sep 2023 13:59:17 +0200 Subject: [PATCH 37/39] one more distributed test disable --- test/Events.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Events.jl b/test/Events.jl index 72fd4d4c67..1dbfdaee03 100644 --- a/test/Events.jl +++ b/test/Events.jl @@ -14,7 +14,7 @@ import UUIDs: UUID end 🍭 = ServerSession() 🍭.options.server.on_event = test_listener - # 🍭.options.evaluation.workspace_use_distributed = false + 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("[1,1,[1]]"), From fd8c68a7248a4c98224be1a6c5eebd476ba444ed Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 18 Sep 2023 13:59:27 +0200 Subject: [PATCH 38/39] test Malt by default again... --- src/evaluation/WorkspaceManager.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 0d952a7d55..6568704793 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -55,7 +55,7 @@ function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false Malt.InProcessWorker elseif something( session.options.evaluation.workspace_use_distributed_stdlib, - true + false # VERSION < v"1.8.0-0" ) Malt.DistributedStdlibWorker From 010666383f5253d755919ac503c251ccc58de325 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 18 Sep 2023 14:10:19 +0200 Subject: [PATCH 39/39] Disable Malt by default again --- src/evaluation/WorkspaceManager.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 6568704793..0d952a7d55 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -55,7 +55,7 @@ function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false Malt.InProcessWorker elseif something( session.options.evaluation.workspace_use_distributed_stdlib, - false + true # VERSION < v"1.8.0-0" ) Malt.DistributedStdlibWorker