From f7423ca7668e3af11e6d105f06f3bbd0cacd40d6 Mon Sep 17 00:00:00 2001 From: Jose Esparza <28990958+pebeto@users.noreply.github.com> Date: Sun, 27 Aug 2023 18:54:54 -0500 Subject: [PATCH 01/18] Little README addition --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 07917d2..60e9717 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ We assume MLJDecisionTreeClassifier is in the user's active Julia package environment. ```julia -using MLJ +using MLJ # Requires MLJ.jl version 0.19.3 or higher ``` We first define a logger, providing the address of our running MLflow. The experiment From c1d9c7f50904e8a2179da4a9ab552ebe82ed5293 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 6 Sep 2023 17:34:56 +1200 Subject: [PATCH 02/18] address breaking changes in MLJBase 0.22 --- Project.toml | 3 ++- src/MLJFlow.jl | 3 +-- src/base.jl | 5 ++++- src/service.jl | 43 ++++++++++++++++++++++++++++++++++++++++--- test/runtests.jl | 3 +++ test/service.jl | 7 +++++++ 6 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 test/service.jl diff --git a/Project.toml b/Project.toml index 935a7ae..1084505 100644 --- a/Project.toml +++ b/Project.toml @@ -18,7 +18,8 @@ julia = "1.6" MLFlowClient = "64a0f543-368b-4a9a-827a-e71edb2a0b83" MLJDecisionTreeInterface = "c6f25543-311c-4c74-83dc-3ea6d1015661" MLJModels = "d491faf4-2d78-11e9-2867-c94bc002c0b7" +StatisticalMeasures = "a19d573c-0a75-4610-95b3-7071388c7541" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "MLFlowClient", "MLJModels", "MLJDecisionTreeInterface"] +test = ["Test", "MLFlowClient", "MLJModels", "MLJDecisionTreeInterface", "StatisticalMeasures"] diff --git a/src/MLJFlow.jl b/src/MLJFlow.jl index 82ec796..a434b6d 100644 --- a/src/MLJFlow.jl +++ b/src/MLJFlow.jl @@ -1,7 +1,6 @@ module MLJFlow -using MLJBase: info, name, Model, - Machine +using MLJBase: Model, Machine, name using MLJModelInterface: flat_params using MLFlowClient: MLFlow, logparam, logmetric, createrun, MLFlowRun, updaterun, diff --git a/src/base.jl b/src/base.jl index 2505703..880b595 100644 --- a/src/base.jl +++ b/src/base.jl @@ -3,7 +3,10 @@ function log_evaluation(logger::MLFlowLogger, performance_evaluation) artifact_location=logger.artifact_location) run = createrun(logger.service, experiment; tags=[ - Dict("key" => "resampling", "value" => string(performance_evaluation.resampling)), + Dict( + "key" => "resampling", + "value" => string(performance_evaluation.resampling) + ), Dict("key" => "repeats", "value" => string(performance_evaluation.repeats)), Dict("key" => "model type", "value" => name(performance_evaluation.model)), ] diff --git a/src/service.jl b/src/service.jl index cd4a12b..b9c90f7 100644 --- a/src/service.jl +++ b/src/service.jl @@ -18,20 +18,57 @@ function logmodelparams(service::MLFlow, run::MLFlowRun, model::Model) end end +const MLFLOW_CHAR_SET = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-. /" + +""" + good_name(measure) + +**Private method.** + +Returns a string representation of `measure` that can be used as a valid name in +MLflow. Includes the value of the first hyperparameter, if there is one. + +```julia +julia> good_name(macro_f1score) +"MulticlassFScore-beta_1.0" + +""" +function good_name(measure) + name = string(measure) + name = replace(name, ", …" => "") + name = replace(name, " = " => "_") + name = replace(name, "()" => "") + name = replace(name, ")" => "") + map(collect(name)) do char + char in ['(', ','] && return '-' + char == '=' && return '_' + char in MLFLOW_CHAR_SET && return char + " " + end |> join +end + """ logmachinemeasures(service::MLFlow, run::MLFlowRun, model::Model) Extracts the parameters of a model and logs them to the MLFlow server. # Arguments -- `service::MLFlow`: An MLFlow service. See [MLFlowClient.jl](https://juliaai.github.io/MLFlowClient.jl/dev/reference/#MLFlowClient.MLFlow) -- `run::MLFlowRun`: An MLFlow run. See [MLFlowClient.jl](https://juliaai.github.io/MLFlowClient.jl/dev/reference/#MLFlowClient.MLFlowRun) + +- `service::MLFlow`: An MLFlow service. See + [MLFlowClient.jl](https://juliaai.github.io/MLFlowClient.jl/dev/reference/#MLFlowClient.MLFlow) + +- `run::MLFlowRun`: An MLFlow run. See + [MLFlowClient.jl](https://juliaai.github.io/MLFlowClient.jl/dev/reference/#MLFlowClient.MLFlowRun) + - `measures`: A vector of measures. + - `measurements`: A vector of measurements. + """ function logmachinemeasures(service::MLFlow, run::MLFlowRun, measures, measurements) - measure_names = measures .|> info .|> x -> x.name + measure_names = measures .|> good_name for (name, value) in zip(measure_names, measurements) logmetric(service, run, name, value) end diff --git a/test/runtests.jl b/test/runtests.jl index 20f209c..ce5912d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,6 +6,9 @@ using MLJBase using MLJModels using MLFlowClient using MLJModelInterface +using StatisticalMeasures include("base.jl") include("types.jl") +include("service.jl") + diff --git a/test/service.jl b/test/service.jl new file mode 100644 index 0000000..0c0da6b --- /dev/null +++ b/test/service.jl @@ -0,0 +1,7 @@ +@testset "good_name" begin + @test MLJFlow.good_name(rms) == "RootMeanSquaredError" + @test MLJFlow.good_name(macro_f1score) == "MulticlassFScore-beta_1.0" + @test MLJFlow.good_name(log_score) == "LogScore-tol_2.22045e-16" +end + +true From aef3f57f45f3332a2ce093f34be5600b30e44f2e Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 6 Sep 2023 17:41:13 +1200 Subject: [PATCH 03/18] bump [compat] MLJBase = "0.22" --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 1084505..f154fbb 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,7 @@ MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" [compat] MLFlowClient = "0.4.4" -MLJBase = "0.21.14" +MLJBase = "0.22" MLJModelInterface = "1.9.1" julia = "1.6" From 40cadb45198d471fc51716a19d4f6cd3a1456d2b Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 21 Sep 2023 15:39:54 +1200 Subject: [PATCH 04/18] temporary revert [compat] MLJBase="0.21" for development --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index f154fbb..d71c683 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,7 @@ MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" [compat] MLFlowClient = "0.4.4" -MLJBase = "0.22" +MLJBase = "0.21" MLJModelInterface = "1.9.1" julia = "1.6" From ef30d622ae53b4e867aca004b7e9b928eb72625f Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Tue, 26 Sep 2023 09:50:54 +1300 Subject: [PATCH 05/18] bump [compat] MLJBase = "1" --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index d71c683..c9fcc43 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,7 @@ MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" [compat] MLFlowClient = "0.4.4" -MLJBase = "0.21" +MLJBase = "1" MLJModelInterface = "1.9.1" julia = "1.6" From 76ad34f27376e73f78fd5449e4a22bfe6dcd8c40 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Tue, 26 Sep 2023 09:51:27 +1300 Subject: [PATCH 06/18] bump 0.2.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index c9fcc43..9d19ee3 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MLJFlow" uuid = "7b7b8358-b45c-48ea-a8ef-7ca328ad328f" authors = ["Jose Esparza "] -version = "0.1.1" +version = "0.2.0" [deps] MLFlowClient = "64a0f543-368b-4a9a-827a-e71edb2a0b83" From bed1dfc496a0b56b682bfbf3060de61d17d2ff8e Mon Sep 17 00:00:00 2001 From: "Anthony Blaom, PhD" Date: Thu, 28 Sep 2023 10:02:22 +1300 Subject: [PATCH 07/18] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 60e9717..f1a51f2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# MLJFlow +# MLJFlow.jl | Branch | Build | Coverage | | :---: | :---: | :---: | From 3ce673331e0d439299126aff35aa0a593d2e4be3 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 28 Sep 2023 11:13:03 +1300 Subject: [PATCH 08/18] fix mljflow to 2.6.0 in CI --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 855ed4a..6aa4bbc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -31,7 +31,7 @@ jobs: python-version: '3.10' - name: Setup mlflow locally run: | - pip install mlflow + pip install mlflow==2.6.0 python3 /opt/hostedtoolcache/Python/3.10.12/x64/bin/mlflow server --host 0.0.0.0 --port 5000 & sleep 5 - uses: julia-actions/setup-julia@v1 From 56eb71168a3df7c143b2f57984c1e883e29e6da6 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 28 Sep 2023 11:52:49 +1300 Subject: [PATCH 09/18] add sleep --- .github/workflows/CI.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6aa4bbc..dd4ffec 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -11,6 +11,13 @@ on: tags: '*' jobs: test: + services: + mlflow: + image: adacotechjp/mlflow:2.3.1 + ports: + - 5000:5000 + run: | + sleep 5 name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} strategy: @@ -26,14 +33,6 @@ jobs: - x64 steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Setup mlflow locally - run: | - pip install mlflow==2.6.0 - python3 /opt/hostedtoolcache/Python/3.10.12/x64/bin/mlflow server --host 0.0.0.0 --port 5000 & - sleep 5 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} From 87b96786e4306b6b8d7acf4ab427753d0e544159 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 28 Sep 2023 11:13:03 +1300 Subject: [PATCH 10/18] fix mljflow to 2.6.0 in CI --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 855ed4a..6aa4bbc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -31,7 +31,7 @@ jobs: python-version: '3.10' - name: Setup mlflow locally run: | - pip install mlflow + pip install mlflow==2.6.0 python3 /opt/hostedtoolcache/Python/3.10.12/x64/bin/mlflow server --host 0.0.0.0 --port 5000 & sleep 5 - uses: julia-actions/setup-julia@v1 From 22ee27068fb49848cda92be1a7e8fec3170f0975 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 28 Sep 2023 11:37:23 +1300 Subject: [PATCH 11/18] try launching mlflow service like MLFlowClient.jl does in its CI --- .github/workflows/CI.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6aa4bbc..62ed463 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -11,6 +11,11 @@ on: tags: '*' jobs: test: + services: + mlflow: + image: adacotechjp/mlflow:2.3.1 + ports: + - 5000:5000 name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} strategy: @@ -26,14 +31,6 @@ jobs: - x64 steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Setup mlflow locally - run: | - pip install mlflow==2.6.0 - python3 /opt/hostedtoolcache/Python/3.10.12/x64/bin/mlflow server --host 0.0.0.0 --port 5000 & - sleep 5 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} From bdb1bc59705fc29dd68436e191cda0b3babe38bc Mon Sep 17 00:00:00 2001 From: Jose Esparza <28990958+pebeto@users.noreply.github.com> Date: Wed, 27 Sep 2023 22:32:07 -0500 Subject: [PATCH 12/18] Making Python version fixed --- .github/workflows/CI.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 62ed463..8929567 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -11,11 +11,6 @@ on: tags: '*' jobs: test: - services: - mlflow: - image: adacotechjp/mlflow:2.3.1 - ports: - - 5000:5000 name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} strategy: @@ -31,6 +26,15 @@ jobs: - x64 steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + with: + python-version: '3.10.13' + - name: Setup mlflow locally + run: | + pip install mlflow + pip show mlflow + python3 /opt/hostedtoolcache/Python/3.10.13/x64/bin/mlflow server --host 0.0.0.0 --port 5000 & + sleep 5 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} @@ -51,3 +55,4 @@ jobs: - uses: codecov/codecov-action@v2 with: files: lcov.info + From 03435ccc5f825cef0f18b9705f66f064b58f1282 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 29 Sep 2023 08:41:29 +1300 Subject: [PATCH 13/18] update codecov config --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d986789..9b88cbd 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -59,7 +59,7 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 with: files: lcov.info From 4981f214b8fe39b1fd4a0c2972d5b0d93a2ee68f Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 29 Sep 2023 08:46:33 +1300 Subject: [PATCH 14/18] add forgotten file --- .github/codecov.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/codecov.yml diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000..6d1acf8 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,6 @@ +coverage: + status: + project: + default: + threshold: 0.5% + target: 90% \ No newline at end of file From 9808fd6b9e6a53046b68baee227c349126ecf1a8 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 29 Sep 2023 08:48:36 +1300 Subject: [PATCH 15/18] rm tabs from yml file --- .github/codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index 6d1acf8..e7e6cf1 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -3,4 +3,4 @@ coverage: project: default: threshold: 0.5% - target: 90% \ No newline at end of file + target: 90% \ No newline at end of file From c045b004b1c25dc012304750dd9c4929a2539d9e Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 29 Sep 2023 08:50:17 +1300 Subject: [PATCH 16/18] rm redundant service from ci --- .github/workflows/CI.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9b88cbd..d1b24b0 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -11,13 +11,6 @@ on: tags: '*' jobs: test: - services: - mlflow: - image: adacotechjp/mlflow:2.3.1 - ports: - - 5000:5000 - run: | - sleep 5 name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} strategy: From e59311b6642ef0b9222531b8f6423ea34ff7fb9b Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 29 Sep 2023 09:06:36 +1300 Subject: [PATCH 17/18] drop target from codecov --- .github/codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index e7e6cf1..dbe9c9d 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -3,4 +3,4 @@ coverage: project: default: threshold: 0.5% - target: 90% \ No newline at end of file + target: auto \ No newline at end of file From 4005b79bb97abcce62df48bd76febae81195e1f3 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 29 Sep 2023 10:00:14 +1300 Subject: [PATCH 18/18] add test to bump coverage --- test/service.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/service.jl b/test/service.jl index 0c0da6b..5f14913 100644 --- a/test/service.jl +++ b/test/service.jl @@ -2,6 +2,7 @@ @test MLJFlow.good_name(rms) == "RootMeanSquaredError" @test MLJFlow.good_name(macro_f1score) == "MulticlassFScore-beta_1.0" @test MLJFlow.good_name(log_score) == "LogScore-tol_2.22045e-16" + @test MLJFlow.good_name("??") == " " end true