From 4f3597345389a44afe48b3ec38c43a5a8550ebfa Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 23 Jun 2022 19:07:22 +1200 Subject: [PATCH 1/8] Add preliminary Metalhead.jl integration and fix #162 first attempt Metalhead integration (with hack); tests lacking minor add docstring comment rm invalidated test mv metalhead stuff out to separate src file add show methods for Metalhead wraps add forgotten files with tests fix test rename metal -> image_builder --- Project.toml | 2 + src/MLJFlux.jl | 4 +- src/builders.jl | 8 +- src/metalhead.jl | 152 ++++++++++++++++++++++++++++++++++++ src/types.jl | 4 +- test/builders.jl | 16 +++- test/metalhead.jl | 59 ++++++++++++++ test/mlj_model_interface.jl | 4 - test/runtests.jl | 4 + 9 files changed, 241 insertions(+), 12 deletions(-) create mode 100644 src/metalhead.jl create mode 100644 test/metalhead.jl diff --git a/Project.toml b/Project.toml index a2f70565..dd10ab8b 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" +Metalhead = "dbeba491-748d-5e0e-a39e-b530a07fa0cc" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" @@ -19,6 +20,7 @@ CategoricalArrays = "0.10" ColorTypes = "0.10.3, 0.11" ComputationalResources = "0.3.2" Flux = "0.10.4, 0.11, 0.12, 0.13" +Metalhead = "0.7" MLJModelInterface = "1.1.1" ProgressMeter = "1.7.1" Tables = "1.0" diff --git a/src/MLJFlux.jl b/src/MLJFlux.jl index 84bce73f..d3a88064 100644 --- a/src/MLJFlux.jl +++ b/src/MLJFlux.jl @@ -1,4 +1,4 @@ -module MLJFlux +module MLJFlux export CUDALibs, CPU1 @@ -13,10 +13,12 @@ using Statistics using ColorTypes using ComputationalResources using Random +import Metalhead include("penalizers.jl") include("core.jl") include("builders.jl") +include("metalhead.jl") include("types.jl") include("regressor.jl") include("classifier.jl") diff --git a/src/builders.jl b/src/builders.jl index 2c417c20..b106058a 100644 --- a/src/builders.jl +++ b/src/builders.jl @@ -1,4 +1,4 @@ -## BUILDING CHAINS A FROM HYPERPARAMETERS + INPUT/OUTPUT SHAPE +# # BUILDING CHAINS A FROM HYPERPARAMETERS + INPUT/OUTPUT SHAPE # We introduce chain builders as a way of exposing neural network # hyperparameters (describing, architecture, dropout rates, etc) to @@ -9,7 +9,7 @@ # input/output dimensions/shape. # Below n or (n1, n2) etc refers to network inputs, while m or (m1, -# m2) etc refers to outputs. +# m2) etc refers to outputs. abstract type Builder <: MLJModelInterface.MLJType end @@ -38,7 +38,7 @@ using `n_hidden` nodes in the hidden layer and the specified `dropout` (defaulting to 0.5). An activation function `σ` is applied between the hidden and final layers. If `n_hidden=0` (the default) then `n_hidden` is the geometric mean of the number of input and output nodes. The -number of input and output nodes is determined from the data. +number of input and output nodes is determined from the data. The each layer is initialized using `Flux.glorot_uniform(rng)`. If `rng` is an integer, it is instead used as the seed for a @@ -96,6 +96,8 @@ function MLJFlux.build(mlp::MLP, rng, n_in, n_out) end +# # BUILER MACRO + struct GenericBuilder{F} <: Builder apply::F end diff --git a/src/metalhead.jl b/src/metalhead.jl new file mode 100644 index 00000000..d0ec1a07 --- /dev/null +++ b/src/metalhead.jl @@ -0,0 +1,152 @@ +#= + +TODO: After https://github.com/FluxML/Metalhead.jl/issues/176: + +- Export and externally document `metal` method + +- Delete definition of `ResNetHack` below + +- Change default builder in ImageClassifier (see /src/types.jl) from + `image_builder(ResNetHack(...))` to `image_builder(Metalhead.ResNet(...))`, + +- Add nicer `show` methods for `MetalheadBuilder` instances + +=# + +const DISALLOWED_KWARGS = [:imsize, :inchannels, :nclasses] +const human_disallowed_kwargs = join(map(s->"`$s`", DISALLOWED_KWARGS), ", ", " and ") +const ERR_METALHEAD_DISALLOWED_KWARGS = ArgumentError( + "Keyword arguments $human_disallowed_kwargs are disallowed "* + "as their values are inferred from data. " +) + +# # WRAPPING + +struct MetalheadWrapper{F} <: MLJFlux.Builder + metalhead_constructor::F +end + +struct MetalheadBuilder{F} <: MLJFlux.Builder + metalhead_constructor::F + args + kwargs +end + +Base.show(io::IO, w::MetalheadWrapper) = + print(io, "image_builder($(repr(w.metalhead_constructor)))") + +function Base.show(io::IO, ::MIME"text/plain", w::MetalheadBuilder) + println(io, "builder wrapping $(w.metalhead_constructor)") + if !isempty(w.args) + println(io, " args:") + for (i, arg) in enumerate(w.args) + println(io, " 1: $arg") + end + end + if !isempty(w.kwargs) + println(io, " kwargs:") + for kwarg in w.kwargs + println(io, " $(first(kwarg)) = $(last(kwarg))") + end + end +end + +Base.show(io::IO, w::MetalheadBuilder) = + print(io, "image_builder($(repr(w.metalhead_constructor)))(…)") + + +""" + image_builder(constructor)(args...; kwargs...) + +Return an MLJFlux builder object based on the Metalhead.jl constructor/type +`constructor` (eg, `Metalhead.ResNet`). Here `args` and `kwargs` are +passed to the `MetalheadType` constructor at "build time", along with +the extra keyword specifiers `imsize=...`, `inchannels=...` and +`nclasses=...`, with values inferred from the data. + +# Example + +If in Metalhead.jl you would do + +```julia +using Metalhead +model = ResNet(50, pretrain=true, inchannels=1, nclasses=10) +``` + +then in MLJFlux, it suffices to do + +```julia +using MLJFlux, Metalhead +builder = image_builder(ResNet)(50, pretrain=true) +``` + +which can be used in `ImageClassifier` as in + +```julia +clf = ImageClassifier( + builder=builder, + epochs=500, + optimiser=Flux.ADAM(0.001), + loss=Flux.crossentropy, + batch_size=5, +) +``` + +The keyord arguments `imsize`, `inchannels` and `nclasses` are +dissallowed in `kwargs` (see above). + +""" +image_builder(metalhead_constructor) = MetalheadWrapper(metalhead_constructor) + +function (pre_builder::MetalheadWrapper)(args...; kwargs...) + kw_names = keys(kwargs) + isempty(intersect(kw_names, DISALLOWED_KWARGS)) || + throw(ERR_METALHEAD_DISALLOWED_KWARGS) + return MetalheadBuilder(pre_builder.metalhead_constructor, args, kwargs) +end + +MLJFlux.build( + b::MetalheadBuilder, + rng, + n_in, + n_out, + n_channels +) = b.metalhead_constructor( + b.args...; + b.kwargs..., + imsize=n_in, + inchannels=n_channels, + nclasses=n_out +) + +# See above "TODO" list. +function VGGHack( + depth::Integer=16; + imsize=nothing, + inchannels=3, + nclasses=1000, + batchnorm=false, + pretrain=false, +) + + # Note `imsize` is ignored, as here: + # https://github.com/FluxML/Metalhead.jl/blob/9edff63222720ff84671b8087dd71eb370a6c35a/src/convnets/vgg.jl#L165 + + @assert( + depth in keys(Metalhead.vgg_config), + "depth must be from one in $(sort(collect(keys(Metalhead.vgg_config))))" + ) + model = Metalhead.VGG((224, 224); + config = Metalhead.vgg_conv_config[Metalhead.vgg_config[depth]], + inchannels, + batchnorm, + nclasses, + fcsize = 4096, + dropout = 0.5) + if pretrain && !batchnorm + Metalhead.loadpretrain!(model, string("VGG", depth)) + elseif pretrain + Metalhead.loadpretrain!(model, "VGG$(depth)-BN)") + end + return model +end diff --git a/src/types.jl b/src/types.jl index bf5674af..6a36c2be 100644 --- a/src/types.jl +++ b/src/types.jl @@ -50,6 +50,8 @@ doc_classifier(model_name) = doc_regressor(model_name)*""" for Model in [:NeuralNetworkClassifier, :ImageClassifier] + default_builder_ex = Model == :ImageClassifier ? :(image_builder(VGGHack)()) : Short() + ex = quote mutable struct $Model{B,F,O,L} <: MLJFluxProbabilistic builder::B @@ -65,7 +67,7 @@ for Model in [:NeuralNetworkClassifier, :ImageClassifier] acceleration::AbstractResource # eg, `CPU1()` or `CUDALibs()` end - function $Model(; builder::B = Short() + function $Model(; builder::B = $default_builder_ex , finaliser::F = Flux.softmax , optimiser::O = Flux.Optimise.ADAM() , loss::L = Flux.crossentropy diff --git a/test/builders.jl b/test/builders.jl index 030cbfa0..cd9d4f00 100644 --- a/test/builders.jl +++ b/test/builders.jl @@ -1,3 +1,11 @@ +# # Helpers + +function an_image(rng, n_in, n_channels) + n_channels == 3 && + return coerce(rand(rng, Float32, n_in..., 3), ColorImage) + return coerce(rand(rng, Float32, n_in...), GreyImage) +end + # to control chain initialization: myinit(n, m) = reshape(convert(Vector{Float32}, (1:n*m)), n , m) @@ -52,9 +60,11 @@ end end @testset_accelerated "@builder" accel begin - builder = MLJFlux.@builder(Flux.Chain(Flux.Dense(n_in, 4, - init = (out, in) -> randn(rng, out, in)), - Flux.Dense(4, n_out))) + builder = MLJFlux.@builder(Flux.Chain(Flux.Dense( + n_in, + 4, + init = (out, in) -> randn(rng, out, in) + ), Flux.Dense(4, n_out))) rng = StableRNGs.StableRNG(123) chain = MLJFlux.build(builder, rng, 5, 3) ps = Flux.params(chain) diff --git a/test/metalhead.jl b/test/metalhead.jl new file mode 100644 index 00000000..8c937e54 --- /dev/null +++ b/test/metalhead.jl @@ -0,0 +1,59 @@ +using StableRNGs +using MLJFlux +const Metalhead = MLJFlux.Metalhead + +@testset "display" begin + io = IOBuffer() + builder = MLJFlux.image_builder(MLJFlux.Metalhead.ResNet)(50, pretrain=false) + show(io, MIME("text/plain"), builder) + @test String(take!(io)) == + "builder wrapping Metalhead.ResNet\n args:\n"* + " 1: 50\n kwargs:\n pretrain = false\n" + show(io, builder) + @test String(take!(io)) == "image_builder(Metalhead.ResNet)(…)" + close(io) +end + +@testset "disallowed kwargs" begin + @test_throws( + MLJFlux.ERR_METALHEAD_DISALLOWED_KWARGS, + MLJFlux.image_builder(MLJFlux.Metalhead.VGG)(imsize=(241, 241)), + ) + @test_throws( + MLJFlux.ERR_METALHEAD_DISALLOWED_KWARGS, + MLJFlux.image_builder(MLJFlux.Metalhead.VGG)(inchannels=2), + ) + @test_throws( + MLJFlux.ERR_METALHEAD_DISALLOWED_KWARGS, + MLJFlux.image_builder(MLJFlux.Metalhead.VGG)(nclasses=10), + ) +end + +@testset "constructors" begin + depth = 16 + imsize = (128, 128) + nclasses = 10 + inchannels = 1 + wrapped = MLJFlux.image_builder(Metalhead.VGG) + @test wrapped.metalhead_constructor == Metalhead.VGG + builder = wrapped(depth, batchnorm=true) + @test builder.metalhead_constructor == Metalhead.VGG + @test builder.args == (depth, ) + @test (; builder.kwargs...) == (; batchnorm=true) + ref_chain = Metalhead.VGG( + imsize; + config = Metalhead.vgg_conv_config[Metalhead.vgg_config[depth]], + inchannels, + batchnorm=true, + nclasses, + fcsize = 4096, + dropout = 0.5 + ) + # needs https://github.com/FluxML/Metalhead.jl/issues/176 + # chain = + # MLJFlux.build(builder, StableRNGs.StableRNG(123), imsize, nclasses, inchannels) + # @test length.(MLJFlux.Flux.params(ref_chain)) == + # length.(MLJFlux.Flux.params(chain)) +end + +true diff --git a/test/mlj_model_interface.jl b/test/mlj_model_interface.jl index 6b15aca4..24b9a59e 100644 --- a/test/mlj_model_interface.jl +++ b/test/mlj_model_interface.jl @@ -6,10 +6,6 @@ ModelType = MLJFlux.NeuralNetworkRegressor @test model == clone clone.optimiser.eta *= 10 @test model != clone - - clone = deepcopy(model) - clone.builder.dropout *= 0.5 - @test clone != model end @testset "clean!" begin diff --git a/test/runtests.jl b/test/runtests.jl index ab44a92f..b0e84fd0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -57,6 +57,10 @@ end include("builders.jl") end +@testset "metalhead" begin + include("metalhead.jl") +end + @testset "mlj_model_interface" begin include("mlj_model_interface.jl") end From 4ed6a8c6c49f07238b21112b989c293cee4ee274 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Tue, 28 Jun 2022 15:37:17 +1200 Subject: [PATCH 2/8] get rid of intermediate wrapper --- src/metalhead.jl | 31 ++++++++++++------------------- src/types.jl | 2 +- test/metalhead.jl | 18 ++++++++++-------- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/metalhead.jl b/src/metalhead.jl index d0ec1a07..48602a71 100644 --- a/src/metalhead.jl +++ b/src/metalhead.jl @@ -2,14 +2,12 @@ TODO: After https://github.com/FluxML/Metalhead.jl/issues/176: -- Export and externally document `metal` method +- Export and externally document `image_builder` method - Delete definition of `ResNetHack` below - Change default builder in ImageClassifier (see /src/types.jl) from - `image_builder(ResNetHack(...))` to `image_builder(Metalhead.ResNet(...))`, - -- Add nicer `show` methods for `MetalheadBuilder` instances + `image_builder(ResNetHack)` to `image_builder(Metalhead.ResNet)`. =# @@ -22,19 +20,12 @@ const ERR_METALHEAD_DISALLOWED_KWARGS = ArgumentError( # # WRAPPING -struct MetalheadWrapper{F} <: MLJFlux.Builder - metalhead_constructor::F -end - struct MetalheadBuilder{F} <: MLJFlux.Builder metalhead_constructor::F args kwargs end -Base.show(io::IO, w::MetalheadWrapper) = - print(io, "image_builder($(repr(w.metalhead_constructor)))") - function Base.show(io::IO, ::MIME"text/plain", w::MetalheadBuilder) println(io, "builder wrapping $(w.metalhead_constructor)") if !isempty(w.args) @@ -52,14 +43,14 @@ function Base.show(io::IO, ::MIME"text/plain", w::MetalheadBuilder) end Base.show(io::IO, w::MetalheadBuilder) = - print(io, "image_builder($(repr(w.metalhead_constructor)))(…)") + print(io, "image_builder($(repr(w.metalhead_constructor)), …)") """ - image_builder(constructor)(args...; kwargs...) + image_builder(metalhead_constructor, args...; kwargs...) Return an MLJFlux builder object based on the Metalhead.jl constructor/type -`constructor` (eg, `Metalhead.ResNet`). Here `args` and `kwargs` are +`metalhead_constructor` (eg, `Metalhead.ResNet`). Here `args` and `kwargs` are passed to the `MetalheadType` constructor at "build time", along with the extra keyword specifiers `imsize=...`, `inchannels=...` and `nclasses=...`, with values inferred from the data. @@ -77,7 +68,7 @@ then in MLJFlux, it suffices to do ```julia using MLJFlux, Metalhead -builder = image_builder(ResNet)(50, pretrain=true) +builder = image_builder(ResNet, 50, pretrain=true) ``` which can be used in `ImageClassifier` as in @@ -96,13 +87,15 @@ The keyord arguments `imsize`, `inchannels` and `nclasses` are dissallowed in `kwargs` (see above). """ -image_builder(metalhead_constructor) = MetalheadWrapper(metalhead_constructor) - -function (pre_builder::MetalheadWrapper)(args...; kwargs...) +function image_builder( + metalhead_constructor, + args...; + kwargs... +) kw_names = keys(kwargs) isempty(intersect(kw_names, DISALLOWED_KWARGS)) || throw(ERR_METALHEAD_DISALLOWED_KWARGS) - return MetalheadBuilder(pre_builder.metalhead_constructor, args, kwargs) + return MetalheadBuilder(metalhead_constructor, args, kwargs) end MLJFlux.build( diff --git a/src/types.jl b/src/types.jl index 6a36c2be..968dacbf 100644 --- a/src/types.jl +++ b/src/types.jl @@ -50,7 +50,7 @@ doc_classifier(model_name) = doc_regressor(model_name)*""" for Model in [:NeuralNetworkClassifier, :ImageClassifier] - default_builder_ex = Model == :ImageClassifier ? :(image_builder(VGGHack)()) : Short() + default_builder_ex = Model == :ImageClassifier ? :(image_builder(VGGHack)) : Short() ex = quote mutable struct $Model{B,F,O,L} <: MLJFluxProbabilistic diff --git a/test/metalhead.jl b/test/metalhead.jl index 8c937e54..4260ff78 100644 --- a/test/metalhead.jl +++ b/test/metalhead.jl @@ -4,28 +4,28 @@ const Metalhead = MLJFlux.Metalhead @testset "display" begin io = IOBuffer() - builder = MLJFlux.image_builder(MLJFlux.Metalhead.ResNet)(50, pretrain=false) + builder = MLJFlux.image_builder(MLJFlux.Metalhead.ResNet, 50, pretrain=false) show(io, MIME("text/plain"), builder) @test String(take!(io)) == "builder wrapping Metalhead.ResNet\n args:\n"* " 1: 50\n kwargs:\n pretrain = false\n" show(io, builder) - @test String(take!(io)) == "image_builder(Metalhead.ResNet)(…)" + @test String(take!(io)) == "image_builder(Metalhead.ResNet, …)" close(io) end @testset "disallowed kwargs" begin @test_throws( MLJFlux.ERR_METALHEAD_DISALLOWED_KWARGS, - MLJFlux.image_builder(MLJFlux.Metalhead.VGG)(imsize=(241, 241)), + MLJFlux.image_builder(MLJFlux.Metalhead.VGG, imsize=(241, 241)), ) @test_throws( MLJFlux.ERR_METALHEAD_DISALLOWED_KWARGS, - MLJFlux.image_builder(MLJFlux.Metalhead.VGG)(inchannels=2), + MLJFlux.image_builder(MLJFlux.Metalhead.VGG, inchannels=2), ) @test_throws( MLJFlux.ERR_METALHEAD_DISALLOWED_KWARGS, - MLJFlux.image_builder(MLJFlux.Metalhead.VGG)(nclasses=10), + MLJFlux.image_builder(MLJFlux.Metalhead.VGG, nclasses=10), ) end @@ -34,9 +34,11 @@ end imsize = (128, 128) nclasses = 10 inchannels = 1 - wrapped = MLJFlux.image_builder(Metalhead.VGG) - @test wrapped.metalhead_constructor == Metalhead.VGG - builder = wrapped(depth, batchnorm=true) + builder = MLJFlux.image_builder( + Metalhead.VGG, + depth, + batchnorm=true + ) @test builder.metalhead_constructor == Metalhead.VGG @test builder.args == (depth, ) @test (; builder.kwargs...) == (; batchnorm=true) From d04a50cb48198e5a6c2dbcc6b6f3a69e977f74fb Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 8 Jul 2022 09:27:10 +1200 Subject: [PATCH 3/8] rm redundant test helper --- test/builders.jl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/builders.jl b/test/builders.jl index cd9d4f00..8aafa862 100644 --- a/test/builders.jl +++ b/test/builders.jl @@ -1,11 +1,3 @@ -# # Helpers - -function an_image(rng, n_in, n_channels) - n_channels == 3 && - return coerce(rand(rng, Float32, n_in..., 3), ColorImage) - return coerce(rand(rng, Float32, n_in...), GreyImage) -end - # to control chain initialization: myinit(n, m) = reshape(convert(Vector{Float32}, (1:n*m)), n , m) From c2df0d56f8c86548cac10e66a013835456aa6837 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 12 Jul 2022 04:11:56 +0000 Subject: [PATCH 4/8] address some Flux.Optimiser changes; improve image testing fix Flux compat --- Project.toml | 2 +- src/core.jl | 28 +++------------------------- src/metalhead.jl | 2 +- src/types.jl | 6 +++--- test/classifier.jl | 2 +- test/core.jl | 6 +++--- test/image.jl | 40 ++++++++++++++++++++++------------------ test/regressor.jl | 2 +- test/runtests.jl | 29 ++++++++++++++++++++--------- 9 files changed, 55 insertions(+), 62 deletions(-) diff --git a/Project.toml b/Project.toml index dd10ab8b..229b13a9 100644 --- a/Project.toml +++ b/Project.toml @@ -19,7 +19,7 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" CategoricalArrays = "0.10" ColorTypes = "0.10.3, 0.11" ComputationalResources = "0.3.2" -Flux = "0.10.4, 0.11, 0.12, 0.13" +Flux = "0.13" Metalhead = "0.7" MLJModelInterface = "1.1.1" ProgressMeter = "1.7.1" diff --git a/src/core.jl b/src/core.jl index de2a982d..d94d9f22 100644 --- a/src/core.jl +++ b/src/core.jl @@ -1,30 +1,8 @@ ## EXPOSE OPTIMISERS TO MLJ (for eg, tuning) -# Here we make the optimiser structs "transparent" so that their -# field values are exposed by calls to MLJ.params - -for opt in (:Descent, - :Momentum, - :Nesterov, - :RMSProp, - :ADAM, - :RADAM, - :AdaMax, - :OADAM, - :ADAGrad, - :ADADelta, - :AMSGrad, - :NADAM, - :AdaBelief, - :Optimiser, - :InvDecay, :ExpDecay, :WeightDecay, - :ClipValue, - :ClipNorm) # last updated: Flux.jl 0.12.3 - - @eval begin - MLJModelInterface.istransparent(m::Flux.$opt) = true - end -end +# make the optimiser structs "transparent" so that their field values +# are exposed by calls to MLJ.params: +MLJModelInterface.istransparent(m::Flux.Optimise.AbstractOptimiser) = true ## GENERAL METHOD TO OPTIMIZE A CHAIN diff --git a/src/metalhead.jl b/src/metalhead.jl index 48602a71..f42bb514 100644 --- a/src/metalhead.jl +++ b/src/metalhead.jl @@ -77,7 +77,7 @@ which can be used in `ImageClassifier` as in clf = ImageClassifier( builder=builder, epochs=500, - optimiser=Flux.ADAM(0.001), + optimiser=Flux.Adam(0.001), loss=Flux.crossentropy, batch_size=5, ) diff --git a/src/types.jl b/src/types.jl index 968dacbf..7d3166a0 100644 --- a/src/types.jl +++ b/src/types.jl @@ -13,7 +13,7 @@ Instantiate an MLJFlux model. Available hyperparameters: `MLJFlux.Short(n_hidden=0, dropout=0.5, σ=Flux.σ)` (classifiers) - `optimiser`: The optimiser to use for training. Default = - `Flux.ADAM()` + `Flux.Adam()` - `loss`: The loss function used for training. Default = `Flux.mse` (regressors) and `Flux.crossentropy` (classifiers) @@ -69,7 +69,7 @@ for Model in [:NeuralNetworkClassifier, :ImageClassifier] function $Model(; builder::B = $default_builder_ex , finaliser::F = Flux.softmax - , optimiser::O = Flux.Optimise.ADAM() + , optimiser::O = Flux.Optimise.Adam() , loss::L = Flux.crossentropy , epochs = 10 , batch_size = 1 @@ -123,7 +123,7 @@ for Model in [:NeuralNetworkRegressor, :MultitargetNeuralNetworkRegressor] end function $Model(; builder::B = Linear() - , optimiser::O = Flux.Optimise.ADAM() + , optimiser::O = Flux.Optimise.Adam() , loss::L = Flux.mse , epochs = 10 , batch_size = 1 diff --git a/test/classifier.jl b/test/classifier.jl index 135c3020..55bade43 100644 --- a/test/classifier.jl +++ b/test/classifier.jl @@ -19,7 +19,7 @@ end |> categorical; # TODO: replace Short2 -> Short when # https://github.com/FluxML/Flux.jl/issues/1372 is resolved: builder = Short2() -optimiser = Flux.Optimise.ADAM(0.03) +optimiser = Flux.Optimise.Adam(0.03) losses = [] diff --git a/test/core.jl b/test/core.jl index 75e03636..823ca16d 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4,7 +4,7 @@ stable_rng = StableRNGs.StableRNG(123) rowvec(y) = y rowvec(y::Vector) = reshape(y, 1, length(y)) -@test MLJFlux.MLJModelInterface.istransparent(Flux.ADAM(0.1)) +@test MLJFlux.MLJModelInterface.istransparent(Flux.Adam(0.1)) @testset "nrows" begin Xmatrix = rand(stable_rng, 10, 3) @@ -112,7 +112,7 @@ epochs = 10 _chain_yes_drop, history = MLJFlux.fit!(model.loss, penalty, chain_yes_drop, - Flux.Optimise.ADAM(0.001), + Flux.Optimise.Adam(0.001), epochs, 0, data[1], @@ -124,7 +124,7 @@ epochs = 10 _chain_no_drop, history = MLJFlux.fit!(model.loss, penalty, chain_no_drop, - Flux.Optimise.ADAM(0.001), + Flux.Optimise.Adam(0.001), epochs, 0, data[1], diff --git a/test/image.jl b/test/image.jl index 1866b1ed..f3d6837c 100644 --- a/test/image.jl +++ b/test/image.jl @@ -1,4 +1,22 @@ -## BASIC IMAGE TESTS GREY +# # HELPERS + +function make_images(rng; n_classes=33, n_images=50, color=false, noise=0.05) + n_channels = color ? 3 : 1 + image_bag = map(1:n_classes) do _ + rand(stable_rng, Float32, 6, 6, n_channels) + end + labels = rand(stable_rng, 1:3, n_images) + images = map(labels) do j + image_bag[j] + noise*rand(stable_rng, Float32, 6, 6, n_channels) + end + T = color ? ColorImage : GrayImage + X = coerce(cat(images...; dims=4), T) + y = coerce(labels, Multiclass) + return X, y +end + + +# # BASIC IMAGE TESTS GREY Random.seed!(123) stable_rng = StableRNGs.StableRNG(123) @@ -18,16 +36,9 @@ function MLJFlux.build(model::MyNeuralNetwork, rng, ip, op, n_channels) end builder = MyNeuralNetwork((2,2), (2,2)) - -# collection of gray images as a 4D array in WHCN format: -raw_images = rand(stable_rng, Float32, 6, 6, 1, 50); - -# as a vector of Matrix{<:AbstractRGB} -images = coerce(raw_images, GrayImage); -@test scitype(images) == AbstractVector{GrayImage{6,6}} -labels = categorical(rand(stable_rng, 1:5, 50)); - +images, labels = make_images(stable_rng) losses = [] + @testset_accelerated "ImageClassifier basic tests" accel begin Random.seed!(123) @@ -136,14 +147,7 @@ reference = losses[1] ## BASIC IMAGE TESTS COLOR builder = MyNeuralNetwork((2,2), (2,2)) - -# collection of color images as a 4D array in WHCN format: -raw_images = rand(stable_rng, Float32, 6, 6, 3, 50); - -images = coerce(raw_images, ColorImage); -@test scitype(images) == AbstractVector{ColorImage{6,6}} -labels = categorical(rand(1:5, 50)); - +images, labels = make_images(stable_rng, color=true) losses = [] @testset_accelerated "ColorImages" accel begin diff --git a/test/regressor.jl b/test/regressor.jl index 0b6c7c7f..0f05ee72 100644 --- a/test/regressor.jl +++ b/test/regressor.jl @@ -6,7 +6,7 @@ X = MLJBase.table(randn(Float32, N, 5)); # TODO: replace Short2 -> Short when # https://github.com/FluxML/Flux.jl/pull/1618 is resolved: builder = Short2(σ=identity) -optimiser = Flux.Optimise.ADAM() +optimiser = Flux.Optimise.Adam() losses = [] diff --git a/test/runtests.jl b/test/runtests.jl index b0e84fd0..fc235899 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -45,38 +45,49 @@ seed!(123) include("test_utils.jl") -@testset "penalizers" begin +# enable conditional testing of modules by providing test_args +# e.g. `Pkg.test("MLJBase", test_args=["misc"])` +RUN_ALL_TESTS = isempty(ARGS) +macro conditional_testset(name, expr) + name = string(name) + esc(quote + if RUN_ALL_TESTS || $name in ARGS + @testset $name $expr + end + end) +end +@conditional_testset "penalizers" begin include("penalizers.jl") end -@testset "core" begin +@conditional_testset "core" begin include("core.jl") end -@testset "builders" begin +@conditional_testset "builders" begin include("builders.jl") end -@testset "metalhead" begin +@conditional_testset "metalhead" begin include("metalhead.jl") end -@testset "mlj_model_interface" begin +@conditional_testset "mlj_model_interface" begin include("mlj_model_interface.jl") end -@testset "regressor" begin +@conditional_testset "regressor" begin include("regressor.jl") end -@testset "classifier" begin +@conditional_testset "classifier" begin include("classifier.jl") end -@testset "image" begin +@conditional_testset "image" begin include("image.jl") end -@testset "integration" begin +@conditional_testset "integration" begin include("integration.jl") end From 64825703da71aa9dadeced2cc1f5bc842a824fd4 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 12 Jul 2022 04:24:47 +0000 Subject: [PATCH 5/8] remove MNIST tests --- Project.toml | 3 +-- test/image.jl | 59 --------------------------------------------------- 2 files changed, 1 insertion(+), 61 deletions(-) diff --git a/Project.toml b/Project.toml index 229b13a9..6ce654f4 100644 --- a/Project.toml +++ b/Project.toml @@ -28,7 +28,6 @@ julia = "1.6" [extras] LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458" MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" @@ -37,4 +36,4 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["LinearAlgebra", "MLDatasets", "MLJBase", "Random", "StableRNGs", "Statistics", "StatsBase", "Test"] +test = ["LinearAlgebra", "MLJBase", "Random", "StableRNGs", "Statistics", "StatsBase", "Test"] diff --git a/test/image.jl b/test/image.jl index f3d6837c..48fb4fd3 100644 --- a/test/image.jl +++ b/test/image.jl @@ -85,65 +85,6 @@ reference = losses[1] @test all(x->abs(x - reference)/reference < 5e-4, losses[2:end]) -## MNIST IMAGES TEST - -mutable struct MyConvBuilder <: MLJFlux.Builder end - -using MLDatasets - -ENV["DATADEPS_ALWAYS_ACCEPT"] = true -images, labels = MNIST.traindata() -images = coerce(images, GrayImage); -labels = categorical(labels); - -function flatten(x::AbstractArray) - return reshape(x, :, size(x)[end]) -end - -function MLJFlux.build(builder::MyConvBuilder, rng, n_in, n_out, n_channels) - cnn_output_size = [3,3,32] - init = Flux.glorot_uniform(rng) - return Chain( - Conv((3, 3), n_channels=>16, pad=(1,1), relu, init=init), - MaxPool((2,2)), - Conv((3, 3), 16=>32, pad=(1,1), relu, init=init), - MaxPool((2,2)), - Conv((3, 3), 32=>32, pad=(1,1), relu, init=init), - MaxPool((2,2)), - flatten, - Dense(prod(cnn_output_size), n_out, init=init)) -end - -losses = [] - -@testset_accelerated "Image MNIST" accel begin - - Random.seed!(123) - stable_rng = StableRNGs.StableRNG(123) - - model = MLJFlux.ImageClassifier(builder=MyConvBuilder(), - acceleration=accel, - batch_size=50, - rng=stable_rng) - - @time fitresult, cache, _report = - MLJBase.fit(model, 0, images[1:500], labels[1:500]); - first_last_training_loss = _report[1][[1, end]] - push!(losses, first_last_training_loss[2]) -# @show first_last_training_loss - - pred = mode.(MLJBase.predict(model, fitresult, images[501:600])); - error = misclassification_rate(pred, labels[501:600]) - @test error < 0.2 - -end - -# check different resources (CPU1, CUDALibs, etc)) give about the same loss: -reference = losses[1] -@info "Losses for each computational resource: $losses" -@test all(x->abs(x - reference)/reference < 0.05, losses[2:end]) - - ## BASIC IMAGE TESTS COLOR builder = MyNeuralNetwork((2,2), (2,2)) From 1550b8fbda27a557f094a220b7e2abafe204b066 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 14 Jul 2022 00:52:41 +0000 Subject: [PATCH 6/8] add MLJFlux.make_images oops --- src/MLJFlux.jl | 4 ++-- src/utilities.jl | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 src/utilities.jl diff --git a/src/MLJFlux.jl b/src/MLJFlux.jl index d3a88064..b0d779a4 100644 --- a/src/MLJFlux.jl +++ b/src/MLJFlux.jl @@ -1,7 +1,5 @@ module MLJFlux -export CUDALibs, CPU1 - import Flux using MLJModelInterface using MLJModelInterface.ScientificTypesBase @@ -15,6 +13,7 @@ using ComputationalResources using Random import Metalhead +include("utilities.jl") include("penalizers.jl") include("core.jl") include("builders.jl") @@ -38,5 +37,6 @@ MLJModelInterface.metadata_pkg.((NeuralNetworkRegressor, export NeuralNetworkRegressor, MultitargetNeuralNetworkRegressor export NeuralNetworkClassifier, ImageClassifier +export CUDALibs, CPU1 end #module diff --git a/src/utilities.jl b/src/utilities.jl new file mode 100644 index 00000000..88573d82 --- /dev/null +++ b/src/utilities.jl @@ -0,0 +1,44 @@ +# # IMAGE COERCION + +# Taken from ScientificTypes.jl to avoid as dependency. + +_4Dcollection = AbstractArray{<:Real, 4} + +function coerce(y::_4Dcollection, T2::Type{GrayImage}) + size(y, 3) == 1 || error("Multiple color channels encountered. "* + "Perhaps you want to use `coerce(image_collection, ColorImage)`.") + y = dropdims(y, dims=3) + return [ColorTypes.Gray.(y[:,:,idx]) for idx=1:size(y,3)] +end + +function coerce(y::_4Dcollection, T2::Type{ColorImage}) + return [broadcast(ColorTypes.RGB, y[:,:,1, idx], y[:,:,2,idx], y[:,:,3, idx]) for idx=1:size(y,4)] +end + + +# # SYNTHETIC IMAGES + +""" + make_images(rng; image_size=(6, 6), n_classes=33, n_images=50, color=false, noise=0.05) + +Return synthetic data of the form `(images, labels)` suitable for use +with MLJ's `ImageClassifier` model. All `images` are distortions of +`n_classes` fixed images. Two images with the same label correspond to +the same undistorted image. + +""" +function make_images(rng; image_size=(6, 6), n_classes=33, n_images=50, color=false, noise=0.05) + n_channels = color ? 3 : 1 + image_bag = map(1:n_classes) do _ + rand(rng, Float32, image_size..., n_channels) + end + labels = rand(rng, 1:3, n_images) + images = map(labels) do j + image_bag[j] + noise*rand(rng, Float32, image_size..., n_channels) + end + T = color ? ColorImage : GrayImage + X = coerce(cat(images...; dims=4), T) + y = categorical(labels) + return X, y +end + From c29b733c3b1757d72cf0c20bc344e6d8e9b59b15 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 14 Jul 2022 00:54:44 +0000 Subject: [PATCH 7/8] image testing improvements --- src/metalhead.jl | 7 ++++--- src/types.jl | 3 ++- test/image.jl | 48 ++++++++++++++++++++++-------------------------- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/metalhead.jl b/src/metalhead.jl index f42bb514..48d2eaa0 100644 --- a/src/metalhead.jl +++ b/src/metalhead.jl @@ -115,21 +115,22 @@ MLJFlux.build( # See above "TODO" list. function VGGHack( depth::Integer=16; - imsize=nothing, + imsize=(242,242), inchannels=3, nclasses=1000, batchnorm=false, pretrain=false, ) - # Note `imsize` is ignored, as here: + # Adapted from # https://github.com/FluxML/Metalhead.jl/blob/9edff63222720ff84671b8087dd71eb370a6c35a/src/convnets/vgg.jl#L165 + # But we do not ignore `imsize`. @assert( depth in keys(Metalhead.vgg_config), "depth must be from one in $(sort(collect(keys(Metalhead.vgg_config))))" ) - model = Metalhead.VGG((224, 224); + model = Metalhead.VGG(imsize; config = Metalhead.vgg_conv_config[Metalhead.vgg_config[depth]], inchannels, batchnorm, diff --git a/src/types.jl b/src/types.jl index 7d3166a0..df160c56 100644 --- a/src/types.jl +++ b/src/types.jl @@ -50,7 +50,8 @@ doc_classifier(model_name) = doc_regressor(model_name)*""" for Model in [:NeuralNetworkClassifier, :ImageClassifier] - default_builder_ex = Model == :ImageClassifier ? :(image_builder(VGGHack)) : Short() + default_builder_ex = + Model == :ImageClassifier ? :(image_builder(VGGHack)) : Short() ex = quote mutable struct $Model{B,F,O,L} <: MLJFluxProbabilistic diff --git a/test/image.jl b/test/image.jl index 48fb4fd3..fd038472 100644 --- a/test/image.jl +++ b/test/image.jl @@ -1,21 +1,3 @@ -# # HELPERS - -function make_images(rng; n_classes=33, n_images=50, color=false, noise=0.05) - n_channels = color ? 3 : 1 - image_bag = map(1:n_classes) do _ - rand(stable_rng, Float32, 6, 6, n_channels) - end - labels = rand(stable_rng, 1:3, n_images) - images = map(labels) do j - image_bag[j] + noise*rand(stable_rng, Float32, 6, 6, n_channels) - end - T = color ? ColorImage : GrayImage - X = coerce(cat(images...; dims=4), T) - y = coerce(labels, Multiclass) - return X, y -end - - # # BASIC IMAGE TESTS GREY Random.seed!(123) @@ -36,7 +18,7 @@ function MLJFlux.build(model::MyNeuralNetwork, rng, ip, op, n_channels) end builder = MyNeuralNetwork((2,2), (2,2)) -images, labels = make_images(stable_rng) +images, labels = MLJFlux.make_images(stable_rng) losses = [] @testset_accelerated "ImageClassifier basic tests" accel begin @@ -85,10 +67,12 @@ reference = losses[1] @test all(x->abs(x - reference)/reference < 5e-4, losses[2:end]) -## BASIC IMAGE TESTS COLOR +# # BASIC IMAGE TESTS COLOR + +# In this case we use the default ResNet builder builder = MyNeuralNetwork((2,2), (2,2)) -images, labels = make_images(stable_rng, color=true) +images, labels = MLJFlux.make_images(stable_rng, color=true) losses = [] @testset_accelerated "ColorImages" accel begin @@ -100,20 +84,18 @@ losses = [] epochs=10, acceleration=accel, rng=stable_rng) - # tests update logic, etc (see test_utililites.jl): @test basictest(MLJFlux.ImageClassifier, images, labels, model.builder, model.optimiser, 0.95, accel) - @time fitresult, cache, _report = MLJBase.fit(model, 0, images, labels) + @time fitresult, cache, _report = MLJBase.fit(model, 0, images, labels); pred = MLJBase.predict(model, fitresult, images[1:6]) first_last_training_loss = _report[1][[1, end]] push!(losses, first_last_training_loss[2]) -# @show first_last_training_loss # try with batch_size > 1: - model = MLJFlux.ImageClassifier(builder=builder, - epochs=10, + model = MLJFlux.ImageClassifier(epochs=10, + builder=builder, batch_size=2, acceleration=accel, rng=stable_rng) @@ -129,4 +111,18 @@ reference = losses[1] @info "Losses for each computational resource: $losses" @test all(x->abs(x - reference)/reference < 1e-5, losses[2:end]) + +# # SMOKE TEST FOR DEFAULT BUILDER + +images, labels = MLJFlux.make_images(stable_rng, image_size=(32, 32), n_images=12, noise=0.2, color=true); + +@testset_accelerated "ImageClassifier basic tests" accel begin + model = MLJFlux.ImageClassifier(epochs=10, + batch_size=4, + acceleration=accel, + rng=stable_rng) + fitresult, _, _ = MLJBase.fit(model, 0, images, labels); + predict(model, fitresult, images) +end + true From 677e1f0f32d7d0ee255b551d4a6f1ad13347690d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 14 Jul 2022 01:16:43 +0000 Subject: [PATCH 8/8] add catches in fit for builders or built chains incompatible with data --- src/mlj_model_interface.jl | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/mlj_model_interface.jl b/src/mlj_model_interface.jl index bfac2987..1f2e09d4 100644 --- a/src/mlj_model_interface.jl +++ b/src/mlj_model_interface.jl @@ -40,6 +40,9 @@ end # # FIT AND UPDATE +const ERR_BUILDER = + "Builder does not appear to build an architecture compatible with supplied data. " + true_rng(model) = model.rng isa Integer ? MersenneTwister(model.rng) : model.rng function MLJModelInterface.fit(model::MLJFluxModel, @@ -51,10 +54,24 @@ function MLJModelInterface.fit(model::MLJFluxModel, rng = true_rng(model) shape = MLJFlux.shape(model, X, y) - chain = build(model, rng, shape) |> move + + chain = try + build(model, rng, shape) |> move + catch ex + @error ERR_BUILDER + end + penalty = Penalty(model) data = move.(collate(model, X, y)) + x = data |> first |> first + try + chain(x) + catch ex + @error ERR_BUILDER + throw(ex) + end + optimiser = deepcopy(model.optimiser) chain, history = fit!(model.loss,