Updated to Flux 0.14, removed POMDPSimulators, updated CI

.github/workflows/CI.yml

@@ -1,16 +1,36 @@
 name: CI
-
-on: [push, pull_request]
-
+on:
+  push:
+    branches:
+      - master
+    tags: '*'
+  pull_request:
 jobs:
   test:
-    runs-on: ubuntu-latest
-
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "1"
+          - "1.9" # min supported version
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+        arch:
+          - x64
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
         with:
-          version: '1'
-          arch: x64
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v3
+        with:
+          files: lcov.info

.gitignore

@@ -4,4 +4,6 @@
 *log**/
 log*
 *.bson
-events.out.tfevents*
+events.out.tfevents*
+.vscode
+Manifest.toml

Project.toml

@@ -1,7 +1,7 @@
 name = "DeepQLearning"
 uuid = "de0a67f4-c691-11e8-0034-5fc6e16e22d3"
 repo = "https://github.com/JuliaPOMDP/DeepQLearning.jl"
-version = "0.6.5"
+version = "0.7.0"
 
 [deps]
 BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
@@ -21,21 +21,20 @@ TensorBoardLogger = "899adc3e-224a-11e9-021f-63837185c80f"
 [compat]
 BSON = "0.2, 0.3"
 CommonRLInterface = "0.2, 0.3"
-EllipsisNotation = "0.4, 1.0"
-Flux = "0.10, 0.11, 0.12"
+EllipsisNotation = "1"
+Flux = "0.14"
 POMDPLinter = "0.1"
 POMDPTools = "0.1"
 POMDPs = "0.9"
 Parameters = "0.12"
-StatsBase = "0.32, 0.33"
+StatsBase = "0.32, 0.33, 0.34"
 TensorBoardLogger = "0.1"
-julia = "1"
+julia = "1.9"
 
 [extras]
 POMDPModels = "355abbd5-f08e-5560-ac9e-8b5f2592a0ca"
-POMDPSimulators = "e0d0a172-29c6-5d4e-96d0-f262df5d01fd"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["POMDPModels", "POMDPSimulators", "StaticArrays", "Test"]
+test = ["POMDPModels", "StaticArrays", "Test"]

README.md

@@ -1,7 +1,7 @@
 # DeepQLearning
 
-[![Build status](https://github.com/JuliaPOMDP/DeepQLearning.jl/workflows/CI/badge.svg)](https://github.com/JuliaPOMDP/DeepQLearning.jl/actions)
-[![CodeCov](https://codecov.io/gh/JuliaPOMDP/DeepQLearning.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/JuliaPOMDP/DeepQLearning.jl)
+[![Build status](https://github.com/JuliaPOMDP/DeepQLearning.jl/workflows/CI/badge.svg)](https://github.com/JuliaPOMDP/DeepQLearning.jl/actions/workflows/CI.yml)
+[![codecov](https://codecov.io/github/JuliaPOMDP/DeepQLearning.jl/branch/master/graph/badge.svg?token=EfDZPMisVB)](https://codecov.io/github/JuliaPOMDP/DeepQLearning.jl)
 
 This package provides an implementation of the Deep Q learning algorithm for solving MDPs. For more information see https://arxiv.org/pdf/1312.5602.pdf.
 It uses POMDPs.jl and Flux.jl
@@ -17,12 +17,6 @@ It supports the following innovations:
 
 ```Julia
 using Pkg
-# Pkg.Registry.add("https://github.com/JuliaPOMDP/Registry) # for julia 1.1+
-
-# for julia 1.0 add the registry throught the POMDP package
-# Pkg.add("POMDPs")
-# using POMDPs
-# POMDPs.add_registry() 
 Pkg.add("DeepQLearning")
 ```
 

src/dueling.jl

@@ -50,9 +50,9 @@ function create_dueling_network(m::Chain)
         @assert isa(l, Dense) error_str
     end
     nlayers = length(m.layers)
-    _, last_layer_size = size(m[end].W)
+    _, last_layer_size = size(m[end].weight)
     val = Chain([deepcopy(m[i]) for i=duel_layer+1:nlayers-1]..., Dense(last_layer_size, 1))
     adv = Chain([deepcopy(m[i]) for i=duel_layer+1:nlayers]...)
-    base = Chain([deepcopy(m[i]) for i=1:duel_layer+1-1]...)
+    base = Chain(identity, [deepcopy(m[i]) for i=1:duel_layer+1-1]...)
     return DuelingNetwork(base, val, adv)
 end

src/solver.jl

@@ -37,7 +37,7 @@
     return solve(solver, env)
 end
 
-function POMDPs.solve(solver::DeepQLearningSolver, env::AbstractEnv)
+function POMDPs.solve(solver::DeepQLearningSolver, env::AbstractEnv)
     action_map = collect(actions(env))
     action_indices = Dict(a=>i for (i, a) in enumerate(action_map))
 
@@ -56,14 +56,14 @@
     return dqn_train!(solver, env, policy, replay)
 end
 
-function dqn_train!(solver::DeepQLearningSolver, env::AbstractEnv, policy::AbstractNNPolicy, replay)
+function dqn_train!(solver::DeepQLearningSolver, env::AbstractEnv, policy::AbstractNNPolicy, replay)
     if solver.logdir !== nothing 
         logger = TBLogger(solver.logdir)
         solver.logdir = logger.logdir
     end
     active_q = getnetwork(policy) # shallow copy
     target_q = deepcopy(active_q)
-    optimizer = ADAM(solver.learning_rate)
+    optimizer = Adam(solver.learning_rate)
     # start training
     resetstate!(policy)
     reset!(env)
@@ -177,7 +177,7 @@
     return policy
 end
 
-function initialize_replay_buffer(solver::DeepQLearningSolver, env::AbstractEnv, action_indices)
+function initialize_replay_buffer(solver::DeepQLearningSolver, env::AbstractEnv, action_indices)
     # init and populate replay buffer
     if solver.recurrence
         replay = EpisodeReplayBuffer(env, solver.buffer_size, solver.batch_size, solver.trace_length)
@@ -200,7 +200,7 @@
     s_batch, a_batch, r_batch, sp_batch, done_batch, indices, importance_weights = sample(replay)
 
     active_q = getnetwork(policy) 
-    p = params(active_q)
+    p = Flux.params(active_q)
 
     loss_val = nothing
     td_vals = nothing
@@ -237,7 +237,7 @@
 
 # for RNNs
 function batch_train!(solver::DeepQLearningSolver,
-                      env::AbstractEnv,
+                      env::AbstractEnv,
                       policy::AbstractNNPolicy,
                       optimizer,
                       target_q,
@@ -249,7 +249,7 @@
     Flux.reset!(active_q)
     Flux.reset!(target_q)
 
-    p = params(active_q)
+    p = Flux.params(active_q)
 
     loss_val = nothing
     td_vals = nothing
@@ -289,7 +289,7 @@
 
 function save_model(solver::DeepQLearningSolver, active_q, scores_eval::Float64, saved_mean_reward::Float64, model_saved::Bool)
     if scores_eval >= saved_mean_reward
-        bson(joinpath(solver.logdir, "qnetwork.bson"), qnetwork=[w for w in params(active_q)])
+        bson(joinpath(solver.logdir, "qnetwork.bson"), qnetwork=[w for w in Flux.params(active_q)])
         if solver.verbose
             @printf("Saving new model with eval reward %1.3f \n", scores_eval)
         end
@@ -304,7 +304,7 @@
     restore_best_model(solver, env)
 end
 
-function restore_best_model(solver::DeepQLearningSolver, env::AbstractEnv)
+function restore_best_model(solver::DeepQLearningSolver, env::AbstractEnv)
     if solver.dueling
         active_q = create_dueling_network(solver.qnetwork)
     else

test/flux_test.jl

@@ -3,7 +3,6 @@ using POMDPs
 using Random
 using DeepQLearning
 using POMDPModels
-using POMDPSimulators
 using POMDPTools
 using RLInterface
 using Test
@@ -114,8 +113,8 @@ l, td = loss(q_sa, q_targets)
 
 Flux.data(l)
 
-optimizer = ADAM(Flux.params(active_q), 1e-3)
+optimizer = Adam(Flux.params(active_q), 1e-3)
 
 # use deep copy to update the target network 
 
-# use Flux.reset to reset RNN if necessary
+# use Flux.reset to reset RNN if necessary

test/runtests.jl

@@ -1,6 +1,5 @@
 using DeepQLearning
 using POMDPModels
-using POMDPSimulators
 using POMDPTools
 using Flux
 using Random
@@ -203,6 +202,10 @@ end
     end
 
     RL.reset!(env::SimpleEnv) = env.s = 1
+    RL.state(env::SimpleEnv) = env.s
+    RL.setstate!(env::SimpleEnv, s::Int) = env.s = s
+    RL.setstate!(env::SimpleEnv, s::Float32) = env.s = Int(s)
+    RL.setstate!(env::SimpleEnv, s::Vector{Float32}) = env.s = Int(s[1])
     RL.actions(env::SimpleEnv) = [-1, 1]
     RL.observe(env::SimpleEnv) = Float32[env.s]
     RL.terminated(env::SimpleEnv) = env.s >= 3