add IQN (#710)

Author: findmyway

src/ReinforcementLearningCore/src/policies/learners.jl

@@ -14,7 +14,7 @@ end
 
 @functor Approximator (model,)
 
-(A::Approximator)(x) = A.model(x)
+(A::Approximator)(args...) = A.model(args...)
 
 RLBase.optimise!(A::Approximator, gs) =
     Flux.Optimise.update!(A.optimiser, Flux.params(A), gs)

src/ReinforcementLearningCore/src/utils/networks.jl

@@ -397,7 +397,7 @@ TwinNetwork(x; kw...) = TwinNetwork(; source=x, target=deepcopy(x), kw...)
 
 @functor TwinNetwork (source,)
 
-(model::TwinNetwork)(x) = model.source(x)
+(model::TwinNetwork)(args...) = model.source(args...)
 
 function RLBase.optimise!(A::Approximator{<:TwinNetwork}, gs)
     Flux.Optimise.update!(A.optimiser, Flux.params(A), gs)

src/ReinforcementLearningExperiments/deps/experiments/experiments/DQN/JuliaRL_IQN_CartPole.jl

@@ -3,7 +3,7 @@
 # title: JuliaRL\_IQN\_CartPole
 # cover: assets/JuliaRL_IQN_CartPole.png
 # description: IQN applied to CartPole
-# date: 2021-05-22
+# date: 2022-06-27
 # author: "[Jun Tian](https://github.com/findmyway)"
 # ---
 
@@ -12,18 +12,16 @@ using ReinforcementLearning
 using StableRNGs
 using Flux
 using Flux.Losses
-using CUDA
 
 function RL.Experiment(
     ::Val{:JuliaRL},
     ::Val{:IQN},
     ::Val{:CartPole},
-    ::Nothing;
-    seed = 123,
+    ; seed=123
 )
     rng = StableRNG(seed)
-    device_rng = CUDA.functional() ? CUDA.CURAND.RNG() : rng
-    env = CartPoleEnv(; T = Float32, rng = rng)
+    device_rng = rng
+    env = CartPoleEnv(; T=Float32, rng=rng)
     ns, na = length(state(env)), length(action_space(env))
     init = glorot_uniform(rng)
     Nₑₘ = 16
@@ -32,51 +30,60 @@ function RL.Experiment(
 
     nn_creator() =
         ImplicitQuantileNet(
-            ψ = Dense(ns, n_hidden, relu; init = init),
-            ϕ = Dense(Nₑₘ, n_hidden, relu; init = init),
-            header = Dense(n_hidden, na; init = init),
+            ψ=Dense(ns, n_hidden, relu; init=init),
+            ϕ=Dense(Nₑₘ, n_hidden, relu; init=init),
+            header=Dense(n_hidden, na; init=init),
         ) |> gpu
 
     agent = Agent(
-        policy = QBasedPolicy(
-            learner = IQNLearner(
-                approximator = NeuralNetworkApproximator(
-                    model = nn_creator(),
-                    optimizer = ADAM(0.001),
+        policy=QBasedPolicy(
+            learner=IQNLearner(
+                approximator=Approximator(
+                    model=TwinNetwork(
+                        ImplicitQuantileNet(
+                            ψ=Dense(ns, n_hidden, relu; init=init),
+                            ϕ=Dense(Nₑₘ, n_hidden, relu; init=init),
+                            header=Dense(n_hidden, na; init=init),
+                        ),
+                        sync_freq=100
+                    ),
+                    optimiser=ADAM(0.001),
                 ),
-                target_approximator = NeuralNetworkApproximator(model = nn_creator()),
-                κ = κ,
-                N = 8,
-                N′ = 8,
-                Nₑₘ = Nₑₘ,
-                K = 32,
-                γ = 0.99f0,
-                stack_size = nothing,
-                batch_size = 32,
-                update_horizon = 1,
-                min_replay_history = 100,
-                update_freq = 1,
-                target_update_freq = 100,
-                default_priority = 1.0f2,
-                rng = rng,
-                device_rng = device_rng,
+                κ=κ,
+                N=8,
+                N′=8,
+                Nₑₘ=Nₑₘ,
+                K=32,
+                γ=0.99f0,
+                rng=rng,
+                device_rng=device_rng,
             ),
-            explorer = EpsilonGreedyExplorer(
-                kind = :exp,
-                ϵ_stable = 0.01,
-                decay_steps = 500,
-                rng = rng,
+            explorer=EpsilonGreedyExplorer(
+                kind=:exp,
+                ϵ_stable=0.01,
+                decay_steps=500,
+                rng=rng,
             ),
         ),
-        trajectory = CircularArrayPSARTTrajectory(
-            capacity = 1000,
-            state = Vector{Float32} => (ns,),
-        ),
+        trajectory=Trajectory(
+            container=CircularArraySARTTraces(
+                capacity=1000,
+                state=Float32 => (ns,),
+            ),
+            sampler=BatchSampler{SS′ART}(
+                batch_size=32,
+                rng=rng
+            ),
+            controller=InsertSampleRatioController(
+                threshold=100,
+                n_inserted=-1
+            )
+        )
     )
 
     stop_condition = StopAfterStep(10_000, is_show_progress=!haskey(ENV, "CI"))
     hook = TotalRewardPerEpisode()
-    Experiment(agent, env, stop_condition, hook, "")
+    Experiment(agent, env, stop_condition, hook)
 end
 
 

src/ReinforcementLearningExperiments/src/ReinforcementLearningExperiments.jl

@@ -13,6 +13,7 @@ include(joinpath(EXPERIMENTS_DIR, "JuliaRL_DQN_CartPole.jl"))
 include(joinpath(EXPERIMENTS_DIR, "JuliaRL_PrioritizedDQN_CartPole.jl"))
 include(joinpath(EXPERIMENTS_DIR, "JuliaRL_QRDQN_CartPole.jl"))
 include(joinpath(EXPERIMENTS_DIR, "JuliaRL_REMDQN_CartPole.jl"))
+include(joinpath(EXPERIMENTS_DIR, "JuliaRL_IQN_CartPole.jl"))
 
 # dynamic loading environments
 function __init__() end

src/ReinforcementLearningExperiments/test/runtests.jl

@@ -8,9 +8,9 @@ run(E`JuliaRL_DQN_CartPole`)
 run(E`JuliaRL_PrioritizedDQN_CartPole`)
 run(E`JuliaRL_QRDQN_CartPole`)
 run(E`JuliaRL_REMDQN_CartPole`)
+run(E`JuliaRL_IQN_CartPole`)
 # run(E`JuliaRL_BC_CartPole`)
 # run(E`JuliaRL_Rainbow_CartPole`)
-# run(E`JuliaRL_IQN_CartPole`)
 # run(E`JuliaRL_VMPO_CartPole`)
 # run(E`JuliaRL_VPG_CartPole`)
 # run(E`JuliaRL_BasicDQN_MountainCar`)

src/ReinforcementLearningZoo/src/algorithms/dqns/dqns.jl

@@ -3,6 +3,6 @@ include("dqn.jl")
 include("prioritized_dqn.jl")
 include("qr_dqn.jl")
 include("rem_dqn.jl")
+include("iqn.jl")
 # include("rainbow.jl")
-# include("iqn.jl")
 # include("common.jl")