Dev (#18)

* make DiscreteSpace more flexible

* format

* add VBasedPolicy

* add policies

* set compat of Julia to v1.3

* drop ReinforcementLearningEnvironments.jl in deps

* update compat

Author: findmyway

Project.toml

@@ -4,29 +4,23 @@ authors = ["Jun Tian <[email protected]>"]
 version = "0.1.0"
 
 [deps]
-CuArrays = "3a865a2d-5b23-5a0f-bc46-62713ec82fae"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
-Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 ReinforcementLearningBase = "e575027e-6cd6-5018-9292-cdc6200d2b44"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 
 [compat]
-Reexport = "0.2"
-StatsBase = "0.32"
-MacroTools = "0.5"
-ProgressMeter = "1.2"
 Distributions = "0.22"
-ReinforcementLearningBase = "0.5"
-CuArrays = "1.7"
-Flux = "0.10"
-julia = "1"
+ProgressMeter = "1.2"
+ReinforcementLearningBase = "0.6"
+StatsBase = "0.32"
+julia = "1.3"
 
 [extras]
+ReinforcementLearningEnvironments = "25e41dd2-4622-11e9-1641-f1adca772921"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test"]
+test = ["Test", "ReinforcementLearningEnvironments"]

src/ReinforcementLearningCore.jl

@@ -1,12 +1,10 @@
 module ReinforcementLearningCore
 
-using Reexport
+using ReinforcementLearningBase
 
 const RLCore = ReinforcementLearningCore
 export RLCore
 
-@reexport using ReinforcementLearningBase
-
 include("utils/utils.jl")
 include("core/core.jl")
 include("components/components.jl")

src/components/agents/agent.jl

@@ -23,6 +23,8 @@ Base.@kwdef mutable struct Agent{P<:AbstractPolicy,T<:AbstractTrajectory,R} <: A
     role::R = DEFAULT_PLAYER
 end
 
+RLBase.get_role(agent::Agent) = agent.role
+
 function (agent::Agent{<:AbstractPolicy,<:EpisodicCompactSARTSATrajectory})(
     ::PreEpisodeStage,
     obs,
@@ -35,9 +37,9 @@ function (agent::Agent{<:AbstractPolicy,<:EpisodicCompactSARTSATrajectory})(
     ::PreActStage,
     obs,
 )
-    update!(agent.policy, agent.trajectory)
     action = agent.policy(obs)
     push!(agent.trajectory; state = get_state(obs), action = action)
+    update!(agent.policy, agent.trajectory)
     action
 end
 
@@ -55,6 +57,7 @@ function (agent::Agent{<:AbstractPolicy,<:EpisodicCompactSARTSATrajectory})(
 )
     action = agent.policy(obs)
     push!(agent.trajectory; state = get_state(obs), action = action)
+    update!(agent.policy, agent.trajectory)
     action
 end
 
@@ -72,9 +75,9 @@ function (agent::Agent{<:AbstractPolicy,<:CircularCompactSARTSATrajectory})(
     ::PreActStage,
     obs,
 )
-    update!(agent.policy, agent.trajectory)
     action = agent.policy(obs)
     push!(agent.trajectory; state = get_state(obs), action = action)
+    update!(agent.policy, agent.trajectory)
     action
 end
 
@@ -92,6 +95,7 @@ function (agent::Agent{<:AbstractPolicy,<:CircularCompactSARTSATrajectory})(
 )
     action = agent.policy(obs)
     push!(agent.trajectory; state = get_state(obs), action = action)
+    update!(agent.policy, agent.trajectory)
     action
 end
 
@@ -109,9 +113,9 @@ function (agent::Agent{<:AbstractPolicy,<:VectorialCompactSARTSATrajectory})(
     ::PreActStage,
     obs,
 )
-    update!(agent.policy, agent.trajectory)
     action = agent.policy(obs)
     push!(agent.trajectory; state = get_state(obs), action = action)
+    update!(agent.policy, agent.trajectory)
     action
 end
 
@@ -129,5 +133,6 @@ function (agent::Agent{<:AbstractPolicy,<:VectorialCompactSARTSATrajectory})(
 )
     action = agent.policy(obs)
     push!(agent.trajectory; state = get_state(obs), action = action)
+    update!(agent.policy, agent.trajectory)
     action
 end

src/components/agents/agents.jl

@@ -1 +1,2 @@
 include("agent.jl")
+include("dyna_agent.jl")

src/components/agents/dyna_agent.jl

@@ -0,0 +1,60 @@
+export DynaAgent
+
+Base.@kwdef struct DynaAgent{P<:AbstractPolicy, B<:AbstractTrajectory, M<:AbstractEnvironmentModel, R} <: AbstractAgent
+    policy::P
+    model::M
+    trajectory::B
+    role::R = DEFAULT_PLAYER
+    plan_step::Int = 10
+end
+
+RLBase.get_role(agent::DynaAgent) = agent.role
+
+function (agent::DynaAgent{<:AbstractPolicy,<:EpisodicCompactSARTSATrajectory})(
+    ::PreEpisodeStage,
+    obs,
+)
+    empty!(agent.trajectory)
+    nothing
+end
+
+function (agent::DynaAgent{<:AbstractPolicy,<:EpisodicCompactSARTSATrajectory})(
+    ::PreActStage,
+    obs,
+)
+    action = agent.policy(obs)
+    push!(agent.trajectory; state = get_state(obs), action = action)
+    update!(agent.model, agent.trajectory, agent.policy)  # model learning
+    update!(agent.policy, agent.trajectory)  # direct learning
+    update!(agent.policy, agent.model, agent.trajectory, agent.plan_step)  # policy learning
+    action
+end
+
+function (agent::DynaAgent{<:AbstractPolicy,<:EpisodicCompactSARTSATrajectory})(
+    ::PostActStage,
+    obs,
+)
+    push!(agent.trajectory; reward = get_reward(obs), terminal = get_terminal(obs))
+    nothing
+end
+
+function (agent::DynaAgent{<:AbstractPolicy,<:EpisodicCompactSARTSATrajectory})(
+    ::PostEpisodeStage,
+    obs,
+)
+    action = agent.policy(obs)
+    push!(agent.trajectory; state = get_state(obs), action = action)
+    update!(agent.model, agent.trajectory, agent.policy)  # model learning
+    update!(agent.policy, agent.trajectory)  # direct learning
+    update!(agent.policy, agent.model, agent.trajectory, agent.plan_step)  # policy learning
+    action
+end
+
+"By default, only use trajectory to update model"
+RLBase.update!(model::AbstractEnvironmentModel, t::AbstractTrajectory, π::AbstractPolicy) =
+    update!(model, t)
+
+function RLBase.update!(model::AbstractEnvironmentModel, buffer::AbstractTrajectory)
+    transitions = extract_experience(buffer, model)
+    isnothing(transitions) || update!(model, transitions)
+end

src/components/approximators/tabular_approximator.jl

@@ -1,21 +1,48 @@
 export TabularApproximator
 
 """
-    TabularApproximator(table::Vector{Float64}) -> TabularApproximator
-    TabularApproximator(;n_state::Int, init::Float64=0.0) -> TabularApproximator
+    TabularApproximator(table<:AbstractArray)
 
-Use a `table` of type `Vector{Float64}` of length `ns` to record the state values.
+For `table` of 1-d, it will create a [`VApproximator`](@ref). For `table` of 2-d, it will create a [`QApproximator`].
+
+!!! warning
+    For `table` of 2-d, the first dimension is action and the second dimension is state.
 """
-struct TabularApproximator <: AbstractApproximator
-    table::Vector{Float64}
+struct TabularApproximator{N, T<:AbstractArray} <: AbstractApproximator
+    table::T
+    function TabularApproximator(table::T) where {T<:AbstractArray}
+        n = ndims(table)
+        n <= 2 || throw(ArgumentError("the dimention of table must be <= 2"))
+        new{n,T}(table)
+    end
+end
+
+function TabularApproximator(;n_state, n_action=nothing, init=0.)
+    table = isnothing(n_action) ? fill(init, n_state) : fill(init, n_action, n_state)
+    TabularApproximator(table)
 end
 
-TabularApproximator(; n_state::Int, init::Float64 = 0.0) =
-    TabularApproximator(fill(init, n_state))
+(app::TabularApproximator{1})(s::Int) = @views app.table[s]
 
-(v::TabularApproximator)(s::Int) = v.table[s]
+(app::TabularApproximator{2})(s::Int) = @views app.table[:, s]
+(app::TabularApproximator{2})(s::Int, a::Int) = app(s)[a]
 
-function RLBase.update!(v::TabularApproximator, correction::Pair{Int,Float64})
+function RLBase.update!(app::TabularApproximator{1}, correction::Pair)
     s, e = correction
-    v.table[s] += e
+    app.table[s] += e
+end
+
+function RLBase.update!(app::TabularApproximator{2}, correction::Pair)
+    (s, a), e = correction
+    app.table[a, s] += e
+end
+
+function RLBase.update!(Q::TabularApproximator{2}, correction::Pair{Int,Vector{Float64}})
+    s, errors = correction
+    for (a, e) in enumerate(errors)
+        Q.table[a, s] += e
+    end
 end
+
+RLBase.ApproximatorStyle(::TabularApproximator{1}) = VApproximator()
+RLBase.ApproximatorStyle(::TabularApproximator{2}) = QApproximator()

src/components/components.jl

@@ -1,8 +1,7 @@
-include("spaces/spaces.jl")
-include("environments/environments.jl")
-include("policies/random_policy.jl")
+include("learners/learners.jl")
+include("policies/policies.jl")
 include("approximators/approximators.jl")
 include("explorers/explorers.jl")
 include("trajectories/trajectories.jl")
 include("preprocessors.jl")
-include("agents/agent.jl")
+include("agents/agents.jl")