Merge branch 'master' into gabor

Author: sharanry

.travis.yml

@@ -1,5 +1,8 @@
 ## Documentation: http://docs.travis-ci.com/user/languages/julia/
 language: julia
+branches:
+  only:
+    - master
 os:
   - linux
   - osx
@@ -8,22 +11,21 @@ julia:
   - 1.3
   - 1.4
   - nightly
-# because of Zygote needs to allow failing on nightly
-matrix:
-  allow_failures:
-    - julia: nightly
 notifications:
   email: false
 after_success:
-  # push coverage results to Coveralls
-  - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Coveralls.submit(process_folder())'
+  - if [[ $TRAVIS_JULIA_VERSION = 1.4 ]] && [[ $TRAVIS_OS_NAME = linux ]]; then
+      julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Coveralls.submit(process_folder())';
+    fi
 jobs:
   include:
     - stage: "Documentation"
-      julia: 1.0
+      julia: 1.4
       os: linux
       script:
         - export DOCUMENTER_DEBUG=true
         - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
         - julia --project=docs/ docs/make.jl
       after_succes: skip
+  allow_failures:
+    - julia: nightly

Project.toml

@@ -18,7 +18,7 @@ Compat = "2.2, 3"
 Distances = "0.8"
 Requires = "1.0.1"
 SpecialFunctions = "0.8, 0.9, 0.10"
-StatsBase = "0.32"
+StatsBase = "0.32, 0.33"
 StatsFuns = "0.8, 0.9"
 ZygoteRules = "0.2"
 julia = "1.0"

README.md

@@ -1,7 +1,10 @@
-[![Build Status](https://travis-ci.org/theogf/KernelFunctions.jl.svg?branch=master)](https://travis-ci.org/theogf/KernelFunctions.jl)
-[![Coverage Status](https://coveralls.io/repos/github/theogf/KernelFunctions.jl/badge.svg?branch=master)](https://coveralls.io/github/theogf/KernelFunctions.jl?branch=master)
-[![Documentation](https://img.shields.io/badge/docs-dev-blue.svg)](https://theogf.github.io/KernelFunctions.jl/dev/)
 # KernelFunctions.jl
+
+[![Build Status](https://travis-ci.com/JuliaGaussianProcesses/KernelFunctions.jl.svg?branch=master)](https://travis-ci.com/JuliaGaussianProcesses/KernelFunctions.jl)
+[![Coverage Status](https://coveralls.io/repos/github/JuliaGaussianProcesses/KernelFunctions.jl/badge.svg?branch=master)](https://coveralls.io/github/JuliaGaussianProcesses/KernelFunctions.jl?branch=master)
+[![Documentation (stable)](https://img.shields.io/badge/docs-stable-blue.svg)](https://juliagaussianprocesses.github.io/KernelFunctions.jl/stable)
+[![Documentation (latest)](https://img.shields.io/badge/docs-dev-blue.svg)](https://juliagaussianprocesses.github.io/KernelFunctions.jl/dev)
+
 ## Kernel functions for machine learning
 
 KernelFunctions.jl provide a flexible and complete framework for kernel functions, pretransforming the input data.
@@ -43,4 +46,4 @@ Directly inspired by the [MLKernels](https://github.com/trthatcher/MLKernels.jl)
 
 ## Issues/Contributing
 
-If you notice a problem or would like to contribute by adding more kernel functions or features please [submit an issue](https://github.com/theogf/KernelFunctions.jl/issues).
+If you notice a problem or would like to contribute by adding more kernel functions or features please [submit an issue](https://github.com/JuliaGaussianProcesses/KernelFunctions.jl/issues).

docs/make.jl

@@ -12,6 +12,7 @@ makedocs(
              "Transform"=>"transform.md",
              "Metrics"=>"metrics.md",
              "Theory"=>"theory.md",
+             "Custom Kernels"=>"create_kernel.md",
              "API"=>"api.md"]
 )
 
@@ -20,6 +21,6 @@ makedocs(
 # for more information.
 deploydocs(
     deps = Deps.pip("mkdocs", "python-markdown-math"),
-    repo = "github.com/theogf/KernelFunctions.jl.git",
+    repo = "github.com/JuliaGaussianProcesses/KernelFunctions.jl.git",
     target = "build"
 )

docs/src/create_kernel.md

@@ -0,0 +1,20 @@
+## Creating your own kernel
+
+KernelFunctions.jl contains the most popular kernels already but you might want to make your own!
+
+Here is for example how one can define the Squared Exponential Kernel again :
+
+```julia
+struct MyKernel <: Kernel end
+
+KernelFunctions.kappa(::MyKernel, d2::Real) = exp(-d2)
+KernelFunctions.metric(::MyKernel) = SqEuclidean()
+```
+
+For a "Base" kernel, where the kernel function is simply a function applied on some metric between two vectors of real, you only need to:
+ - Define your struct inheriting from `Kernel`.
+ - Define a `kappa` function.
+ - Define the metric used `SqEuclidean`, `DotProduct` etc. Note that the term "metric" is here overabused.
+ - Optional : Define any parameter of your kernel as `trainable` by Flux.jl if you want to perform optimization on the parameters. We recommend wrapping all parameters in arrays to allow them to be mutable.
+
+Once these functions are defined, you can use all the wrapping functions of KernelFuntions.jl

src/KernelFunctions.jl

@@ -1,11 +1,12 @@
 """
-KernelFunctions. [Github](https://github.com/theogf/KernelFunctions.jl) [Documentation](https://theogf.github.io/KernelFunctions.jl/dev/)
+KernelFunctions. [Github](https://github.com/JuliaGaussianProcesses/KernelFunctions.jl)
+[Documentation](https://juliagaussianprocesses.github.io/KernelFunctions.jl/stable/)
 """
 module KernelFunctions
 
 export kernelmatrix, kernelmatrix!, kerneldiagmatrix, kerneldiagmatrix!, kappa
 export transform
-export params, duplicate, set! # Helpers
+export duplicate, set! # Helpers
 
 export Kernel
 export ConstantKernel, WhiteKernel, EyeKernel, ZeroKernel
@@ -45,7 +46,7 @@ include("distances/dotproduct.jl")
 include("distances/delta.jl")
 include("transform/transform.jl")
 
-for k in ["exponential","matern","polynomial","constant","rationalquad","exponentiated","cosine","maha","gabor"]
+for k in ["exponential","matern","polynomial","constant","rationalquad","exponentiated","cosine","maha","fbm","gabor"]
     include(joinpath("kernels",k*".jl"))
 end
 include("kernels/transformedkernel.jl")

src/generic.jl

@@ -16,6 +16,7 @@ Base.show(io::IO,κ::Kernel) = print(io,nameof(typeof(κ)))
 
 ### Syntactic sugar for creating matrices and using kernel functions
 for k in subtypes(BaseKernel)
+    if  k ∈ [FBMKernel] continue end #for kernels without `metric` or `kappa`
     @eval begin
         @inline (κ::$k)(d::Real) = kappa(κ,d) #TODO Add test
         @inline (κ::$k)(x::AbstractVector{<:Real}, y::AbstractVector{<:Real}) = kappa(κ, x, y)

src/kernels/constant.jl

@@ -1,5 +1,5 @@
 """
-ZeroKernel()
+    ZeroKernel()
 
 Create a kernel that always returning zero
 ```
@@ -13,28 +13,40 @@ kappa(κ::ZeroKernel, d::T) where {T<:Real} = zero(T)
 
 metric(::ZeroKernel) = Delta()
 
+Base.show(io::IO, ::ZeroKernel) = print(io, "Zero Kernel")
+
+
 """
-`WhiteKernel()`
+    WhiteKernel()
 
 ```
     κ(x,y) = δ(x,y)
 ```
-Kernel function working as an equivalent to add white noise.
+Kernel function working as an equivalent to add white noise. Can also be called via `EyeKernel()`
 """
 struct WhiteKernel <: BaseKernel end
 
+"""
+    EyeKernel()
+
+See [WhiteKernel](@ref)
+"""
 const EyeKernel = WhiteKernel
 
-kappa(κ::WhiteKernel,δₓₓ::Real) = δₓₓ
+kappa(κ::WhiteKernel, δₓₓ::Real) = δₓₓ
 
 metric(::WhiteKernel) = Delta()
 
+Base.show(io::IO, ::WhiteKernel) = print(io, "White Kernel")
+
+
 """
-`ConstantKernel(c=1.0)`
+    ConstantKernel(; c=1.0)
+
+Kernel function always returning a constant value `c`
 ```
     κ(x,y) = c
 ```
-Kernel function always returning a constant value `c`
 """
 struct ConstantKernel{Tc<:Real} <: BaseKernel
     c::Vector{Tc}
@@ -46,3 +58,5 @@ end
 kappa(κ::ConstantKernel,x::Real) = first(κ.c)*one(x)
 
 metric(::ConstantKernel) = Delta()
+
+Base.show(io::IO, κ::ConstantKernel) = print(io, "Constant Kernel (c = $(first(κ.c)))")

src/kernels/cosine.jl

@@ -1,14 +1,14 @@
 """
-    CosineKernel
+    CosineKernel()
 
 The cosine kernel is a stationary kernel for a sinusoidal given by
 ```
     κ(x,y) = cos( π * (x-y) )
 ```
-
 """
 struct CosineKernel <: BaseKernel end
 
 kappa(κ::CosineKernel, d::Real) = cospi(d)
-
 metric(::CosineKernel) = Euclidean()
+
+Base.show(io::IO, ::CosineKernel) = print(io, "Cosine Kernel")

src/kernels/exponential.jl

@@ -1,18 +1,18 @@
 """
-`SqExponentialKernel()`
+    SqExponentialKernel()
 
-The squared exponential kernel is an isotropic Mercer kernel given by the formula:
+The squared exponential kernel is a Mercer kernel given by the formula:
 ```
     κ(x,y) = exp(-‖x-y‖²)
 ```
+Can also be called via `SEKernel`, `GaussianKernel` or `SEKernel`.
 See also [`ExponentialKernel`](@ref) for a
 related form of the kernel or [`GammaExponentialKernel`](@ref) for a generalization.
 """
 struct SqExponentialKernel <: BaseKernel end
 
 kappa(κ::SqExponentialKernel, d²::Real) = exp(-d²)
 iskroncompatible(::SqExponentialKernel) = true
-
 metric(::SqExponentialKernel) = SqEuclidean()
 
 Base.show(io::IO,::SqExponentialKernel) = print(io,"Squared Exponential Kernel")
@@ -23,10 +23,11 @@ const GaussianKernel = SqExponentialKernel
 const SEKernel = SqExponentialKernel
 
 """
-`ExponentialKernel([ρ=1.0])`
-The exponential kernel is an isotropic Mercer kernel given by the formula:
+    ExponentialKernel()
+
+The exponential kernel is a Mercer kernel given by the formula:
 ```
-    κ(x,y) = exp(-ρ‖x-y‖)
+    κ(x,y) = exp(-‖x-y‖)
 ```
 """
 struct ExponentialKernel <: BaseKernel end
@@ -35,21 +36,24 @@ kappa(κ::ExponentialKernel, d::Real) = exp(-d)
 iskroncompatible(::ExponentialKernel) = true
 metric(::ExponentialKernel) = Euclidean()
 
-Base.show(io::IO,::ExponentialKernel) = print(io,"Exponential Kernel")
+Base.show(io::IO, ::ExponentialKernel) = print(io, "Exponential Kernel")
 
 ## Alias ##
 const LaplacianKernel = ExponentialKernel
 
 """
-`GammaExponentialKernel([ρ=1.0, [γ=2.0]])`
+    GammaExponentialKernel(; γ = 2.0)
+
 The γ-exponential kernel is an isotropic Mercer kernel given by the formula:
 ```
-    κ(x,y) = exp(-ρ^(2γ)‖x-y‖^(2γ))
+    κ(x,y) = exp(-‖x-y‖^(2γ))
 ```
+Where `γ > 0`, (the keyword `γ` can be replaced by `gamma`)
+For `γ = 1`, see `SqExponentialKernel` and `γ = 0.5`, see `ExponentialKernel`
 """
 struct GammaExponentialKernel{Tγ<:Real} <: BaseKernel
     γ::Vector{Tγ}
-    function GammaExponentialKernel(;gamma::T=2.0, γ::T=gamma) where {T<:Real}
+    function GammaExponentialKernel(; gamma::T=2.0, γ::T=gamma) where {T<:Real}
         @check_args(GammaExponentialKernel, γ, γ >= zero(T), "γ > 0")
         return new{T}([γ])
     end
@@ -58,3 +62,5 @@ end
 kappa(κ::GammaExponentialKernel, d²::Real) = exp(-d²^first(κ.γ))
 iskroncompatible(::GammaExponentialKernel) = true
 metric(::GammaExponentialKernel) = SqEuclidean()
+
+Base.show(io::IO, κ::GammaExponentialKernel) = print(io, "Gamma Exponential Kernel (γ = $(first(κ.γ)))")

src/kernels/exponentiated.jl

@@ -1,14 +1,15 @@
 """
-`ExponentiatedKernel([ρ=1])`
+    ExponentiatedKernel()
+
 The exponentiated kernel is a Mercer kernel given by:
 ```
-    κ(x,y) = exp(ρ²xᵀy)
+    κ(x,y) = exp(xᵀy)
 ```
 """
 struct ExponentiatedKernel <: BaseKernel end
 
 kappa(κ::ExponentiatedKernel, xᵀy::Real) = exp(xᵀy)
-
 metric(::ExponentiatedKernel) = DotProduct()
-
 iskroncompatible(::ExponentiatedKernel) = true
+
+Base.show(io::IO, ::ExponentiatedKernel) = print(io, "Exponentiated Kernel")

src/kernels/fbm.jl

@@ -0,0 +1,94 @@
+"""
+    FBMKernel(; h::Real=0.5)
+
+Fractional Brownian motion kernel with Hurst index `h` from (0,1) given by
+```
+    κ(x,y) =  ( |x|²ʰ + |y|²ʰ - |x-y|²ʰ ) / 2
+```
+
+For `h=1/2`, this is the Wiener Kernel, for `h>1/2`, the increments are
+positively correlated and for `h<1/2` the increments are negatively correlated.
+"""
+struct FBMKernel{T<:Real} <: BaseKernel
+    h::T
+    function FBMKernel(; h::T=0.5) where {T<:Real}
+        @assert h<=1.0 && h>=0.0 "FBMKernel: Given Hurst index h is invalid."
+        return new{T}(h)
+    end
+end
+
+Base.show(io::IO, κ::FBMKernel) = print(io, "Fractional Brownian Motion Kernel (h = $(k.h))")
+
+_fbm(modX, modY, modXY, h) = (modX^h + modY^h - modXY^h)/2
+
+function kernelmatrix(κ::FBMKernel, X::AbstractMatrix; obsdim::Int = defaultobs)
+    @assert obsdim ∈ [1,2] "obsdim should be 1 or 2 (see docs of kernelmatrix))"
+    modX = sum(abs2, X; dims = 3 - obsdim)
+    modXX = pairwise(SqEuclidean(), X, dims = obsdim)
+    return _fbm.(vec(modX), reshape(modX, 1, :), modXX, κ.h)
+end
+
+function kernelmatrix!(K::AbstractMatrix, κ::FBMKernel, X::AbstractMatrix; obsdim::Int = defaultobs)
+    @assert obsdim ∈ [1,2] "obsdim should be 1 or 2 (see docs of kernelmatrix))"
+    modX = sum(abs2, X; dims = 3 - obsdim)
+    modXX = pairwise(SqEuclidean(), X, dims = obsdim)
+    K .= _fbm.(vec(modX), reshape(modX, 1, :), modXX, κ.h)
+    return K
+end
+
+function kernelmatrix(
+    κ::FBMKernel,
+    X::AbstractMatrix,
+    Y::AbstractMatrix;
+    obsdim::Int = defaultobs,
+)
+    @assert obsdim ∈ [1,2] "obsdim should be 1 or 2 (see docs of kernelmatrix))"
+    modX = sum(abs2, X, dims=3-obsdim)
+    modY = sum(abs2, Y, dims=3-obsdim)
+    modXY = pairwise(SqEuclidean(), X, Y,dims=obsdim)
+    return _fbm.(vec(modX), reshape(modY, 1, :), modXY, κ.h)
+end
+
+function kernelmatrix!(
+    K::AbstractMatrix,
+    κ::FBMKernel,
+    X::AbstractMatrix,
+    Y::AbstractMatrix;
+    obsdim::Int = defaultobs,
+)
+    @assert obsdim ∈ [1,2] "obsdim should be 1 or 2 (see docs of kernelmatrix))"
+    modX = sum(abs2, X, dims=3-obsdim)
+    modY = sum(abs2, Y, dims=3-obsdim)
+    modXY = pairwise(SqEuclidean(), X, Y,dims=obsdim)
+    K .= _fbm.(vec(modX), reshape(modY, 1, :), modXY, κ.h)
+    return K
+end
+
+## Apply kernel on two vectors ##
+function _kernel(
+        κ::FBMKernel,
+        x::AbstractVector,
+        y::AbstractVector;
+        obsdim::Int = defaultobs
+    )
+    @assert length(x) == length(y) "x and y don't have the same dimension!"
+    return κ(x,y)
+end
+
+#Syntactic Sugar
+function (κ::FBMKernel)(x::AbstractVector{<:Real}, y::AbstractVector{<:Real})
+    modX = sum(abs2, x)
+    modY = sum(abs2, y)
+    modXY = sqeuclidean(x, y)
+    return (modX^κ.h + modY^κ.h - modXY^κ.h)/2
+end
+
+(κ::FBMKernel)(x::Real, y::Real) = (abs2(x)^κ.h + abs2(y)^κ.h - abs2(x-y)^κ.h)/2
+
+function (κ::FBMKernel)(X::AbstractMatrix{<:Real}, Y::AbstractMatrix{<:Real}; obsdim::Integer=defaultobs)
+    return kernelmatrix(κ, X, Y, obsdim=obsdim)
+end
+
+function (κ::FBMKernel)(X::AbstractMatrix{<:Real}; obsdim::Integer=defaultobs)
+    return kernelmatrix(κ, X, obsdim=obsdim)
+end