Merge branch 'master' into compathelper/new_version/2021-09-20-00-46-08-917-00277351318

Author: st--

.github/workflows/benchmark.yml

@@ -0,0 +1,24 @@
+name: Run benchmarks
+
+on:
+  pull_request:
+
+jobs:
+  Benchmark:
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'performance critical')
+    env:
+      JULIA_DEBUG: BenchmarkCI
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: 1.6
+      - name: Install dependencies
+        run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI"'
+      - name: Run benchmarks
+        run: julia -e "using BenchmarkCI; BenchmarkCI.judge()"
+      - name: Post results
+        run: julia -e "using BenchmarkCI; BenchmarkCI.postjudge()"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -58,20 +58,8 @@ jobs:
           GROUP: ${{ matrix.group }}
       - uses: julia-actions/julia-processcoverage@v1
         if: matrix.version == '1' && matrix.os == 'ubuntu-latest'
-      - name: Coveralls parallel
+      - name: Send coverage to CodeCov
         if: matrix.version == '1' && matrix.os == 'ubuntu-latest'
-        uses: coverallsapp/github-action@master
+        uses: codecov/codecov-action@v2
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          path-to-lcov: ./lcov.info
-          flag-name: run-${{ matrix.group }}
-          parallel: true
-  finish:
-    needs: test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Send coverage
-        uses: coverallsapp/github-action@master
-        with:
-          github-token: ${{ secrets.github_token }}
-          parallel-finished: true
+            file: lcov.info

.gitignore

@@ -4,3 +4,4 @@
 /test/Manifest.toml
 coverage/
 .DS_store
+benchmark/Manifest.toml

Project.toml

@@ -1,6 +1,6 @@
 name = "KernelFunctions"
 uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
-version = "0.10.18"
+version = "0.10.27"
 
 [deps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
@@ -30,7 +30,7 @@ Functors = "0.1, 0.2"
 IrrationalConstants = "0.1"
 LogExpFunctions = "0.2.1, 0.3"
 Requires = "1.0.1"
-SpecialFunctions = "0.8, 0.9, 0.10, 1"
+SpecialFunctions = "0.8, 0.9, 0.10, 1, 2"
 StatsBase = "0.32, 0.33"
 TensorCore = "0.1"
 ZygoteRules = "0.2"

README.md

@@ -1,7 +1,7 @@
 # KernelFunctions.jl
 
 ![CI](https://github.com/JuliaGaussianProcesses/KernelFunctions.jl/workflows/CI/badge.svg?branch=master)
-[![Coverage Status](https://coveralls.io/repos/github/JuliaGaussianProcesses/KernelFunctions.jl/badge.svg?branch=master)](https://coveralls.io/github/JuliaGaussianProcesses/KernelFunctions.jl?branch=master)
+[![codecov](https://codecov.io/gh/JuliaGaussianProcesses/KernelFunctions.jl/branch/master/graph/badge.svg?token=rmDh3gb7hN)](https://codecov.io/gh/JuliaGaussianProcesses/KernelFunctions.jl)
 [![Documentation (stable)](https://img.shields.io/badge/docs-stable-blue.svg)](https://juliagaussianprocesses.github.io/KernelFunctions.jl/stable)
 [![Documentation (latest)](https://img.shields.io/badge/docs-dev-blue.svg)](https://juliagaussianprocesses.github.io/KernelFunctions.jl/dev)
 [![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac)
@@ -12,9 +12,11 @@
 
 ## Kernel functions for machine learning
 
-KernelFunctions.jl provide a flexible and complete framework for kernel functions, pretransforming the input data.
+KernelFunctions.jl provides a flexible framework for defining kernel functions, and an extensive collection of implementations.
 
-The aim is to make the API as model-agnostic as possible while still being user-friendly.
+The aim is to make the API as model-agnostic as possible while still being user-friendly, and to interoperate well with generic packages for handling parameters like [ParameterHandling.jl](https://github.com/invenia/ParameterHandling.jl/) and FluxML's [Functors.jl](https://github.com/FluxML/Functors.jl/).
+
+Where appropriate, kernels are AD-compatible.
 
 ## Examples
 
@@ -46,12 +48,12 @@ plot(
   <img src="docs/src/assets/heatmap_combination.png" width=400px>
 </p>
 
-## Packages goals (by priority)
-- Ensure AD Compatibility (already the case for Zygote, ForwardDiff)
-- Toeplitz Matrices compatibility
+## Related Work
 
 Directly inspired by the [MLKernels](https://github.com/trthatcher/MLKernels.jl) package.
 
+See the JuliaGaussianProcesses [Github organisation](https://github.com/JuliaGaussianProcesses) and [website](https://juliagaussianprocesses.github.io/) for more related packages.
+
 ## Issues/Contributing
 
 If you notice a problem or would like to contribute by adding more kernel functions or features please [submit an issue](https://github.com/JuliaGaussianProcesses/KernelFunctions.jl/issues), or open a PR (please see the [ColPrac](https://github.com/SciML/ColPrac) contribution guidelines).

benchmark/MLKernels.jl

@@ -0,0 +1,3 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"

benchmark/Project.toml

@@ -1,23 +1,46 @@
 using BenchmarkTools
-using Random
-using Distances, LinearAlgebra
+using KernelFunctions
 
-const SUITE = BenchmarkGroup()
+N1 = 10
+N2 = 20
 
-Random.seed!(1234)
+X = rand(N1, N2)
+Xc = ColVecs(X)
+Xr = RowVecs(X)
+Xv = collect.(eachcol(X))
+Y = rand(N1, N2)
+Yc = ColVecs(Y)
+Yr = RowVecs(Y)
+Yv = collect.(eachcol(Y))
 
-dim = 50
-N1 = 1000;
-N2 = 500;
-alpha = 2.0
+# Create the general suite of benchmarks
+SUITE = BenchmarkGroup()
 
-X = rand(Float64, N1, dim)
-Y = rand(Float64, N2, dim)
+kernels = Dict(
+    "SqExponential" => SqExponentialKernel(), "Exponential" => ExponentialKernel()
+)
 
-KXY = rand(Float64, N1, N2)
-KX = rand(Float64, N1, N1)
-sKX = Symmetric(rand(Float64, N1, N1))
-kX = rand(Float64, N1)
+inputtypes = Dict("ColVecs" => (Xc, Yc), "RowVecs" => (Xr, Yr), "Vecs" => (Xv, Yv))
 
-include("kernelmatrix.jl")
-include("MLKernels.jl")
+functions = Dict(
+    "kernelmatrixX" => (kernel, X, Y) -> kernelmatrix(kernel, X),
+    "kernelmatrixXY" => (kernel, X, Y) -> kernelmatrix(kernel, X, Y),
+    "kernelmatrix_diagX" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X),
+    "kernelmatrix_diagXY" => (kernel, X, Y) -> kernelmatrix_diag(kernel, X, Y),
+)
+
+for (kname, kernel) in kernels
+    SUITE[kname] = sk = BenchmarkGroup()
+    for (inputname, (X, Y)) in inputtypes
+        sk[inputname] = si = BenchmarkGroup()
+        for (fname, f) in functions
+            si[fname] = @benchmarkable $f($kernel, $X, $Y)
+        end
+    end
+end
+
+# Uncomment the following to run benchmark locally
+
+# tune!(SUITE)
+
+# results = run(SUITE, verbose=true)

benchmark/benchmarks.jl

@@ -84,7 +84,22 @@ To find out more about the background, read this [review of kernels for vector-v
 
 KernelFunctions also provides miscellaneous utility functions.
 ```@docs
-kernelpdmat
 nystrom
 NystromFact
 ```
+
+## Conditional Utilities
+To keep the dependencies of KernelFunctions lean, some functionality is only available if specific other packages are explicitly loaded (`using`).
+
+### Kronecker.jl
+[*https://github.com/MichielStock/Kronecker.jl*](https://github.com/MichielStock/Kronecker.jl)
+```@docs
+kronecker_kernelmatrix
+kernelkronmat
+```
+
+### PDMats.jl
+[*https://github.com/JuliaStats/PDMats.jl*](https://github.com/JuliaStats/PDMats.jl)
+```@docs
+kernelpdmat
+```

benchmark/kernelmatrix.jl

@@ -27,6 +27,7 @@ CosineKernel
 
 ```@docs
 ExponentialKernel
+GibbsKernel
 LaplacianKernel
 SqExponentialKernel
 SEKernel
@@ -128,6 +129,9 @@ NormalizedKernel
 ```
 
 ## Multi-output Kernels
+Kernelfunctions implements multi-output kernels as scalar kernels on an extended output domain. For more details on this read [the section on inputs for multi-output GPs](@ref Inputs-for-Multiple-Outputs).
+
+For a function ``f(x) \\rightarrow y`` denote the inputs as ``x, x'``, such that we compute the covariance between output components ``y_{p}`` and ``y_{p'}``. The total number of outputs is ``m``.
 
 ```@docs
 MOKernel

docs/src/api.md

@@ -120,6 +120,7 @@ kernels = [
     LinearKernel(),
     compose(PeriodicKernel(), ScaleTransform(0.2)),
     NeuralNetworkKernel(),
+    GibbsKernel(; lengthscale=x -> sum(exp ∘ sin, x)),
 ]
 plot(
     [visualize(k) for k in kernels]...;

docs/src/kernels.md

@@ -1,5 +1,8 @@
 # # Support Vector Machine
 #
+# In this notebook we show how you can use KernelFunctions.jl to generate
+# kernel matrices for classification with a support vector machine, as
+# implemented by LIBSVM.
 
 using Distributions
 using KernelFunctions
@@ -8,39 +11,45 @@ using LinearAlgebra
 using Plots
 using Random
 
-## Set plotting theme
-theme(:wong)
-
 ## Set seed
 Random.seed!(1234);
 
-# Number of samples:
-N = 100;
+# ## Generate half-moon dataset
+
+# Number of samples per class:
+nin = nout = 50;
 
-# Select randomly between two classes:
-y_train = rand([-1, 1], N);
+# We generate data based on SciKit-Learn's sklearn.datasets.make_moons function:
 
-# Random attributes for both classes:
-X = Matrix{Float64}(undef, 2, N)
-rand!(MvNormal(randn(2), I), view(X, :, y_train .== 1))
-rand!(MvNormal(randn(2), I), view(X, :, y_train .== -1));
-x_train = ColVecs(X);
+class1x = cos.(range(0, π; length=nout))
+class1y = sin.(range(0, π; length=nout))
+class2x = 1 .- cos.(range(0, π; length=nin))
+class2y = 1 .- sin.(range(0, π; length=nin)) .- 0.5
+X = hcat(vcat(class1x, class2x), vcat(class1y, class2y))
+X .+= 0.1randn(size(X))
+x_train = RowVecs(X)
+y_train = vcat(fill(-1, nout), fill(1, nin));
 
-# Create a 2D grid:
+# Create a 100×100 2D grid for evaluation:
 test_range = range(floor(Int, minimum(X)), ceil(Int, maximum(X)); length=100)
 x_test = ColVecs(mapreduce(collect, hcat, Iterators.product(test_range, test_range)));
 
+# ## SVM model
+#
 # Create kernel function:
-k = SqExponentialKernel() ∘ ScaleTransform(2.0)
+k = SqExponentialKernel() ∘ ScaleTransform(1.5)
 
 # [LIBSVM](https://github.com/JuliaML/LIBSVM.jl) can make use of a pre-computed kernel matrix.
 # KernelFunctions.jl can be used to produce that.
-# Precomputed matrix for training (corresponds to linear kernel)
+#
+# Precomputed matrix for training
 model = svmtrain(kernelmatrix(k, x_train), y_train; kernel=LIBSVM.Kernel.Precomputed)
 
 # Precomputed matrix for prediction
-y_pr, _ = svmpredict(model, kernelmatrix(k, x_train, x_test));
+y_pred, _ = svmpredict(model, kernelmatrix(k, x_train, x_test));
 
-# Compute prediction on a grid:
-contourf(test_range, test_range, y_pr)
-scatter!(X[1, :], X[2, :]; color=y_train, lab="data", widen=false)
+# Visualize prediction on a grid:
+plot(; lim=extrema(test_range), aspect_ratio=1)
+contourf!(test_range, test_range, y_pred; levels=1, color=cgrad(:redsblues), alpha=0.7)
+scatter!(X[y_train .== -1, 1], X[y_train .== -1, 2]; color=:red, label="class 1")
+scatter!(X[y_train .== +1, 1], X[y_train .== +1, 2]; color=:blue, label="class 2")

examples/gaussian-process-priors/script.jl

@@ -17,6 +17,7 @@ export PiecewisePolynomialKernel
 export PeriodicKernel, NeuralNetworkKernel
 export KernelSum, KernelProduct, KernelTensorProduct
 export TransformedKernel, ScaledKernel, NormalizedKernel
+export GibbsKernel
 
 export Transform,
     SelectTransform,
@@ -53,11 +54,14 @@ using Functors
 using LinearAlgebra
 using Requires
 using SpecialFunctions: loggamma, besselk, polygamma
-using IrrationalConstants: logtwo, twoπ
+using IrrationalConstants: logtwo, twoπ, invsqrt2
 using LogExpFunctions: softplus
 using StatsBase
 using TensorCore
-using ZygoteRules: ZygoteRules
+using ZygoteRules: ZygoteRules, AContext, literal_getproperty, literal_getfield
+
+# Hack to work around Zygote type inference problems.
+const Distances_pairwise = Distances.pairwise
 
 abstract type Kernel end
 abstract type SimpleKernel <: Kernel end
@@ -94,6 +98,7 @@ include("basekernels/rational.jl")
 include("basekernels/sm.jl")
 include("basekernels/wiener.jl")
 
+include("kernels/gibbskernel.jl")
 include("kernels/scaledkernel.jl")
 include("kernels/normalizedkernel.jl")
 include("matrix/kernelmatrix.jl")