Merge pull request #36 from JuliaMath/dev

Crown421 · web-flow · commit aa9d3a7a96ef · 2020-02-03T22:49:08.000Z
Adding BinaryProvider and tests for mutating functions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -18,7 +18,7 @@ jobs:
         os:
           - ubuntu-latest
           - macOS-latest
-#           - windows-latest
+          - windows-latest
         arch:
           - x64
     steps:
@@ -27,40 +27,36 @@ jobs:
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - run: julia --color=yes --project -e 'using Pkg; Pkg.add(PackageSpec(url=\"https://github.com/JuliaComputing/MKL.jl\")); Pkg.instantiate()'
-        if: matrix.os == 'windows-latest'
-      - run: julia --color=yes --project -e 'using Pkg; Pkg.add(PackageSpec(url="https://github.com/JuliaComputing/MKL.jl")); Pkg.instantiate()'
-        if: matrix.os != 'windows-latest'
       - uses: julia-actions/julia-buildpkg@latest
       - uses: julia-actions/julia-runtest@latest
 
-  testAptMKL:
-    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        version:
-          - '1.0'
-          - '1.3'
-        os:
-          - ubuntu-latest
-        arch:
-          - x64
-    env: 
-        LD_LIBRARY_PATH: /opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64_lin/
-    steps:
-      - name: Install MKL via apt
-        run: |
-          curl https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB > GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
-          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
-          rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
-          sudo sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
-          sudo apt-get update
-          sudo apt-get install -y intel-mkl-64bit-2019.4-070
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@latest
-        with:
-          version: ${{ matrix.version }}
-          arch: ${{ matrix.arch }}
-      - uses: julia-actions/julia-buildpkg@latest
-      - uses: julia-actions/julia-runtest@latest
+  # testAptMKL:
+  #   name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
+  #   runs-on: ${{ matrix.os }}
+  #   strategy:
+  #     matrix:
+  #       version:
+  #         - '1.0'
+  #         - '1.3'
+  #       os:
+  #         - ubuntu-latest
+  #       arch:
+  #         - x64
+  #   env: 
+  #       LD_LIBRARY_PATH: /opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64_lin/
+  #   steps:
+  #     - name: Install MKL via apt
+  #       run: |
+  #         curl https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB > GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+  #         sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+  #         rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+  #         sudo sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
+  #         sudo apt-get update
+  #         sudo apt-get install -y intel-mkl-64bit-2019.4-070
+  #     - uses: actions/checkout@v2
+  #     - uses: julia-actions/setup-julia@latest
+  #       with:
+  #         version: ${{ matrix.version }}
+  #         arch: ${{ matrix.arch }}
+  #     - uses: julia-actions/julia-buildpkg@latest
+  #     - uses: julia-actions/julia-runtest@latest
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -19,7 +19,7 @@ jobs:
         os:
           - ubuntu-latest
           - macOS-latest
-#           - windows-latest
+          - windows-latest
         arch:
           - x64
     steps:
@@ -28,40 +28,36 @@ jobs:
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - run: julia --color=yes --project -e 'using Pkg; Pkg.add(PackageSpec(url=\"https://github.com/JuliaComputing/MKL.jl\")); Pkg.instantiate()'
-        if: matrix.os == 'windows-latest'
-      - run: julia --color=yes --project -e 'using Pkg; Pkg.add(PackageSpec(url="https://github.com/JuliaComputing/MKL.jl")); Pkg.instantiate()'
-        if: matrix.os != 'windows-latest'
       - uses: julia-actions/julia-buildpkg@latest
       - uses: julia-actions/julia-runtest@latest
 
-  testAptMKL:
-    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        version:
-          - 'nightly'
-        os:
-          - ubuntu-latest
-        arch:
-          - x64
-    env: 
-        LD_LIBRARY_PATH: /opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64_lin/
-    steps:
-      - name: Install MKL via apt
-        run: |
-          curl https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB > GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
-          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
-          rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
-          sudo sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
-          sudo apt-get update
-          sudo apt-get install -y intel-mkl-64bit-2019.4-070
-          source /opt/intel/mkl/bin/mklvars.sh intel64
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@latest
-        with:
-          version: ${{ matrix.version }}
-          arch: ${{ matrix.arch }}
-      - uses: julia-actions/julia-buildpkg@latest
-      - uses: julia-actions/julia-runtest@latest
+  # testAptMKL:
+  #   name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
+  #   runs-on: ${{ matrix.os }}
+  #   strategy:
+  #     matrix:
+  #       version:
+  #         - 'nightly'
+  #       os:
+  #         - ubuntu-latest
+  #       arch:
+  #         - x64
+  #   env: 
+  #       LD_LIBRARY_PATH: /opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64_lin/
+  #   steps:
+  #     - name: Install MKL via apt
+  #       run: |
+  #         curl https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB > GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+  #         sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+  #         rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+  #         sudo sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list'
+  #         sudo apt-get update
+  #         sudo apt-get install -y intel-mkl-64bit-2019.4-070
+  #         source /opt/intel/mkl/bin/mklvars.sh intel64
+  #     - uses: actions/checkout@v2
+  #     - uses: julia-actions/setup-julia@latest
+  #       with:
+  #         version: ${{ matrix.version }}
+  #         arch: ${{ matrix.arch }}
+  #     - uses: julia-actions/julia-buildpkg@latest
+  #     - uses: julia-actions/julia-runtest@latest
diff --git a/Project.toml b/Project.toml
@@ -1,16 +1,17 @@
 name = "IntelVectorMath"
 uuid = "c8ce9da6-5d36-5c03-b118-5a70151be7bc"
-version = "0.2.1"
+version = "0.3"
 
 [deps]
+BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
 CpuId = "adafc99b-e345-5852-983c-f28acb93d879"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"
 
 [compat]
-julia = "0.7, 1.0"
 CpuId = "0.2"
 SpecialFunctions = "0.8, 0.9, 0.10"
+julia = "0.7, 1.0"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
diff --git a/README.md b/README.md
@@ -10,30 +10,20 @@ Similar packages are [Yeppp.jl](https://github.com/JuliaMath/Yeppp.jl), which wr
 
 ## Basic install
 
-To use IntelVectorMath.jl, you must have the shared libraries of the Intel Vector Math Library available on your system, for which you currently have two options:
-
-### 1. MKL.jl
-The easiest option is to use [MKL.jl](https://github.com/JuliaComputing/MKL.jl) via
+To install IntelVectorMath.jl run
 ```julia
-julia> ] add https://github.com/JuliaComputing/MKL.jl.git
+julia> ] add IntelVectorMath
 ```
-#### Note:
-This will overwrite your julia sysimage to use MKL instead of OpenBLAS which could break compatability with certain packages and is difficult to reverse without reinstalling julia from scratch. In addition, MKL.jl is currently broken on Windows, so you can only use the following option. Please see the MKL.jl repository for more information.
-
-### 2. Standalone MKL
-You can also install MKL directly [from intel](https://software.intel.com/en-us/mkl/choose-download). For macOS and Windows this requires a free registration, on Linux this can be done via the command line, as seen [here](https://github.com/JuliaMath/IntelVectorMath.jl/blob/d4f8dd4083cf228cd493a4aed9964f1bc0f08d4f/.github/workflows/main.yml#L53).
-There is also the `intel-mkl-slim` package in the AUR that works well.
-
-Note that intel MKL has a separate license, which you may want to check for commercial projects (see [FAQ]( https://software.intel.com/en-us/mkl/license-faq)).
+Since version 0.3 IntelVectorMath downloads its own version of MKL and keeps only the required files in its own directory. As such installing MKL.jl or MKL via intel are no longer required, and may mean some duplicate files if they are present. However, this package will adopt the new artifact system in the next minor version update and fix this issue. 
 
-### IntelVectorMath
-To install IntelVectorMath.jl run
+In the event that MKL was not installed properly you will get an error when first `using` it. Please try running 
 ```julia
-julia> ] add IntelVectorMath
+julia> ] build IntelVectorMath
 ```
-If you used this package prior to its renaming, you may have to run `] rm VML` first. Otherwise there will be a conflict due to the UUID.  
+If this does not work, please open an issue and include the output of `<packagedir>/deps/build.log`.
 
-In the event that MKL was not installed properly you will get an error when first `using` it.
+#### Renaming from VML
+If you used this package prior to its renaming, you may have to run `] rm VML` first. Otherwise there will be a conflict due to the UUID.  
 
 ## Using IntelVectorMath
 After loading `IntelVectorMath`, you have the supported function listed below, for example `IntelVectorMath.sin(rand(100))`. These should provide a significant speed-up over broadcasting the Base functions.
@@ -253,14 +243,13 @@ Next steps for this package
 * [x] Move Testing to GitHub Actions
 * [x] Add test for using standalone MKL
 * [x] Update Benchmarks
-* [ ] Add tests for mutating functions
+* [x] Add tests for mutating functions
+* [x] Add own dependency management via BinaryProvider
+* [ ] Update function list in README
+* [ ] Adopt Julia 1.3 artifact system, breaking backwards compatibility
 
 
 
 ## Advanced
-IntelVectorMath.jl works via Libdl which loads the relevant shared libraries. Libdl automatically finds the relevant libraries if the location of the binaries has been added to the system search paths.
-This already taken care of if you use MKL.jl, but the stand-alone may require you to source `mklvars.sh` in the shell you are opening the REPL in. The default command on Mac and Ubuntu is `source /opt/intel/mkl/bin/mklvars.sh intel64`. You may want to add this to your `.bashrc`.
-Adding a new `*.conf` file in `/etc/ld.so.conf.d` also works, as the `intel-mkl-slim` package in the AUR does automatically.
-
-Further, IntelVectorMath.jl uses [CpuId.jl](https://github.com/m-j-w/CpuId.jl) to detect if your processor supports the newer `avx2` instructions, and if not defaults to `libmkl_vml_avx`. If your system does not have AVX this package will currently not work for you.
+IntelVectorMath.jl uses [CpuId.jl](https://github.com/m-j-w/CpuId.jl) to detect if your processor supports the newer `avx2` instructions, and if not defaults to `libmkl_vml_avx`. If your system does not have AVX this package will currently not work for you.
 If the CPU feature detection does not work for you, please open an issue.
diff --git a/deps/.gitignore b/deps/.gitignore
@@ -1,2 +1,4 @@
 deps.jl
-build.log
+build.log
+
+usr/
diff --git a/deps/build.jl b/deps/build.jl
@@ -1,37 +1,62 @@
-
+using BinaryProvider # requires BinaryProvider 0.3.0 or later
 using CpuId
-using Libdl
-
-## this lets us load CpuId only once 
 
 if cpufeature(:AVX2)
-    libsuffix = :avx2
+    avxlib = "avx2"
     println("AVX2 support detected, vml_avx2 selected")
 else
-    libsuffix = :avx
+    avxlib = "avx"
     println("AVX2 support missing, vml_avx selected")
 end
 
-if Sys.iswindows()
-    rtlib = :mkl_rt
-    corelib = :mkl_core
-    lib = Symbol(:mkl_vml_, libsuffix)
-else
-    rtlib = :libmkl_rt
-    corelib = :libmkl_core
-    lib = Symbol(:libmkl_vml_, libsuffix)
-end
+# Parse some basic command-line arguments
+const verbose = "--verbose" in ARGS
+const prefix = Prefix(get([a for a in ARGS if a != "--verbose"], 1, joinpath(@__DIR__, "usr")))
+
+baseproducts = [("mkl_core", :libmkl_core), 
+                ("mkl_rt", :libmkl_rt),
+                ("mkl_vml_" * avxlib, :libmkl_vml_avx)]
+# if Sys.iswindows()
+#     push!(baseproducts, ("mkl_intel_thread", :libmkl_intel_thread))
+# end
+
+products = [LibraryProduct(prefix, ["lib"*bp[1], bp[1]], bp[2]) for bp in baseproducts]
 
+# Download binaries from hosted location
+bin_prefix = "https://github.com/JuliaPackaging/Yggdrasil/releases/download/MKL-v2019.0.117"
 
-depsjl_path = joinpath(@__DIR__, "deps.jl")
-open(depsjl_path, "w") do depsjl_file
-    println(depsjl_file, strip("""
-        ## This file was autogenerated by build.jl.
-        ## Do not edit.
-        import Libdl
+# Listing of files generated by BinaryBuilder:
+download_info = Dict(
+    Linux(:x86_64)   => ("$bin_prefix/MKL.v2019.0.117.x86_64-linux-gnu.tar.gz", "9a496908c05eccdb331218f4cf32229b024790b30bf7bc0ca0f5587e030d34e6"),
+    Linux(:i686)     => ("$bin_prefix/MKL.v2019.0.117.i686-linux-gnu.tar.gz", "ccdce675bf48738f28878bc831231498c7c9560a94f756da8c114f664790ffee"),
+    MacOS(:x86_64)   => ("$bin_prefix/MKL.v2019.0.117.x86_64-apple-darwin14.tar.gz", "605da525b16f61837bc35834f3a6ff90609b1d0a5f1606faa25bf45180ced6e9"),
+    Windows(:x86_64) => ("$bin_prefix/MKL.v2019.0.117.x86_64-w64-mingw32.tar.gz", "0a9aaac254421fde0f26a95856b595deb8cce85d039050d8167293d049fd716d"),
+    Windows(:i686)   => ("$bin_prefix/MKL.v2019.0.117.i686-w64-mingw32.tar.gz", "53cac3a29bbb2acdc383297c256e3b68b3d4712e1b83417226da4971582e80cd"),
+)
 
-        const lib = :$lib
-        const rtlib = :$rtlib
-        const corelib = :$corelib
-        """))
+# Install unsatisfied or updated dependencies:
+unsatisfied = any(!satisfied(p; verbose=verbose) for p in products)
+dl_info = choose_download(download_info, platform_key_abi())
+if dl_info === nothing && unsatisfied
+    # If we don't have a compatible .tar.gz to download, complain.
+    # Alternatively, you could attempt to install from a separate provider,
+    # build from source or something even more ambitious here.
+    error("Your platform (\"$(Sys.MACHINE)\", parsed as \"$(triplet(platform_key_abi()))\") is not supported by this package!")
 end
+
+# If we have a download, and we are unsatisfied (or the version we're
+# trying to install is not itself installed) then load it up!
+if unsatisfied || !isinstalled(dl_info...; prefix=prefix)
+    # Download and install binaries
+    install(dl_info...; prefix=prefix, force=true, verbose=verbose)
+end
+
+# Write out a deps.jl file that will contain mappings for our products
+write_deps_file(joinpath(@__DIR__, "deps.jl"), products, verbose=verbose)
+
+# cleanup for lean install 
+libfiles = readdir("usr/lib")
+idx = .!mapreduce(p -> occursin.(p[1], libfiles), (x,y) -> x.|y, baseproducts)
+rm.("usr/lib/" .* libfiles[idx])
+
+rm("usr/downloads/", recursive = true)
diff --git a/src/IntelVectorMath.jl b/src/IntelVectorMath.jl
@@ -7,8 +7,8 @@ const IVM = IntelVectorMath
 
 # import Base: .^, ./
 using SpecialFunctions
-using Libdl
-include(joinpath(dirname(@__DIR__), "deps/deps.jl"))
+# using Libdl
+include("../deps/deps.jl")
 
 include("setup.jl")
 
diff --git a/src/setup.jl b/src/setup.jl
diff --git a/test/complex.jl b/test/complex.jl
diff --git a/test/real.jl b/test/real.jl

-Original file line number
+Diff line change
@@ @@ -1,2 +1,4 @@ @@
 deps.jl
 -build.log
 +build.log
++
 +usr/