ci: still use aws for caches

Author: pietroalbini

.azure-pipelines/auto.fanout.yml

@@ -160,7 +160,7 @@ jobs:
 
   - script: |
       REM md sccache
-      REM powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe"
+      REM powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc"
       echo ##vso[task.prependpath]%CD%\sccache
     displayName: Download and install sccache
 
@@ -392,7 +392,7 @@ jobs:
       mkdir -p $HOME/rustsrc    
       echo "##vso[task.setvariable variable=PATH;]$PATH"
 
-      curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache
+      curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin
       chmod +x /usr/local/bin/sccache
 
       curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin

.azure-pipelines/auto.yml

@@ -1,11 +1,6 @@
 #
 # Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows.
 # 
-# Notes:
-# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled
-# and with openssl statically-linked for Linux.
-# Here's the build (with a backpointer to source) of where the bits came from:
-# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275
 
 pr: none
 # trigger:

.azure-pipelines/steps/macos.yml

@@ -3,21 +3,21 @@ steps:
 
 - bash: |
     export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/
-    mkdir -p $HOME/rustsrc    
+    mkdir -p $HOME/rustsrc
     echo "##vso[task.setvariable variable=PATH;]$PATH"
 
-    curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache
+    curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin
     chmod +x /usr/local/bin/sccache
 
     curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin
     chmod +x /usr/local/bin/stamp
-    
+
     export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang
     echo "##vso[task.setvariable variable=CC]$CC"
-    
+
     export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++
     echo "##vso[task.setvariable variable=CXX]$CXX"
-    
+
     echo "##vso[task.setvariable variable=AR]ar"
   displayName: Prep
 

.azure-pipelines/steps/run-script.yml

@@ -5,7 +5,7 @@ steps:
     date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true)
   displayName: Log time information (before)
 
-- bash: |    
+- bash: |
     which sccache
     stamp sh -x -c "$RUN_SCRIPT"
   env:
@@ -15,8 +15,8 @@ steps:
 
     # Explicitly decrypt secret variables
     # See https://docs.microsoft.com/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables
-    SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET)
-    DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET)
+    AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID)
+    AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY)
   displayName: Run script
 
 - bash: |

.azure-pipelines/steps/windows.yml

@@ -106,7 +106,7 @@ steps:
 
 - script: |
     md sccache
-    powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe"
+    powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc"
     echo ##vso[task.prependpath]%CD%\sccache
   displayName: Download and install sccache
 
@@ -155,6 +155,6 @@ steps:
 
     # explicitly decrypt secret variables
     # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch
-    SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET)
-    DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET)
+    AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID)
+    AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY)
   displayName: Run script

src/ci/docker/run.sh

@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 
-# ignore-tidy-linelength
-
 set -e
 
 export MSYS_NO_PATHCONV=1
@@ -43,21 +41,9 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       cksum=$(sha512sum $hash_key | \
         awk '{print $1}')
 
-      if [ "$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT" != "" ]; then
-        # install azcopy
-        echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod/ xenial main" > azure.list
-        sudo cp ./azure.list /etc/apt/sources.list.d/
-        sudo apt-key adv --keyserver packages.microsoft.com --recv-keys EB3E94ADBE1229CF
-        sudo apt-get update
-        sudo apt-get install azcopy
-
-        url="https://$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$DOCKER_LAYER_CACHE_AZURE_STORAGE_CONTAINER/$cksum"
-        upload="azcopy --quiet --destination $url --dest-key $DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY"
-      else
-        s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
-        url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
-        upload="aws s3 cp - $s3url"
-      fi
+      s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
+      url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
+      upload="aws s3 cp - $s3url"
 
       echo "Attempting to download $url"
       rm -f /tmp/rustci_docker_cache
@@ -134,9 +120,6 @@ if [ "$SCCACHE_BUCKET" != "" ]; then
     args="$args --env SCCACHE_REGION"
     args="$args --env AWS_ACCESS_KEY_ID"
     args="$args --env AWS_SECRET_ACCESS_KEY"
-elif [ "$SCCACHE_AZURE_CONNECTION_STRING" != "" ]; then
-    args="$args --env SCCACHE_AZURE_CONNECTION_STRING"
-    args="$args --env SCCACHE_AZURE_BLOB_CONTAINER"
 else
     mkdir -p $HOME/.cache/sccache
     args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache"

src/ci/docker/scripts/sccache.sh

@@ -1,7 +1,6 @@
 set -ex
 
-echo Use sccache built with Azure Storage support
 curl -fo /usr/local/bin/sccache \
-  https://rustlangtools.blob.core.windows.net/public/stable-x86_64-unknown-linux-musl.sccache
+  https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl
 
 chmod +x /usr/local/bin/sccache