esp/ci: Add package tests

Author: gerekon

.gitlab-ci.yml

@@ -46,7 +46,9 @@ before_script:
     TARGET: "Xtensa;RISCV"
     USE_LINKER: "ld"
     CROSS_BUILD_MINGW: "OFF"
+    BUILD_TARGET_LIBS: "OFF"
     RUN_CORE_TESTS: "OFF"
+    RUN_PKG_TESTS: "OFF"
     RUN_TARGET_LIB_TESTS: "OFF"
     PACK_TOOLCHAIN: "ON"
     PACK_STANDALONE_LIBS: "ON"
@@ -59,6 +61,7 @@ before_script:
         [ ! -f "${BUILD_DIR}/tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/tests.log || true
         [ ! -f "${BUILD_DIR}/compiler-rt-tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/compiler-rt-tests.log || true
         [ ! -f "${BUILD_DIR}/lld-tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/lld-tests.log || true
+        [ ! -f "${BUILD_DIR}/pkg-tests.log" ] || grep -i "internal compiler error\|Killed" ${BUILD_DIR}/pkg-tests.log || true
       fi
   script:
     - *get_toolchain_build_scripts
@@ -72,7 +75,7 @@ before_script:
     # Re-use core tools built in another job.
     # LLVM-xxx
     - >
-      if [ "${PACK_TARGET_LIBS}" == "ON" ]; then
+      if [ "${PACK_TARGET_LIBS}" == "ON" ] || [ "${BUILD_TARGET_LIBS}" == "ON" ]; then
         echo "Enable target libraries build"
         export USE_LIBC="newlib";
         export USE_LIBCXX="libstdcxx";
@@ -134,6 +137,15 @@ before_script:
           chmod o+w ${BUILD_PATH}/compiler-rt-tests.log;
           runuser -u test_runner -- ninja -C ${BUILD_PATH} check-compiler-rt 2>&1 > ${BUILD_PATH}/compiler-rt-tests.log;
         fi
+        if [[ "${RUN_PKG_TESTS}" == "ON" ]]; then
+          echo "Run package tests";
+          # to avoid test failure "fatal: detected dubious ownership in repository at '/builds/llvm-project'"
+          chown -R test_runner $LLVM_PROJECT_PATH;
+          touch ${BUILD_PATH}/pkg-tests.log;
+          chmod o+w ${BUILD_PATH}/pkg-tests.log;
+          runuser -u test_runner -- ninja -C ${BUILD_PATH} check-package-llvm-toolchain 2>&1 > ${BUILD_PATH}/pkg-tests.log;
+          chown -R ${CUR_USER} $LLVM_PROJECT_PATH;
+        fi
         chown -R ${CUR_USER} ${BUILD_PATH};
       fi
     # pack distro
@@ -167,7 +179,11 @@ before_script:
         ARCHIVE_NAME=$(basename ${DISTRO_PACK_PATH})
         echo "${ARCHIVE_NAME}" > ${PWD}/${DIST_DIR}/target_libs_arch_name
       fi
+    - ls -l ${PWD}/${DIST_DIR}
 
+# some Clang/LLVM unit tests fail if we build Clang for RISCV+Xtensa only
+# this job is intended to run unit tests only, so it builds Clang with all backends
+# TODO: LLVM-326 and LLVM-401
 build_and_test:
   tags: [ "amd64", "build" ]
   stage: test_build
@@ -209,40 +225,26 @@ build_and_test:
     - runuser -u test_runner -- ninja -C ${BUILD_PATH} check-lld 2>&1 > ${BUILD_PATH}/lld-tests.log;
     - chown -R ${CUR_USER} ${BUILD_PATH};
 
-test_xesppie:
-  stage: test
-  dependencies:
-    - pack_x86_64-linux-gnu
-  when: manual
-  allow_failure: true
-  only:
-    - tags
-  script:
-    - cd ${DIST_DIR}/
-    - ls -l
-    - DISTRO_PACK_FILE=$(cat dist_name_x86_64-linux-gnu)
-    - tar -xf ${DISTRO_PACK_FILE} 
-    - ls -l
-    - cd esp-clang
-    - ls -l
-    - pwd
-    - export CC="$(pwd)/bin/clang"
-    - export OBJDUMP="$(pwd)/bin/llvm-objdump"
-    - git clone -q --depth=1 "${GITLAB_SSH_SERVER}/idf/esp-compiler-tests.git"
-    - cd esp-compiler-tests/build-only/xesppie
-    - ./test_xesppie.py
-
 .build_linux-gnu_template:
   extends: .build_toolchain_template
   variables:
     USE_LINKER: "gold"
 
+# Actually this job builds and packs distro for x86_64-linux-gnu,
+# but also it runs package tests. So keep it in 'test_build' stage
 build_x86_64-linux-gnu:
   extends: .build_linux-gnu_template
-  stage: build
+  stage: test_build
   variables:
     CONF_HOST: "x86_64-linux-gnu"
-
+    # Build complete distro it is necessary for running package tests
+    BUILD_TARGET_LIBS: "ON"
+    RUN_PKG_TESTS: "ON"
+
+# Target libs are built in build_x86_64-linux-gnu, but due to artifacts
+# size limit we have to produce target libs archive in this job.
+# Archive with target libs from this job will be used in pack jobs
+# for all platform except for x86_64-linux-gnu.
 build_target_libs:
   extends: .build_linux-gnu_template
   stage: build
@@ -306,7 +308,7 @@ build_aarch64-apple-darwin:
     paths:
       - ${DIST_DIR}/
     when: always
-    expire_in: 1 day
+    expire_in: 3 day
   variables:
     PACK_TOOL: "tar cJf"
     UNPACK_TOOL: "tar xJf"
@@ -328,15 +330,6 @@ build_aarch64-apple-darwin:
     - rm -rf ${DISTRO_PACK_DIR}
     - ls -l
 
-pack_x86_64-linux-gnu:
-  extends: .pack_template
-  needs:
-    # needs target libs archive from native build job
-    - job: "build_target_libs"
-    - job: "build_x86_64-linux-gnu"
-  variables:
-    CONF_HOST: "x86_64-linux-gnu"
-
 pack_x86_64-w64-mingw32:
   extends: .pack_template
   needs:
@@ -449,7 +442,7 @@ test_esp_dsp:
     matrix:
       - CHIP: esp32p4
   needs:
-    - job: "pack_x86_64-linux-gnu"
+    - job: "build_x86_64-linux-gnu"
   variables:
     # use IDF 'master' from docker image
     TEST_APP_IDF_CUSTOM_BRANCH: ""
@@ -467,6 +460,28 @@ test_esp_dsp:
     - idf.py set-target ${CHIP} 2>&1 | tee ${BUILD_LOG}
     - idf.py build 2>&1 | tee -a ${BUILD_LOG}
 
+test_xesppie:
+  stage: test
+  dependencies:
+    - build_x86_64-linux-gnu
+  allow_failure: true
+  only:
+    - tags
+  script:
+    - cd ${DIST_DIR}/
+    - ls -l
+    - DISTRO_PACK_FILE=$(cat dist_name_x86_64-linux-gnu)
+    - tar -xf ${DISTRO_PACK_FILE} 
+    - ls -l
+    - cd esp-clang
+    - ls -l
+    - pwd
+    - export CC="$(pwd)/bin/clang"
+    - export OBJDUMP="$(pwd)/bin/llvm-objdump"
+    - git clone -q --depth=1 "${GITLAB_SSH_SERVER}/idf/esp-compiler-tests.git"
+    - cd esp-compiler-tests/build-only/xesppie
+    - ./test_xesppie.py
+
 upload_to_http:
   stage: private_deploy
   when: manual
@@ -476,7 +491,7 @@ upload_to_http:
     # force the fetch strategy to clean old archives up in dist/ dir
     GIT_STRATEGY: fetch
   needs:
-    - job: pack_x86_64-linux-gnu
+    - job: build_x86_64-linux-gnu
   script:
     - cit_add_ssh_key "${HTTP_UPLOAD_KEY}"
     # List of archives
@@ -514,7 +529,7 @@ upload_to_github:
     TOOLCHAIN_SHA256_FILE: clang-${CI_COMMIT_TAG}-checksum.sha256
     LIBS_SHA256_FILE: libs-clang-${CI_COMMIT_TAG}-checksum.sha256
   needs:
-    - job: pack_x86_64-linux-gnu
+    - job: build_x86_64-linux-gnu
     - job: pack_arm-linux-gnueabihf
     - job: pack_aarch64-linux-gnu
     - job: pack_x86_64-w64-mingw32