Merge branch 'master' into fix-typo-in-timer-millis

Author: leandromattioli

.github/ISSUE_TEMPLATE/Issue-report.yml

@@ -41,6 +41,14 @@ body:
       options:
         - latest master (checkout manually)
         - latest development Release Candidate (RC-X)
+        - v3.0.4
+        - v3.0.3
+        - v3.0.2
+        - v3.0.1
+        - v3.0.0
+        - v2.0.17
+        - v2.0.16
+        - v2.0.15
         - v2.0.14
         - v2.0.13
         - v2.0.12

.github/scripts/find_new_boards.sh

@@ -54,7 +54,7 @@ do
     break
     fi
     board_name=$(echo "$line" | cut -d '.' -f1 | cut -d '#' -f1)
-    if [ "$board_name" != "" ]
+    if [ "$board_name" != "" ] && [ "$board_name" != "esp32_family" ]
     then
         if [ "$board_name" != "$previous_board" ]
         then

.github/scripts/install-arduino-cli.sh

@@ -41,6 +41,12 @@ fi
 if [ ! -d "$ARDUINO_IDE_PATH" ] || [ ! -f "$ARDUINO_IDE_PATH/arduino-cli" ]; then
     echo "Installing Arduino CLI on $OS_NAME ..."
     mkdir -p "$ARDUINO_IDE_PATH"
-    curl -fsSL https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh | BINDIR="$ARDUINO_IDE_PATH" sh
+    if [ "$OS_IS_WINDOWS" == "1" ]; then
+        curl -fsSL https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Windows_64bit.zip -o arduino-cli.zip
+        unzip -q arduino-cli.zip -d "$ARDUINO_IDE_PATH"
+        rm arduino-cli.zip
+    else
+        curl -fsSL https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh | BINDIR="$ARDUINO_IDE_PATH" sh
+    fi
 fi
 

.github/scripts/install-arduino-core-esp32.sh

@@ -28,7 +28,11 @@ if [ ! -d "$ARDUINO_ESP32_PATH" ]; then
     #git submodule update --init --recursive > /dev/null 2>&1
 
     echo "Installing Platform Tools ..."
-    cd tools && python get.py
+    if [ "$OS_IS_WINDOWS" == "1" ]; then
+      cd tools && ./get.exe
+    else
+      cd tools && python get.py
+    fi
     cd $script_init_path
 
     echo "ESP32 Arduino has been installed in '$ARDUINO_ESP32_PATH'"

.github/scripts/install-platformio-esp32.sh

@@ -89,7 +89,9 @@ function count_sketches(){ # count_sketches <examples-path>
         if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
             continue
         fi
-        if [[ -f "$sketchdir/.test.skip" ]]; then
+        is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
+        # If the target is listed as false, skip the sketch. Otherwise, include it.
+        if [[ "$is_target" == "false" ]]; then
             continue
         fi
         echo $sketch >> sketches.txt
@@ -161,8 +163,10 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
+        is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
+        # If the target is listed as false, skip the sketch. Otherwise, include it.
         if [ "${sketchdirname}.ino" != "$sketchname" ] \
-        || [ -f "$sketchdir/.test.skip" ]; then
+        || [[ "$is_target" == "false" ]]; then
             continue
         fi
         sketchnum=$(($sketchnum + 1))

.github/scripts/merge_packages.py

@@ -5,7 +5,8 @@
 # Written by Ivan Grokhotkov, 2015
 #
 from __future__ import print_function
-from distutils.version import LooseVersion
+#from distutils.version import LooseVersion
+from packaging.version import Version
 import re
 import json
 import sys
@@ -33,7 +34,7 @@ def merge_objects(versions, obj):
 def pkgVersionNormalized(versionString):
 
     verStr = str(versionString)
-    verParts = re.split('\.|-rc', verStr, flags=re.IGNORECASE)
+    verParts = re.split('\.|-rc|-alpha', verStr, flags=re.IGNORECASE)
 
     if len(verParts) == 3:
         if (sys.version_info > (3, 0)): # Python 3
@@ -74,7 +75,8 @@ def main(args):
             print("Adding platform {0}-{1}".format(name, version), file=sys.stderr)
             pkg1['platforms'].append(platforms[name][version])
 
-    pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: LooseVersion(pkgVersionNormalized(k['version'])), reverse=True)
+    #pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: LooseVersion(pkgVersionNormalized(k['version'])), reverse=True)
+    pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: Version(pkgVersionNormalized(k['version'])), reverse=True)
 
     json.dump({'packages':[pkg1]}, sys.stdout, indent=2)
 

.github/scripts/on-push.sh

@@ -10,7 +10,8 @@ function build(){
     local chunk_index=$3
     local chunks_cnt=$4
     local build_log=$5
-    shift; shift; shift; shift; shift;
+    local sketches_file=$6
+    shift; shift; shift; shift; shift; shift;
     local sketches=$*
 
     local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
@@ -23,6 +24,9 @@ function build(){
     if [ "$OS_IS_LINUX" == "1" ]; then
         args+=" -p $ARDUINO_ESP32_PATH/libraries"
         args+=" -i $chunk_index -m $chunks_cnt"
+        if [ -n "$sketches_file" ]; then
+          args+=" -f $sketches_file"
+        fi
         if [ $build_log -eq 1 ]; then
             args+=" -l $build_log"
         fi
@@ -50,6 +54,7 @@ fi
 CHUNK_INDEX=$1
 CHUNKS_CNT=$2
 BUILD_LOG=$3
+SKETCHES_FILE=$4
 BUILD_PIO=0
 if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then
     CHUNK_INDEX=0
@@ -69,7 +74,6 @@ fi
 
 SCRIPTS_DIR="./.github/scripts"
 if [ "$BUILD_PIO" -eq 0 ]; then
-    #source ${SCRIPTS_DIR}/install-arduino-ide.sh
     source ${SCRIPTS_DIR}/install-arduino-cli.sh
     source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
@@ -95,16 +99,16 @@ if [ "$BUILD_PIO" -eq 0 ]; then
     fi
 
     #build sketches for different targets
-    build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32c6" $FQBN_ESP32C6 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32h2" $FQBN_ESP32H2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32"   $FQBN_ESP32   $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
+    build "esp32s3" $FQBN_ESP32S3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32s2" $FQBN_ESP32S2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32c3" $FQBN_ESP32C3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32c6" $FQBN_ESP32C6 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32h2" $FQBN_ESP32H2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32"   $FQBN_ESP32   "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
 
     if [ "$BUILD_LOG" -eq 1 ]; then
         #remove last comma from the last JSON object
-        sed -i '$ s/.$//' "$sizes_file"
+        sed -i '$ s/,$//' "$sizes_file"
         #echo end of JSON array
         echo "]}" >> $sizes_file
     fi

.github/scripts/on-release.sh

@@ -192,6 +192,9 @@ else
         cp -Rf "$GITHUB_WORKSPACE/variants/${variant}"      "$PKG_DIR/variants/"
     done
 fi
+cp -f  "$GITHUB_WORKSPACE/CMakeLists.txt"                   "$PKG_DIR/"
+cp -f  "$GITHUB_WORKSPACE/idf_component.yml"                "$PKG_DIR/"
+cp -f  "$GITHUB_WORKSPACE/Kconfig.projbuild"                "$PKG_DIR/"
 cp -f  "$GITHUB_WORKSPACE/package.json"                     "$PKG_DIR/"
 cp -f  "$GITHUB_WORKSPACE/programmers.txt"                  "$PKG_DIR/"
 cp -Rf "$GITHUB_WORKSPACE/cores"                            "$PKG_DIR/"
@@ -216,16 +219,23 @@ find "$PKG_DIR" -name '*.git*' -type f -delete
 ##
 RVTC_NAME="riscv32-esp-elf-gcc"
 RVTC_NEW_NAME="esp-rv32"
+X32TC_NAME="xtensa-esp32-elf-gcc"
+X32TC_NEW_NAME="esp-x32"
+XS2TC_NAME="xtensa-esp32s2-elf-gcc"
+XS2TC_NEW_NAME="esp-xs2"
+XS3TC_NAME="xtensa-esp32s3-elf-gcc"
+XS3TC_NEW_NAME="esp-xs3"
 
 # Replace tools locations in platform.txt
 echo "Generating platform.txt..."
 cat "$GITHUB_WORKSPACE/platform.txt" | \
 sed "s/version=.*/version=$RELEASE_TAG/g" | \
+sed 's/tools\.esp32-arduino-libs\.path\.windows=.*//g' | \
 sed 's/{runtime\.platform\.path}.tools.esp32-arduino-libs/\{runtime.tools.esp32-arduino-libs.path\}/g' | \
 sed 's/{runtime\.platform\.path}.tools.xtensa-esp-elf-gdb/\{runtime.tools.xtensa-esp-elf-gdb.path\}/g' | \
-sed 's/{runtime\.platform\.path}.tools.xtensa-esp32-elf/\{runtime.tools.xtensa-esp32-elf-gcc.path\}/g' | \
-sed 's/{runtime\.platform\.path}.tools.xtensa-esp32s2-elf/\{runtime.tools.xtensa-esp32s2-elf-gcc.path\}/g' | \
-sed 's/{runtime\.platform\.path}.tools.xtensa-esp32s3-elf/\{runtime.tools.xtensa-esp32s3-elf-gcc.path\}/g' | \
+sed "s/{runtime\.platform\.path}.tools.xtensa-esp32-elf/\\{runtime.tools.$X32TC_NEW_NAME.path\\}/g" | \
+sed "s/{runtime\.platform\.path}.tools.xtensa-esp32s2-elf/\\{runtime.tools.$XS2TC_NEW_NAME.path\\}/g" | \
+sed "s/{runtime\.platform\.path}.tools.xtensa-esp32s3-elf/\\{runtime.tools.$XS3TC_NEW_NAME.path\\}/g" | \
 sed 's/{runtime\.platform\.path}.tools.riscv32-esp-elf-gdb/\{runtime.tools.riscv32-esp-elf-gdb.path\}/g' | \
 sed "s/{runtime\.platform\.path}.tools.riscv32-esp-elf/\\{runtime.tools.$RVTC_NEW_NAME.path\\}/g" | \
 sed 's/{runtime\.platform\.path}.tools.esptool/\{runtime.tools.esptool_py.path\}/g' | \
@@ -341,7 +351,19 @@ rvtc_jq_arg="\
     (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$RVTC_NAME\")).version = \"$RVTC_VERSION\" |\
     (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$RVTC_NAME\")).name = \"$RVTC_NEW_NAME\" |\
     (.packages[0].tools[] | select(.name==\"$RVTC_NAME\")).version = \"$RVTC_VERSION\" |\
-    (.packages[0].tools[] | select(.name==\"$RVTC_NAME\")).name = \"$RVTC_NEW_NAME\""
+    (.packages[0].tools[] | select(.name==\"$RVTC_NAME\")).name = \"$RVTC_NEW_NAME\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$X32TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$X32TC_NAME\")).name = \"$X32TC_NEW_NAME\" |\
+    (.packages[0].tools[] | select(.name==\"$X32TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].tools[] | select(.name==\"$X32TC_NAME\")).name = \"$X32TC_NEW_NAME\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$XS2TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$XS2TC_NAME\")).name = \"$XS2TC_NEW_NAME\" |\
+    (.packages[0].tools[] | select(.name==\"$XS2TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].tools[] | select(.name==\"$XS2TC_NAME\")).name = \"$XS2TC_NEW_NAME\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$XS3TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$XS3TC_NAME\")).name = \"$XS3TC_NEW_NAME\" |\
+    (.packages[0].tools[] | select(.name==\"$XS3TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].tools[] | select(.name==\"$XS3TC_NAME\")).name = \"$XS3TC_NEW_NAME\""
 cat "$PACKAGE_JSON_TEMPLATE" | jq "$rvtc_jq_arg" > "$OUTPUT_DIR/package-$LIBS_PROJ_NAME-rvfix.json"
 PACKAGE_JSON_TEMPLATE="$OUTPUT_DIR/package-$LIBS_PROJ_NAME-rvfix.json"
 
@@ -399,15 +421,62 @@ if [ "$RELEASE_PRE" == "false" ]; then
     fi
 fi
 
-# Upload package JSONs (temporary halted to fix json generation)
+# Test the package JSONs
+
+echo "Installing arduino-cli ..."
+export PATH="/home/runner/bin:$PATH"
+source ./.github/scripts/install-arduino-cli.sh
+
+echo "Testing $PACKAGE_JSON_DEV install ..."
+echo "Updating index ..."
+arduino-cli core update-index --additional-urls "file://$OUTPUT_DIR/$PACKAGE_JSON_DEV"
+if [ $? -ne 0 ]; then echo "ERROR: Failed to update index ($?)"; exit 1; fi
+
+echo "Installing esp32 ..."
+arduino-cli core install esp32:esp32
+if [ $? -ne 0 ]; then echo "ERROR: Failed to install esp32 ($?)"; exit 1; fi
+
+echo "Compiling example ..."
+arduino-cli compile --fqbn esp32:esp32:esp32 $GITHUB_WORKSPACE/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino
+if [ $? -ne 0 ]; then echo "ERROR: Failed to compile example ($?)"; exit 1; fi
+
+echo "Uninstalling esp32 ..."
+arduino-cli core uninstall esp32:esp32
+if [ $? -ne 0 ]; then echo "ERROR: Failed to uninstall esp32 ($?)"; exit 1; fi
+
+echo "Test successful!"
+
+if [ "$RELEASE_PRE" == "false" ]; then
+    echo "Testing $PACKAGE_JSON_REL install ..."
+    echo "Updating index ..."
+    arduino-cli core update-index --additional-urls "file://$OUTPUT_DIR/$PACKAGE_JSON_REL"
+    if [ $? -ne 0 ]; then echo "ERROR: Failed to update index ($?)"; exit 1; fi
+
+    echo "Installing esp32 ..."
+    arduino-cli core install esp32:esp32
+    if [ $? -ne 0 ]; then echo "ERROR: Failed to install esp32 ($?)"; exit 1; fi
+
+    echo "Compiling example ..."
+    arduino-cli compile --fqbn esp32:esp32:esp32 $GITHUB_WORKSPACE/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino
+    if [ $? -ne 0 ]; then echo "ERROR: Failed to compile example ($?)"; exit 1; fi
+
+    echo "Uninstalling esp32 ..."
+    arduino-cli core uninstall esp32:esp32
+    if [ $? -ne 0 ]; then echo "ERROR: Failed to uninstall esp32 ($?)"; exit 1; fi
+
+    echo "Test successful!"
+fi
+
+# Upload package JSONs
+
 echo "Uploading $PACKAGE_JSON_DEV ..."
 echo "Download URL: "`git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_DEV"`
-# echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV"`
+echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV"`
 echo
 if [ "$RELEASE_PRE" == "false" ]; then
     echo "Uploading $PACKAGE_JSON_REL ..."
     echo "Download URL: "`git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_REL"`
-    # echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL"`
+    echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL"`
     echo
 fi
 

.github/scripts/set_push_chunks.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+build_all=false
+chunks_count=0
+
+if [[ $CORE_CHANGED == 'true' ]] || [[ $IS_PR != 'true' ]]; then
+  echo "Core files changed or not a PR. Building all."
+  build_all=true
+  chunks_count=$MAX_CHUNKS
+elif [[ $LIB_CHANGED == 'true' ]]; then
+  echo "Libraries changed. Building only affected sketches."
+  if [[ $NETWORKING_CHANGED == 'true' ]]; then
+    echo "Networking libraries changed. Building networking related sketches."
+    networking_sketches="$(find libraries/WiFi -name *.ino) "
+    networking_sketches+="$(find libraries/Ethernet -name *.ino) "
+    networking_sketches+="$(find libraries/PPP -name *.ino) "
+    networking_sketches+="$(find libraries/NetworkClientSecure -name *.ino) "
+    networking_sketches+="$(find libraries/WebServer -name *.ino) "
+  fi
+  if [[ $FS_CHANGED == 'true' ]]; then
+    echo "FS libraries changed. Building FS related sketches."
+    fs_sketches="$(find libraries/SD -name *.ino) "
+    fs_sketches+="$(find libraries/SD_MMC -name *.ino) "
+    fs_sketches+="$(find libraries/SPIFFS -name *.ino) "
+    fs_sketches+="$(find libraries/LittleFS -name *.ino) "
+    fs_sketches+="$(find libraries/FFat -name *.ino) "
+  fi
+  sketches="$networking_sketches $fs_sketches"
+  for file in $LIB_FILES; do
+      if [[ $file == *.ino ]]; then
+          # If file ends with .ino, add it to the list of sketches
+          echo "Sketch found: $file"
+          sketches+="$file "
+      elif [[ $(basename $(dirname $file)) == "src" ]]; then
+          # If file is in a src directory, find all sketches in the parent/examples directory
+          echo "Library src file found: $file"
+          lib=$(dirname $(dirname $file))
+          if [[ -d $lib/examples ]]; then
+              lib_sketches=$(find $lib/examples -name *.ino)
+              sketches+="$lib_sketches "
+              echo "Library sketches: $lib_sketches"
+          fi
+      else
+          # If file is in a example folder but it is not a sketch, find all sketches in the current directory
+          echo "File in example folder found: $file"
+          sketch=$(find $(dirname $file) -name *.ino)
+          sketches+="$sketch "
+          echo "Sketch in example folder: $sketch"
+      fi
+      echo ""
+  done
+fi
+
+if [[ -n $sketches ]]; then
+  # Remove duplicates
+  sketches=$(echo $sketches | tr ' ' '\n' | sort | uniq)
+  for sketch in $sketches; do
+      echo $sketch >> sketches_found.txt
+      chunks_count=$((chunks_count+1))
+  done
+  echo "Number of sketches found: $chunks_count"
+  echo "Sketches:"
+  echo "$sketches"
+
+  if [[ $chunks_count -gt $MAX_CHUNKS ]]; then
+    echo "More sketches than the allowed number of chunks found. Limiting to $MAX_CHUNKS chunks."
+    chunks_count=$MAX_CHUNKS
+  fi
+fi
+
+chunks='["0"'
+for i in $(seq 1 $(( $chunks_count - 1 )) ); do
+  chunks+=",\"$i\""
+done
+chunks+="]"
+
+echo "build_all=$build_all" >> $GITHUB_OUTPUT
+echo "build_libraries=$BUILD_LIBRARIES" >> $GITHUB_OUTPUT
+echo "build_static_sketches=$BUILD_STATIC_SKETCHES" >> $GITHUB_OUTPUT
+echo "build_idf=$BUILD_IDF" >> $GITHUB_OUTPUT
+echo "build_platformio=$BUILD_PLATFORMIO" >> $GITHUB_OUTPUT
+echo "chunk_count=$chunks_count" >> $GITHUB_OUTPUT
+echo "chunks=$chunks" >> $GITHUB_OUTPUT