Fix azure-sdk-tools tests bitrot, add tools gh action check (#37580)

* encode the fact that azure-eventhub does not support pypy machines
* add azure-sdk-tools test workflow
* update cspell to account for new workflow
* fix matrix generation when no direct packages are present
* fix a test relying on folder ordering to NOT depend on that folder ordering

Author: scbedd

.github/workflows/azure-sdk-tools.yml

@@ -0,0 +1,31 @@
+name: Test Azure SDK Tools
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      # todo before checkin: multiplex to include running tests on py38 -> 312
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install azure-sdk-tools
+        run: |
+          python -m pip install -e tools/azure-sdk-tools[build,ghtools]
+          python -m pip freeze
+        shell: bash
+
+      - name: Run tests
+        run: |
+          pytest ./tests
+        shell: bash
+        working-directory: tools/azure-sdk-tools

.vscode/cspell.json

@@ -135,7 +135,8 @@
     "conda/conda-recipes/msal-extensions/meta.yaml",
     "conda/conda-recipes/msrest/meta.yaml",
     "conda/conda-recipes/uamqp/meta.yaml",
-    "conda/conda-releaselogs/azure-mgmt.md"
+    "conda/conda-releaselogs/azure-mgmt.md",
+    ".github/workflows/azure-sdk-tools.yml"
   ],
   "words": [
     "msedge",

eng/pipelines/templates/jobs/ci.yml

@@ -138,7 +138,7 @@ jobs:
       parameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
-        IncludeIndirect: false
+        IncludeIndirect: true
 
     - template: /eng/pipelines/templates/steps/build-extended-artifacts.yml
       parameters:
@@ -170,7 +170,7 @@ jobs:
       parameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
         PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
-        IncludeIndirect: false
+        IncludeIndirect: true
 
     - template: ../steps/build-extended-artifacts.yml
       parameters:

eng/scripts/distribute-packages-to-matrix.ps1

@@ -151,7 +151,7 @@ function Update-Matrix {
 
     # we need to ensure the presence of TargetingString in the matrix object
     if ($matrixUpdate) {
-        if ($directBatches) {
+        if ($directBatches -or $indirectBatches) {
             if (-not $Matrix.matrix.PSObject.Properties["TargetingString"]) {
                 $Matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value @()
             }
@@ -217,7 +217,11 @@ $indirectIncludedPackages = $packageProperties | Where-Object { $_.IncludedForVa
 
 # I will assign all the direct included packages first. our goal is to get full coverage of the direct included packages
 # then, for the indirect packages, we will add them as sparse TargetingString bundles to the matrix
-$directBatches = Split-ArrayIntoBatches -InputArray $directIncludedPackages -BatchSize $BATCHSIZE
+$directBatches = @()
+if ($directIncludedPackages) {
+    $directBatches = Split-ArrayIntoBatches -InputArray $directIncludedPackages -BatchSize $BATCHSIZE
+}
+
 $indirectBatches = @()
 if ($indirectIncludedPackages) {
     $indirectBatches = Split-ArrayIntoBatches -InputArray $indirectIncludedPackages -BatchSize $BATCHSIZE

scripts/devops_tasks/tox_harness.py

@@ -318,7 +318,7 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace) -> No
                         if check not in skipped_tox_checks:
                             skipped_tox_checks[check] = []
 
-                    skipped_tox_checks[check].append(parsed_package)
+                        skipped_tox_checks[check].append(parsed_package)
 
             if not filtered_tox_environment_set:
                 logging.info(
@@ -330,7 +330,6 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace) -> No
 
             tox_execution_array.extend(["-e", filtered_tox_environment_set])
 
-
         if parsed_args.tox_env == "apistub":
             local_options_array = []
             if parsed_args.dest_dir:

tools/azure-sdk-tools/.env

@@ -0,0 +1 @@
+PROXY_URL=http://localhost:5000

tools/azure-sdk-tools/ci_tools/functions.py

@@ -43,7 +43,8 @@
 
 TEST_COMPATIBILITY_MAP = {}
 TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP = {
-    "azure-storage-blob": "pypy"
+    "azure-storage-blob": "pypy",
+    "azure-eventhub": "pypy"
 }
 
 omit_regression = (

tools/azure-sdk-tools/setup.py

@@ -17,6 +17,7 @@
     "PyYAML",
     "urllib3",
     "tomli-w==1.0.0",
+    "azure-core",
     # Perf/Build
     "ConfigArgParse>=0.12.0",
 ]

tools/azure-sdk-tools/tests/integration/test_package_discovery.py

@@ -12,7 +12,7 @@ def test_discovery():
     results = discover_targeted_packages("azure*", core_service_root)
 
     # if in a set, this should be empty
-    non_empty_results = discover_targeted_packages("azure-servicemanagement-legacy", core_service_root)
+    non_empty_results = discover_targeted_packages("azure-core", core_service_root)
 
     assert len(results) > 1
     assert len(non_empty_results) == 1
@@ -23,52 +23,56 @@ def test_discovery_omit_mgmt():
     assert [os.path.basename(result) for result in results] == [
         "azure-storage-blob",
         "azure-storage-blob-changefeed",
+        "azure-storage-extensions",
         "azure-storage-file-datalake",
         "azure-storage-file-share",
         "azure-storage-queue"
     ]
 
 def test_discovery_omit_build():
-    results = discover_targeted_packages("azure*", core_service_root, filter_type="Build")
+    results = discover_targeted_packages("*", core_service_root, filter_type="Build")
 
     assert [os.path.basename(result) for result in results] == [
         "azure-core",
         "azure-core-experimental",
         "azure-core-tracing-opencensus",
         "azure-core-tracing-opentelemetry",
         "azure-mgmt-core",
+        "corehttp"
     ]
 
 def test_discovery_single_package():
-    results = discover_targeted_packages("azure-servicemanagement-legacy", core_service_root, filter_type="Build")
+    results = discover_targeted_packages("azure-core", core_service_root, filter_type="Build")
 
     assert [os.path.basename(result) for result in results] == [
-        "azure-servicemanagement-legacy",
+        "azure-core",
     ]
 
 def test_discovery_omit_regression():
-    results = discover_targeted_packages("azure*", core_service_root, filter_type="Regression")
+    results = discover_targeted_packages("*", core_service_root, filter_type="Regression")
 
     assert [os.path.basename(result) for result in results] == [
         "azure-core",
         "azure-core-experimental",
         "azure-core-tracing-opencensus",
-        "azure-core-tracing-opentelemetry"
+        "azure-core-tracing-opentelemetry",
+        "corehttp"
     ]
 
     storage_results = discover_targeted_packages("azure*", storage_service_root, filter_type="Regression")
 
     assert [os.path.basename(result) for result in storage_results] == [
         "azure-storage-blob",
         "azure-storage-blob-changefeed",
+        "azure-storage-extensions",
         "azure-storage-file-datalake",
         "azure-storage-file-share",
         "azure-storage-queue"
     ]
 
 
 def test_discovery_honors_contains_filter():
-    
+
     storage_results = discover_targeted_packages("azure*", storage_service_root, "file", filter_type="Regression")
 
     assert [os.path.basename(result) for result in storage_results] == [
@@ -92,4 +96,3 @@ def test_discovery_honors_override():
         "azure-core-tracing-opentelemetry",
         "azure-mgmt-core",
     ]
-    

tools/azure-sdk-tools/tests/test_optional_functionality.py

@@ -3,7 +3,6 @@
 
 from ci_tools.parsing import ParsedSetup
 from ci_tools.functions import get_config_setting
-from ci_tools.scenario.generation import create_scenario_file
 
 integration_folder = os.path.join(os.path.dirname(__file__), 'integration')
 
@@ -27,17 +26,17 @@ def test_toml_result():
                 'additional_pytest_args': ['-k', '*_async.py']
             },
             {
-                'name': 'no_aiohttp', 
-                'install': [], 
-                'uninstall': ['aiohttp'], 
+                'name': 'no_aiohttp',
+                'install': [],
+                'uninstall': ['aiohttp'],
                 'additional_pytest_args': ['-k', 'not *_async.py']
             }
         ]
     }
 
     assert actual == expected
 
-    
+
 def test_optional_specific_get():
     package_with_toml = os.path.join(integration_folder, 'scenarios', 'optional_environment_two_options')
     actual = get_config_setting(package_with_toml, 'optional')
@@ -49,9 +48,9 @@ def test_optional_specific_get():
             'additional_pytest_args': ['-k', '*_async.py']
         },
         {
-            'name': 'no_aiohttp', 
-            'install': [], 
-            'uninstall': ['aiohttp'], 
+            'name': 'no_aiohttp',
+            'install': [],
+            'uninstall': ['aiohttp'],
             'additional_pytest_args': ['-k', 'not *_async.py']
         }
     ]

tools/azure-sdk-tools/tests/test_parse_functionality.py

@@ -111,7 +111,6 @@ def test_sdk_sample_setup(test_patch):
     assert result.requires == ["requests>=2.18.4", "six>=1.11.0", "typing-extensions>=4.0.1"]
     assert result.is_new_sdk == True
     assert result.setup_filename == os.path.join(package_root, "setup.py")
-    assert result.namespace == "ci_tools"
     assert "pytyped" in result.package_data
     assert result.include_package_data == True
     assert result.folder == package_root
@@ -191,7 +190,6 @@ def test_parse_recognizes_extensions(test_patch):
     assert result.requires == ["requests>=2.18.4", "six>=1.11.0", "typing-extensions>=4.0.1"]
     # todo resolve this conflict assert result.is_new_sdk == True
     assert result.setup_filename == os.path.join(package_root, "setup.py")
-    assert result.namespace == "ci_tools"
     assert "pytyped" in result.package_data
     assert result.include_package_data == True
     assert result.folder == package_root

tools/azure-sdk-tools/tests/test_requirements_parse.py

@@ -45,7 +45,7 @@ def test_replace_dev_reqs_specifiers(tmp_directory_create):
     requirements_file = create_temporary_scenario(tmp_directory_create, target_file)
 
     requirements_before = get_requirements_from_file(requirements_file)
-    replace_dev_reqs(requirements_file, core_location)
+    replace_dev_reqs(requirements_file, core_location, None)
     requirements_after = get_requirements_from_file(requirements_file)
 
     assert requirements_before == requirements_after
@@ -63,18 +63,18 @@ def test_replace_dev_reqs_relative(tmp_directory_create):
     expected_results = [
         os.path.join(expected_output_folder, "coretestserver-1.0.0b1-py3-none-any.whl"),
         os.path.join(expected_output_folder, "coretestserver-1.0.0b1-py3-none-any.whl"),
-        os.path.join(expected_output_folder, "azure_identity-1.16.0b3-py3-none-any.whl"),
-        os.path.join(expected_output_folder, "azure_identity-1.16.0b3-py3-none-any.whl"),
+        os.path.join(expected_output_folder, "azure_identity-1.18.1-py3-none-any.whl"),
+        os.path.join(expected_output_folder, "azure_identity-1.18.1-py3-none-any.whl"),
         os.path.join(expected_output_folder, "azure_mgmt_core-1.4.0-py3-none-any.whl"),
         os.path.join(expected_output_folder, "azure_mgmt_core-1.4.0-py3-none-any.whl"),
         os.path.join(expected_output_folder, "azure_sdk_tools-0.0.0-py3-none-any.whl[build]"),
         os.path.join(expected_output_folder, "azure_sdk_tools-0.0.0-py3-none-any.whl[build]"),
-        os.path.join(expected_output_folder, "azure_core-1.30.2-py3-none-any.whl"),
-        os.path.join(expected_output_folder, "azure_core-1.30.2-py3-none-any.whl"),
+        os.path.join(expected_output_folder, "azure_core-1.31.1-py3-none-any.whl"),
+        os.path.join(expected_output_folder, "azure_core-1.31.1-py3-none-any.whl"),
     ]
 
     requirements_before = get_requirements_from_file(requirements_file)
-    replace_dev_reqs(requirements_file, core_location)
+    replace_dev_reqs(requirements_file, core_location, None)
     requirements_after = get_requirements_from_file(requirements_file)
 
     assert requirements_before != requirements_after
@@ -93,7 +93,7 @@ def test_replace_dev_reqs_remote(tmp_directory_create):
     requirements_file = create_temporary_scenario(tmp_directory_create, target_file)
 
     requirements_before = get_requirements_from_file(requirements_file)
-    replace_dev_reqs(requirements_file, core_location)
+    replace_dev_reqs(requirements_file, core_location, None)
     requirements_after = get_requirements_from_file(requirements_file)
     assert requirements_before == requirements_after
 

tools/azure-sdk-tools/tests/test_whl_discovery.py

@@ -12,7 +12,7 @@
 tags_folder = os.path.join(integration_folder, "scenarios", "sample_interpreter_tags")
 
 
-def create_basic_temp_dir(tmp_directory_create) -> TemporaryDirectory:
+def create_basic_temp_dir(tmp_directory_create) -> str:
     tmp_dir = tmp_directory_create(
         [
             os.path.join("azure-common", "azure_common-1.1.29-py3-none-any.whl"),
@@ -37,13 +37,13 @@ def test_find_discovers_standard_whls(test_patch, tmp_directory_create):
     test_patch.return_value = ["py3-none-any"]
 
     # basic positive cases
-    found_core = find_whl(tmp_dir.name, "azure-core", "1.26.5")
-    found_legacy = find_whl(tmp_dir.name, "azure-servicemanagement-legacy", "0.20.7")
+    found_core = find_whl(tmp_dir, "azure-core", "1.26.5")
+    found_legacy = find_whl(tmp_dir, "azure-servicemanagement-legacy", "0.20.7")
     assert found_core is not None
     assert found_legacy is not None
 
     # basic negative cases
-    not_found_core = find_whl(tmp_dir.name, "azure-core", "1.26.4")
+    not_found_core = find_whl(tmp_dir, "azure-core", "1.26.4")
     assert not_found_core is None
 
 
@@ -52,7 +52,7 @@ def test_find_whl_fails_on_incompatible_interpreter(test_patch, tmp_directory_cr
     tmp_dir = create_basic_temp_dir(tmp_directory_create)
     test_patch.return_value = []
 
-    found = find_whl(tmp_dir.name, "azure-core", "1.26.5")
+    found = find_whl(tmp_dir, "azure-core", "1.26.5")
     assert found is None