chore: merge dev with feature/0.8.0 (#408)

.github/workflows/azdev_linter.yml

@@ -30,7 +30,7 @@ jobs:
     - uses: actions/setup-python@v5
       name: Setup python
       with:
-          python-version: "3.11"
+          python-version: "3.12"
 
     # Lint
     - name: azdev linter

.github/workflows/ci_build.yml

@@ -12,10 +12,10 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v5
         with:
-            python-version: "3.9"
+            python-version: "3.12"
       - name: Build Wheel
         run: |
-          pip install wheel
+          pip install -r dev_requirements.txt
           python -m setup bdist_wheel -d dist
       - name: Upload Wheel Artifact
         uses: actions/upload-artifact@v4

.github/workflows/cluster_cleanup.yml

@@ -11,10 +11,9 @@ on:
         type: string
         description: "Resource group to clean up"
         required: true
-      keyvault_prefix:
-          type: string
-          description: "Prefix of keyvault to delete"
-          required: true
+      custom_prefix:
+        type: string
+        description: "Your custom prefix for things to delete"
     secrets:
       AZURE_CLIENT_ID:
         required: true
@@ -30,10 +29,9 @@ on:
         description: "Resource group to clean up"
         required: true
         default: ops-cli-int-test-rg
-      keyvault_prefix:
+      custom_prefix:
         type: string
-        description: "Prefix of keyvault to delete"
-        default: "opskv"
+        description: "Your custom prefix for things to delete"
         required: false
   # Run every night at midnight (Pacific) to cleanup resources
   schedule:
@@ -42,7 +40,10 @@ on:
 env:
   RESOURCE_GROUP: ${{ inputs.resource_group || 'ops-cli-int-test-rg' }}
   CLUSTER_PREFIX: ${{ inputs.cluster_prefix || 'opt' }}
-  KEYVAULT_PREFIX: ${{ inputs.keyvault_prefix || 'opskv' }}
+  # Note these are just for the init tests since we dont delete them then
+  REGISTRY_PREFIX: 'init-registry'
+  STORAGE_PREFIX: 'initstore'
+  CUSTOM_PREFIX: ${{ inputs.custom_prefix }}
 
 permissions:
   # required for OpenID federation
@@ -65,25 +66,6 @@ jobs:
           for cluster in $(az resource list -g ${{ env.RESOURCE_GROUP }} --resource-type $cluster_type --query "[?starts_with(name, '${{ env.CLUSTER_PREFIX }}')].id" -o tsv); do
             az resource delete -v --id $cluster --verbose
           done
-  keyvault-cleanup:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Az CLI login
-        uses: azure/login@v2
-        with:
-          client-id: ${{ secrets.AZURE_CLIENT_ID }}
-          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      - name: Delete keyvaults
-        run: |
-          for vault in $(az keyvault list -g ${{ env.RESOURCE_GROUP }} --query "[?starts_with(name, '${{ env.KEYVAULT_PREFIX }}')].name" -o tsv); do
-            az keyvault delete -n $vault -g ${{ env.RESOURCE_GROUP }}
-          done
-      - name: Purge keyvaults
-        run: |
-          for vault in $(az keyvault list-deleted --query "[?contains(properties.vaultId, '${{ env.RESOURCE_GROUP }}')] | [?starts_with(name, '${{ env.KEYVAULT_PREFIX }}')].name" -o tsv); do
-            az keyvault purge -n $vault --no-wait
-          done
   resource-cleanup:
     needs: [arc-cleanup]
     runs-on: ubuntu-latest
@@ -99,6 +81,22 @@ jobs:
           for resource in $(az resource list -g ${{ env.RESOURCE_GROUP }} --query "[?starts_with(name, '${{ env.CLUSTER_PREFIX }}')].id" -o tsv); do
             az resource delete -v --id $resource --verbose
           done
+      - name: Delete schema registries
+        run: |
+          for resource in $(az resource list -g ${{ env.RESOURCE_GROUP }} --query "[?starts_with(name, '${{ env.REGISTRY_PREFIX }}')].id" -o tsv); do
+            az resource delete -v --id $resource --verbose
+          done
+      - name: Delete storage accounts
+        run: |
+          for resource in $(az resource list -g ${{ env.RESOURCE_GROUP }} --query "[?starts_with(name, '${{ env.STORAGE_PREFIX }}')].id" -o tsv); do
+            az resource delete -v --id $resource --verbose
+          done
+      - name: Delete your custom stuff
+        if: ${{ env.CUSTOM_PREFIX }}
+        run: |
+          for resource in $(az resource list -g ${{ env.RESOURCE_GROUP }} --query "[?starts_with(name, '${{ env.CUSTOM_PREFIX }}')].id" -o tsv); do
+            az resource delete -v --id $resource --verbose
+          done
   mq-cleanup:
     runs-on: ubuntu-latest
     steps:

.github/workflows/int_test.yml

@@ -134,7 +134,7 @@ jobs:
       - name: "Setup python"
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: "Create k3s cluster"
         run: |
           # vars
@@ -164,8 +164,8 @@ jobs:
           path: ${{ env.EXTENSION_SOURCE_DIRECTORY }}
       - name: "Build and install local IoT Ops extension from source"
         run: |
-          pip install wheel==0.30.0
           cd ${{ env.EXTENSION_SOURCE_DIRECTORY }}
+          pip install -r dev_requirements.txt
           python -m setup bdist_wheel -d dist
           wheel=$(find ./dist/*.whl)
           az extension add --source $wheel -y

.github/workflows/release_build.yml

@@ -12,13 +12,13 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v5
         with:
-            python-version: "3.9"
+            python-version: "3.12"
 
       - uses: actions/checkout@v4
 
       - name: Build Wheel
         run: |
-          pip install wheel==0.30.0
+          pip install -r dev_requirements.txt
           python -m setup bdist_wheel -d dist
       - name: Determine Wheel Version
         run: |

.github/workflows/stage_release.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v5
         with:
-            python-version: "3.9"
+            python-version: "3.12"
       - name: Install and determine version
         run: |
           wheel=$(find ./release/*.whl)

.github/workflows/tox.yml

@@ -29,6 +29,7 @@ jobs:
           - windows-2022
           - macos-13
         py:
+          - "3.12"
           - "3.11"
           - "3.10"
           - "3.9"

azext_edge/constants.py

@@ -7,7 +7,7 @@
 
 import os
 
-VERSION = "0.7.0b1"
+VERSION = "0.7.0b2"
 EXTENSION_NAME = "azure-iot-ops"
 EXTENSION_ROOT = os.path.dirname(os.path.abspath(__file__))
 USER_AGENT = "IotOperationsCliExtension/{}".format(VERSION)

azext_edge/edge/_help.py

@@ -1205,6 +1205,9 @@ def load_iotops_help():
 
                       If the indicated storage account container does not exist it will be created with default
                       settings.
+
+                      This operation will also register the Microsoft.DeviceRegistry resource provider if it is
+                      not registered.
         examples:
         - name: Create a schema registry called 'myregistry' with minimum inputs.
           text: >

azext_edge/edge/providers/orchestration/resources/schema_registries.py

@@ -11,6 +11,7 @@
 from knack.log import get_logger
 from rich.console import Console
 
+
 from ....util.az_client import (
     get_registry_mgmt_client,
     get_storage_mgmt_client,
@@ -66,7 +67,11 @@ def create(
         custom_role_id: Optional[str] = None,
         **kwargs,
     ) -> dict:
+        from ..rp_namespace import register_providers, ADR_PROVIDER
         with console.status("Working...") as c:
+            # Register the schema (ADR) provider
+            register_providers(self.default_subscription_id, ADR_PROVIDER)
+
             if not location:
                 location = self.get_resource_group(name=resource_group_name)["location"]
 
@@ -127,7 +132,7 @@ def create(
             permission_manager = PermissionManager(storage_id_container.subscription_id)
             try:
                 permission_manager.apply_role_assignment(
-                    scope=storage_account["id"],
+                    scope=blob_container["id"],
                     principal_id=result["identity"]["principalId"],
                     role_def_id=target_role_def,
                 )
@@ -137,7 +142,7 @@ def create(
                     get_user_msg_warn_ra(
                         prefix=f"Role assignment failed with:\n{str(e)}.",
                         principal_id=result["identity"]["principalId"],
-                        scope=storage_account["id"],
+                        scope=blob_container["id"],
                     )
                 )
 

azext_edge/edge/providers/orchestration/rp_namespace.py

@@ -4,27 +4,30 @@
 # Licensed under the MIT License. See License file in the project root for license information.
 # ----------------------------------------------------------------------------------------------
 
+from typing import Optional
 from knack.log import get_logger
 
 from ...util.az_client import get_resource_client
 
 logger = get_logger(__name__)
 
 
+ADR_PROVIDER = "Microsoft.DeviceRegistry"
 RP_NAMESPACE_SET = frozenset(
     [
         "Microsoft.IoTOperations",
-        "Microsoft.DeviceRegistry",
         "Microsoft.SecretSyncController",
+        ADR_PROVIDER
     ]
 )
 
 
-def register_providers(subscription_id: str):
+def register_providers(subscription_id: str, resource_provider: Optional[str] = None):
     resource_client = get_resource_client(subscription_id=subscription_id)
     providers_list = resource_client.providers.list()
+    required_providers = [resource_provider] if resource_provider else RP_NAMESPACE_SET
     for provider in providers_list:
-        if "namespace" in provider and provider["namespace"] in RP_NAMESPACE_SET:
+        if "namespace" in provider and provider["namespace"] in required_providers:
             if provider["registrationState"] == "Registered":
                 logger.debug("RP %s is already registered.", provider["namespace"])
                 continue

azext_edge/edge/util/version_check.py

@@ -53,7 +53,7 @@ def check_latest(cli_ctx, force_refresh: Optional[bool] = False, throw_if_upgrad
         only_show_errors = getattr(cli_ctx, "only_show_errors", False)
         if not only_show_errors:
             console.print(
-                f":dim_button: [italic]{update_text.format('[yellow]','[/yellow]','[green]', '[/green]')}",
+                f":dim_button: [italic]{update_text.format('[yellow]', '[/yellow]', '[green]', '[/green]')}",
             )
 
 

azext_edge/tests/edge/conftest.py

@@ -34,6 +34,12 @@ def mocked_urlopen(mocker):
     yield patched
 
 
+@pytest.fixture
+def mocked_register_providers(mocker):
+    patched = mocker.patch("azext_edge.edge.providers.orchestration.rp_namespace.register_providers", autospec=True)
+    yield patched
+
+
 @pytest.fixture
 def mocked_resource_management_client(request, mocker):
     request_results = getattr(request, "param", {})
@@ -152,7 +158,7 @@ def tracked_resources():
         try:
             run(f"az resource delete --id {res} -v")
         except CLIInternalError:
-            logger.warning(f"failed to delete {res}")
+            logger.error(f"Failed to delete {res}")
 
 
 @pytest.fixture(scope="session")

azext_edge/tests/edge/init/conftest.py

@@ -48,12 +48,6 @@ def mocked_verify_cli_client_connections(mocker):
     yield patched
 
 
-@pytest.fixture
-def mocked_register_providers(mocker):
-    patched = mocker.patch("azext_edge.edge.providers.orchestration.rp_namespace.register_providers", autospec=True)
-    yield patched
-
-
 @pytest.fixture
 def mocked_edge_api_keyvault_api_v1(mocker):
     patched = mocker.patch("azext_edge.edge.providers.edge_api.keyvault.KEYVAULT_API_V1", autospec=False)

azext_edge/tests/edge/init/int/test_init_int.py

@@ -36,9 +36,9 @@ def init_test_setup(settings, tracked_resources):
     if not instance_name:
         instance_name = f"testcli{generate_random_string(force_lower=True, size=6)}"
     # set up registry
-    storage_account_name = f"teststore{generate_random_string(force_lower=True, size=6)}"
-    registry_name = f"test-registry-{generate_random_string(force_lower=True, size=6)}"
-    registry_namespace = f"test-namespace-{generate_random_string(force_lower=True, size=6)}"
+    storage_account_name = f"initstore{generate_random_string(force_lower=True, size=6)}"
+    registry_name = f"init-registry-{generate_random_string(force_lower=True, size=6)}"
+    registry_namespace = f"init-namespace-{generate_random_string(force_lower=True, size=6)}"
     storage_account = run(
         f"az storage account create -n {storage_account_name} -g {settings.env.azext_edge_rg} "
         "--enable-hierarchical-namespace --public-network-access Disabled "
@@ -48,7 +48,6 @@ def init_test_setup(settings, tracked_resources):
     registry = run(
         f"az iot ops schema registry create -n {registry_name} -g {settings.env.azext_edge_rg} "
         f"--rn {registry_namespace} --sa-resource-id {storage_account['id']} "
-        "--location eastus2euap"  # TODO: remove once avaliable in all regions
     )
     tracked_resources.append(registry["id"])
 

azext_edge/tests/edge/init/test_base_unit.py

@@ -324,7 +324,8 @@ def test_verify_custom_location_namespace(
         "NotRegistered",
     ],
 )
-def test_register_providers(mocker, registration_state):
+@pytest.mark.parametrize("input_rp", [None, "Microsoft.DeviceRegistry"])
+def test_register_providers(mocker, registration_state, input_rp):
     mocked_get_resource_client: Mock = mocker.patch(
         "azext_edge.edge.providers.orchestration.rp_namespace.get_resource_client"
     )
@@ -338,13 +339,15 @@ def test_register_providers(mocker, registration_state):
         "Microsoft.DeviceRegistry",
         "Microsoft.SecretSyncController",
     ]
+    if input_rp:
+        iot_ops_rps = [input_rp]
     mocked_get_resource_client().providers.list.return_value = [
         {"namespace": namespace, "registrationState": registration_state} for namespace in iot_ops_rps
     ]
 
     for rp in iot_ops_rps:
         assert rp in RP_NAMESPACE_SET
-    assert len(iot_ops_rps) == len(RP_NAMESPACE_SET)
+    assert len(iot_ops_rps) == (1 if input_rp else len(RP_NAMESPACE_SET))
 
     register_providers(ZEROED_SUB)
     mocked_get_resource_client().providers.list.assert_called_once()

azext_edge/tests/edge/orchestration/resources/test_schema_registry_int.py

@@ -23,6 +23,10 @@ def test_schema_registry_lifecycle(settings_with_rg, tracked_resources):
         "--enable-hierarchical-namespace --public-network-access Disabled "
         "--allow-shared-key-access false --allow-blob-public-access false --default-action Deny"
     )
+    blob_service = run(
+        f"az storage account blob-service-properties show -n {storage_account_name}"
+    )
+    container_resource_id = f"{blob_service['id']}/containers/schemas"
     tracked_resources.append(storage_account['id'])
 
     # CREATE 1
@@ -41,7 +45,7 @@ def test_schema_registry_lifecycle(settings_with_rg, tracked_resources):
     # check the roles
     roles = run(
         f"az role assignment list --assignee {registry['identity']['principalId']} "
-        f"--scope {storage_account['id']}"
+        f"--scope {container_resource_id}"
     )
     assert roles
     assert roles[0]["roleDefinitionName"] == "Storage Blob Data Contributor"
@@ -77,6 +81,7 @@ def test_schema_registry_lifecycle(settings_with_rg, tracked_resources):
         f"--sa-container {sa_container} --desc {description} --display-name {display_name} "
         f"--tags {tags_str} --custom-role-id {role_id} "
     )
+    alt_container_resource_id = f"{blob_service['id']}/containers/{sa_container}"
     tracked_resources.append(alt_registry["id"])
     assert_schema_registry(
         registry=alt_registry,
@@ -92,7 +97,7 @@ def test_schema_registry_lifecycle(settings_with_rg, tracked_resources):
     # check the roles
     roles = run(
         f"az role assignment list --assignee {alt_registry['identity']['principalId']} "
-        f"--scope {storage_account['id']}"
+        f"--scope {alt_container_resource_id}"
     )
     assert roles
     assert roles[0]["roleDefinitionName"] == role_name