From a8bfb2467ab3e27874bb969310b32aea311dfe65 Mon Sep 17 00:00:00 2001 From: Michael Kofi Armah Date: Thu, 11 Jan 2024 17:27:44 +0000 Subject: [PATCH] Terraform Organization Support (#315) --- .../.port/resources/blueprints.json | 74 +++++++++++++++++-- .../.port/resources/port-app-config.yaml | 22 +++++- integrations/terraform-cloud/.port/spec.yaml | 5 +- integrations/terraform-cloud/CHANGELOG.md | 11 +++ integrations/terraform-cloud/client.py | 28 ++++++- integrations/terraform-cloud/main.py | 61 ++++++++++++++- integrations/terraform-cloud/pyproject.toml | 2 +- scripts/bump-all.sh | 0 8 files changed, 188 insertions(+), 15 deletions(-) mode change 100755 => 100644 scripts/bump-all.sh diff --git a/integrations/terraform-cloud/.port/resources/blueprints.json b/integrations/terraform-cloud/.port/resources/blueprints.json index 101bc8ddd..7d87e3b1f 100644 --- a/integrations/terraform-cloud/.port/resources/blueprints.json +++ b/integrations/terraform-cloud/.port/resources/blueprints.json @@ -1,4 +1,59 @@ [ + { + "identifier": "terraformCloudOrganization", + "description": "This blueprint represents an organization in Terraform Cloud", + "title": "Terraform Cloud Organization", + "icon": "Terraform", + "schema": { + "properties": { + "externalId": { + "type": "string", + "title": "External ID", + "description": "The external ID of the organization" + }, + "ownerEmail": { + "type": "string", + "title": "Owner Email", + "description": "The email associated with the organization" + }, + "collaboratorAuthPolicy": { + "type": "string", + "title": "Collaborator Authentication Policy", + "description": "Policy for collaborator authentication" + }, + "planExpired": { + "type": "string", + "title": "Plan Expired", + "description": "Indicates if plan is expired" + }, + "planExpiresAt": { + "type": "string", + "format": "date-time", + "title": "Plan Expiry Date", + "description": "The data and time which the plan expires" + }, + "permissions": { + "type": "object", + "title": "Permissions", + "description": "Permissions associated with the organization" + }, + "samlEnabled": { + "type": "boolean", + "title": "SAML Enabled", + "description": "Indicates if SAML is enabled for the organization" + }, + "defaultExecutionMode": { + "type": "string", + "title": "Default Execution Mode", + "description": "The default execution mode for the organization" + } + } + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "aggregationProperties": {}, + "relations": {} + }, { "identifier": "terraformCloudProject", "description": "This blueprint represents a project in Terraform Cloud", @@ -15,18 +70,20 @@ "type": "object", "title": "Permissions", "description": "The permisssions on the project" - }, - "organizationId": { - "type": "string", - "title": "Organization ID", - "description": "The ID of the organization the project belongs to" } } }, "mirrorProperties": {}, "calculationProperties": {}, "aggregationProperties": {}, - "relations": {} + "relations": { + "organization": { + "title": "Terraform Cloud Organization", + "target": "terraformCloudOrganization", + "required": true, + "many": false + } + } }, { "identifier": "terraformCloudWorkspace", @@ -77,6 +134,11 @@ "format": "date-time", "title": "Latest Change", "description": "Timestamp of the latest change in the workspace" + }, + "tags": { + "type": "array", + "title": "Workspace Tags", + "description": "Terraform workspace tags" } } }, diff --git a/integrations/terraform-cloud/.port/resources/port-app-config.yaml b/integrations/terraform-cloud/.port/resources/port-app-config.yaml index 1bdc4a4e3..9c7a90d97 100644 --- a/integrations/terraform-cloud/.port/resources/port-app-config.yaml +++ b/integrations/terraform-cloud/.port/resources/port-app-config.yaml @@ -1,6 +1,24 @@ createMissingRelatedEntities: true deleteDependentEntities: true resources: + - kind: organization + selector: + query: "true" + port: + entity: + mappings: + identifier: .id + title: .attributes.name + blueprint: '"terraformCloudOrganization"' + properties: + externalId: .attributes."external-id" + ownerEmail: .attributes.email + collaboratorAuthPolicy: .attributes."collaborator-auth-policy" + planExpired: .attributes."plan-expired" + planExpiresAt: .attributes."plan-expires-at" + permissions: .attributes.permissions + samlEnabled: .attributes."saml-enabled" + defaultExecutionMode: .attributes."default-execution-mode" - kind: project selector: query: "true" @@ -13,7 +31,8 @@ resources: properties: name: .attributes.name permissions: .attributes.permissions - organizationId: .relationships.organization.data.id + relations: + organization: .relationships.organization.data.id - kind: workspace selector: query: "true" @@ -32,6 +51,7 @@ resources: executionMode: .attributes."execution-mode" resourceCount: .attributes."resource-count" latestChangeAt: .attributes."latest-change-at" + tags: .__tags relations: currentStateVersion: .relationships."current-state-version".data.id project: .relationships.project.data.id diff --git a/integrations/terraform-cloud/.port/spec.yaml b/integrations/terraform-cloud/.port/spec.yaml index 9a4732625..03555c66a 100644 --- a/integrations/terraform-cloud/.port/spec.yaml +++ b/integrations/terraform-cloud/.port/spec.yaml @@ -6,10 +6,11 @@ features: - type: exporter section: IaC resources: + - kind: organization + - kind: project - kind: workspace - - kind: run - kind: state-version - - kind: project + - kind: run configurations: - name: terraformCloudHost required: false diff --git a/integrations/terraform-cloud/CHANGELOG.md b/integrations/terraform-cloud/CHANGELOG.md index 0b9ede8c9..d7935e93f 100644 --- a/integrations/terraform-cloud/CHANGELOG.md +++ b/integrations/terraform-cloud/CHANGELOG.md @@ -6,6 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +# Port_Ocean 0.1.7 (2024-01-11) + +### Features + +- Added support for Terraform Organization (PORT-5917) + +### Improvements + +- Added Tags to terraform cloud (PORT-6043) + # Port_Ocean 0.1.6 (2024-01-11) diff --git a/integrations/terraform-cloud/client.py b/integrations/terraform-cloud/client.py index 9ae44145c..7c50a4189 100644 --- a/integrations/terraform-cloud/client.py +++ b/integrations/terraform-cloud/client.py @@ -2,6 +2,7 @@ from port_ocean.utils import http_async_client import httpx from loguru import logger +from enum import StrEnum from port_ocean.context.event import event @@ -14,6 +15,11 @@ "run:planning", ] + +class CacheKeys(StrEnum): + ORGANIZATIONS = "ORGANIZATIONS" + + PAGE_SIZE = 100 @@ -97,10 +103,22 @@ async def get_paginated_resources( async def get_paginated_organizations( self, ) -> AsyncGenerator[list[dict[str, Any]], None]: + if cache := event.attributes.get(CacheKeys.ORGANIZATIONS): + logger.info("Retrieving organizations data from cache") + yield cache + return + + all_organizations = [] logger.info("Fetching organizations") async for organizations in self.get_paginated_resources("organizations"): + all_organizations.extend(organizations) yield organizations + event.attributes[CacheKeys.ORGANIZATIONS] = all_organizations + logger.info( + f"Total workspaces retrieved across all organizations: {len(all_organizations)}" + ) + async def get_single_workspace(self, workspace_id: str) -> dict[str, Any]: logger.info(f"Fetching workspace with ID: {workspace_id}") workspace = await self.send_api_request(endpoint=f"workspaces/{workspace_id}") @@ -111,6 +129,14 @@ async def get_single_run(self, run_id: str) -> dict[str, Any]: run = await self.send_api_request(endpoint=f"runs/{run_id}") return run.get("data", {}) + async def get_workspace_tags( + self, workspace_id: str + ) -> AsyncGenerator[list[dict[str, Any]], None]: + logger.info(f"Fetching tags for workspace ID: {workspace_id}") + endpoint = f"/workspaces/{workspace_id}/relationships/tags" + async for tags in self.get_paginated_resources(endpoint): + yield tags + async def get_state_version_output(self, state_version_id: str) -> dict[str, Any]: logger.info(f"Fetching state version output for ID: {state_version_id}") outputs = await self.send_api_request( @@ -144,7 +170,7 @@ async def get_paginated_workspaces( all_workspaces.extend(workspaces) yield workspaces - event.attributes[cache_key] = workspaces + event.attributes[cache_key] = all_workspaces logger.info( f"Total workspaces retrieved across all organizations: {len(all_workspaces)}" ) diff --git a/integrations/terraform-cloud/main.py b/integrations/terraform-cloud/main.py index 0872ef7a2..c478178c0 100644 --- a/integrations/terraform-cloud/main.py +++ b/integrations/terraform-cloud/main.py @@ -15,6 +15,10 @@ class ObjectKind(StrEnum): RUN = "run" STATE_VERSION = "state-version" PROJECT = "project" + ORGANIZATION = "organization" + + +SEMAPHORE_LIMIT = 30 def init_terraform_client() -> TerraformClient: @@ -35,7 +39,7 @@ def init_terraform_client() -> TerraformClient: async def enrich_state_versions_with_output_data( http_client: TerraformClient, state_versions: List[dict[str, Any]] ) -> list[dict[str, Any]]: - async with asyncio.BoundedSemaphore(30): + async with asyncio.BoundedSemaphore(SEMAPHORE_LIMIT): tasks = [ http_client.get_state_version_output(state_version["id"]) for state_version in state_versions @@ -54,6 +58,51 @@ async def enrich_state_versions_with_output_data( return enriched_state_versions +async def enrich_workspaces_with_tags( + http_client: TerraformClient, workspaces: List[dict[str, Any]] +) -> list[dict[str, Any]]: + async def get_tags_for_workspace(workspace: dict[str, Any]) -> dict[str, Any]: + async with asyncio.BoundedSemaphore(SEMAPHORE_LIMIT): + try: + tags = [] + async for tag_batch in http_client.get_workspace_tags(workspace["id"]): + tags.extend(tag_batch) + return {**workspace, "__tags": tags} + except Exception as e: + logger.warning( + f"Failed to fetch tags for workspace {workspace['id']}: {e}" + ) + return {**workspace, "__tags": []} + + tasks = [get_tags_for_workspace(workspace) for workspace in workspaces] + enriched_workspaces = [await task for task in asyncio.as_completed(tasks)] + + return enriched_workspaces + + +async def enrich_workspace_with_tags( + http_client: TerraformClient, workspace: dict[str, Any] +) -> dict[str, Any]: + async with asyncio.BoundedSemaphore(30): + try: + tags = [] + async for tag_batch in http_client.get_workspace_tags(workspace["id"]): + tags.extend(tag_batch) + return {**workspace, "__tags": tags} + except Exception as e: + logger.warning(f"Failed to fetch tags for workspace {workspace['id']}: {e}") + return {**workspace, "__tags": []} + + +@ocean.on_resync(ObjectKind.ORGANIZATION) +async def resync_organizations(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + terraform_client = init_terraform_client() + async for organizations in terraform_client.get_paginated_organizations(): + logger.info(f"Received {len(organizations)} batch {kind}s") + print(organizations) + yield organizations + + @ocean.on_resync(ObjectKind.PROJECT) async def resync_projects(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: terraform_client = init_terraform_client() @@ -67,7 +116,10 @@ async def resync_workspaces(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: terraform_client = init_terraform_client() async for workspaces in terraform_client.get_paginated_workspaces(): logger.info(f"Received {len(workspaces)} batch {kind}s") - yield workspaces + enriched_workspace_batch = await enrich_workspaces_with_tags( + terraform_client, workspaces + ) + yield enriched_workspace_batch @ocean.on_resync(ObjectKind.RUN) @@ -110,7 +162,6 @@ async def resync_state_versions(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: enriched_state_versions_batch = await enrich_state_versions_with_output_data( terraform_client, state_versions_batch ) - yield enriched_state_versions_batch @@ -129,9 +180,11 @@ async def handle_webhook_request(data: dict[str, Any]) -> dict[str, Any]: terraform_client.get_single_workspace(workspace_id), ) + enriched_workspace = await enrich_workspace_with_tags(terraform_client, workspace) + await gather( ocean.register_raw(ObjectKind.RUN, [run]), - ocean.register_raw(ObjectKind.WORKSPACE, [workspace]), + ocean.register_raw(ObjectKind.WORKSPACE, [enriched_workspace]), ) logger.info("Terraform webhook event processed") diff --git a/integrations/terraform-cloud/pyproject.toml b/integrations/terraform-cloud/pyproject.toml index 286a6902b..6f99e6c61 100644 --- a/integrations/terraform-cloud/pyproject.toml +++ b/integrations/terraform-cloud/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "terraform-cloud" -version = "0.1.6" +version = "0.1.7" description = "Terraform Cloud Integration for Port" authors = ["Michael Armah "] diff --git a/scripts/bump-all.sh b/scripts/bump-all.sh old mode 100755 new mode 100644