Terraform Organization Support (#315)

integrations/terraform-cloud/.port/resources/blueprints.json

@@ -1,4 +1,59 @@
 [
+  {
+    "identifier": "terraformCloudOrganization",
+    "description": "This blueprint represents an organization in Terraform Cloud",
+    "title": "Terraform Cloud Organization",
+    "icon": "Terraform",
+    "schema": {
+      "properties": {
+        "externalId": {
+          "type": "string",
+          "title": "External ID",
+          "description": "The external ID of the organization"
+        },
+        "ownerEmail": {
+          "type": "string",
+          "title": "Owner Email",
+          "description": "The email associated with the organization"
+        },
+        "collaboratorAuthPolicy": {
+          "type": "string",
+          "title": "Collaborator Authentication Policy",
+          "description": "Policy for collaborator authentication"
+        },
+        "planExpired": {
+          "type": "string",
+          "title": "Plan Expired",
+          "description": "Indicates if plan is expired"
+        },
+        "planExpiresAt": {
+          "type": "string",
+          "format": "date-time",
+          "title": "Plan Expiry Date",
+          "description": "The data and time which the plan expires"
+        },
+        "permissions": {
+          "type": "object",
+          "title": "Permissions",
+          "description": "Permissions associated with the organization"
+        },
+        "samlEnabled": {
+          "type": "boolean",
+          "title": "SAML Enabled",
+          "description": "Indicates if SAML is enabled for the organization"
+        },
+        "defaultExecutionMode": {
+          "type": "string",
+          "title": "Default Execution Mode",
+          "description": "The default execution mode for the organization"
+        }
+      }
+    },
+    "mirrorProperties": {},
+    "calculationProperties": {},
+    "aggregationProperties": {},
+    "relations": {}
+  },
   {
     "identifier": "terraformCloudProject",
     "description": "This blueprint represents a project in Terraform Cloud",
@@ -15,18 +70,20 @@
           "type": "object",
           "title": "Permissions",
           "description": "The permisssions on the project"
-        },
-        "organizationId": {
-          "type": "string",
-          "title": "Organization ID",
-          "description": "The ID of the organization the project belongs to"
         }
       }
     },
     "mirrorProperties": {},
     "calculationProperties": {},
     "aggregationProperties": {},
-    "relations": {}
+    "relations": {
+      "organization": {
+        "title": "Terraform Cloud Organization",
+        "target": "terraformCloudOrganization",
+        "required": true,
+        "many": false
+      }
+    }
   },
   {
     "identifier": "terraformCloudWorkspace",
@@ -77,6 +134,11 @@
           "format": "date-time",
           "title": "Latest Change",
           "description": "Timestamp of the latest change in the workspace"
+        },
+        "tags": {
+          "type": "array",
+          "title": "Workspace Tags",
+          "description": "Terraform workspace tags"
         }
       }
     },

integrations/terraform-cloud/.port/resources/port-app-config.yaml

@@ -1,6 +1,24 @@
 createMissingRelatedEntities: true
 deleteDependentEntities: true
 resources:
+  - kind: organization
+    selector:
+      query: "true"
+    port:
+      entity:
+        mappings:
+          identifier: .id
+          title: .attributes.name
+          blueprint: '"terraformCloudOrganization"'
+          properties:
+            externalId: .attributes."external-id"
+            ownerEmail: .attributes.email
+            collaboratorAuthPolicy: .attributes."collaborator-auth-policy"
+            planExpired: .attributes."plan-expired"
+            planExpiresAt: .attributes."plan-expires-at"
+            permissions: .attributes.permissions
+            samlEnabled: .attributes."saml-enabled"
+            defaultExecutionMode: .attributes."default-execution-mode"
   - kind: project
     selector:
       query: "true"
@@ -13,7 +31,8 @@ resources:
           properties:
             name: .attributes.name
             permissions: .attributes.permissions
-            organizationId: .relationships.organization.data.id
+          relations:
+            organization: .relationships.organization.data.id
   - kind: workspace
     selector:
       query: "true"
@@ -32,6 +51,7 @@ resources:
             executionMode: .attributes."execution-mode"
             resourceCount: .attributes."resource-count"
             latestChangeAt: .attributes."latest-change-at"
+            tags: .__tags
           relations:
             currentStateVersion: .relationships."current-state-version".data.id
             project: .relationships.project.data.id

integrations/terraform-cloud/.port/spec.yaml

@@ -6,10 +6,11 @@ features:
   - type: exporter
     section: IaC
     resources:
+      - kind: organization
+      - kind: project
       - kind: workspace
-      - kind: run
       - kind: state-version
-      - kind: project
+      - kind: run
 configurations:
   - name: terraformCloudHost
     required: false

integrations/terraform-cloud/CHANGELOG.md

@@ -6,6 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 <!-- towncrier release notes start -->
+
+# Port_Ocean 0.1.7 (2024-01-11)
+
+### Features
+
+- Added support for Terraform Organization (PORT-5917)
+
+### Improvements
+
+- Added Tags to terraform cloud (PORT-6043)
+
 
 # Port_Ocean 0.1.6 (2024-01-11)
 

integrations/terraform-cloud/client.py

@@ -2,6 +2,7 @@
 from port_ocean.utils import http_async_client
 import httpx
 from loguru import logger
+from enum import StrEnum
 
 from port_ocean.context.event import event
 
@@ -14,6 +15,11 @@
     "run:planning",
 ]
 
+
+class CacheKeys(StrEnum):
+    ORGANIZATIONS = "ORGANIZATIONS"
+
+
 PAGE_SIZE = 100
 
 
@@ -97,10 +103,22 @@ async def get_paginated_resources(
     async def get_paginated_organizations(
         self,
     ) -> AsyncGenerator[list[dict[str, Any]], None]:
+        if cache := event.attributes.get(CacheKeys.ORGANIZATIONS):
+            logger.info("Retrieving organizations data from cache")
+            yield cache
+            return
+
+        all_organizations = []
         logger.info("Fetching organizations")
         async for organizations in self.get_paginated_resources("organizations"):
+            all_organizations.extend(organizations)
             yield organizations
 
+        event.attributes[CacheKeys.ORGANIZATIONS] = all_organizations
+        logger.info(
+            f"Total workspaces retrieved across all organizations: {len(all_organizations)}"
+        )
+
     async def get_single_workspace(self, workspace_id: str) -> dict[str, Any]:
         logger.info(f"Fetching workspace with ID: {workspace_id}")
         workspace = await self.send_api_request(endpoint=f"workspaces/{workspace_id}")
@@ -111,6 +129,14 @@ async def get_single_run(self, run_id: str) -> dict[str, Any]:
         run = await self.send_api_request(endpoint=f"runs/{run_id}")
         return run.get("data", {})
 
+    async def get_workspace_tags(
+        self, workspace_id: str
+    ) -> AsyncGenerator[list[dict[str, Any]], None]:
+        logger.info(f"Fetching tags for workspace ID: {workspace_id}")
+        endpoint = f"/workspaces/{workspace_id}/relationships/tags"
+        async for tags in self.get_paginated_resources(endpoint):
+            yield tags
+
     async def get_state_version_output(self, state_version_id: str) -> dict[str, Any]:
         logger.info(f"Fetching state version output for ID: {state_version_id}")
         outputs = await self.send_api_request(
@@ -144,7 +170,7 @@ async def get_paginated_workspaces(
                     all_workspaces.extend(workspaces)
                     yield workspaces
 
-        event.attributes[cache_key] = workspaces
+        event.attributes[cache_key] = all_workspaces
         logger.info(
             f"Total workspaces retrieved across all organizations: {len(all_workspaces)}"
         )

integrations/terraform-cloud/main.py

@@ -15,6 +15,10 @@ class ObjectKind(StrEnum):
     RUN = "run"
     STATE_VERSION = "state-version"
     PROJECT = "project"
+    ORGANIZATION = "organization"
+
+
+SEMAPHORE_LIMIT = 30
 
 
 def init_terraform_client() -> TerraformClient:
@@ -35,7 +39,7 @@ def init_terraform_client() -> TerraformClient:
 async def enrich_state_versions_with_output_data(
     http_client: TerraformClient, state_versions: List[dict[str, Any]]
 ) -> list[dict[str, Any]]:
-    async with asyncio.BoundedSemaphore(30):
+    async with asyncio.BoundedSemaphore(SEMAPHORE_LIMIT):
         tasks = [
             http_client.get_state_version_output(state_version["id"])
             for state_version in state_versions
@@ -54,6 +58,51 @@ async def enrich_state_versions_with_output_data(
         return enriched_state_versions
 
 
+async def enrich_workspaces_with_tags(
+    http_client: TerraformClient, workspaces: List[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    async def get_tags_for_workspace(workspace: dict[str, Any]) -> dict[str, Any]:
+        async with asyncio.BoundedSemaphore(SEMAPHORE_LIMIT):
+            try:
+                tags = []
+                async for tag_batch in http_client.get_workspace_tags(workspace["id"]):
+                    tags.extend(tag_batch)
+                return {**workspace, "__tags": tags}
+            except Exception as e:
+                logger.warning(
+                    f"Failed to fetch tags for workspace {workspace['id']}: {e}"
+                )
+                return {**workspace, "__tags": []}
+
+    tasks = [get_tags_for_workspace(workspace) for workspace in workspaces]
+    enriched_workspaces = [await task for task in asyncio.as_completed(tasks)]
+
+    return enriched_workspaces
+
+
+async def enrich_workspace_with_tags(
+    http_client: TerraformClient, workspace: dict[str, Any]
+) -> dict[str, Any]:
+    async with asyncio.BoundedSemaphore(30):
+        try:
+            tags = []
+            async for tag_batch in http_client.get_workspace_tags(workspace["id"]):
+                tags.extend(tag_batch)
+            return {**workspace, "__tags": tags}
+        except Exception as e:
+            logger.warning(f"Failed to fetch tags for workspace {workspace['id']}: {e}")
+            return {**workspace, "__tags": []}
+
+
+@ocean.on_resync(ObjectKind.ORGANIZATION)
+async def resync_organizations(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
+    terraform_client = init_terraform_client()
+    async for organizations in terraform_client.get_paginated_organizations():
+        logger.info(f"Received {len(organizations)} batch {kind}s")
+        print(organizations)
+        yield organizations
+
+
 @ocean.on_resync(ObjectKind.PROJECT)
 async def resync_projects(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
     terraform_client = init_terraform_client()
@@ -67,7 +116,10 @@ async def resync_workspaces(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
     terraform_client = init_terraform_client()
     async for workspaces in terraform_client.get_paginated_workspaces():
         logger.info(f"Received {len(workspaces)} batch {kind}s")
-        yield workspaces
+        enriched_workspace_batch = await enrich_workspaces_with_tags(
+            terraform_client, workspaces
+        )
+        yield enriched_workspace_batch
 
 
 @ocean.on_resync(ObjectKind.RUN)
@@ -110,7 +162,6 @@ async def resync_state_versions(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
         enriched_state_versions_batch = await enrich_state_versions_with_output_data(
             terraform_client, state_versions_batch
         )
-
         yield enriched_state_versions_batch
 
 
@@ -129,9 +180,11 @@ async def handle_webhook_request(data: dict[str, Any]) -> dict[str, Any]:
         terraform_client.get_single_workspace(workspace_id),
     )
 
+    enriched_workspace = await enrich_workspace_with_tags(terraform_client, workspace)
+
     await gather(
         ocean.register_raw(ObjectKind.RUN, [run]),
-        ocean.register_raw(ObjectKind.WORKSPACE, [workspace]),
+        ocean.register_raw(ObjectKind.WORKSPACE, [enriched_workspace]),
     )
 
     logger.info("Terraform webhook event processed")

integrations/terraform-cloud/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "terraform-cloud"
-version = "0.1.6"
+version = "0.1.7"
 description = "Terraform Cloud Integration for Port"
 authors = ["Michael Armah <michaelarmah@getport.io>"]