wip: support launching sessions with data connectors

SwissDataScienceCenter · Oct 10, 2024 · dbd6281 · dbd6281
1 parent f9acac7
commit dbd6281
Show file tree

Hide file tree

Showing 14 changed files with 234 additions and 749 deletions.
diff --git a/.../features/ProjectPageV2/ProjectPageContent/DataSources/useDataSourceConfiguration.hook.ts b/.../features/ProjectPageV2/ProjectPageContent/DataSources/useDataSourceConfiguration.hook.ts
@@ -25,7 +25,7 @@ import type {
   CloudStorageSecretGet,
 } from "../../../projectsV2/api/storagesV2.api";
 
-import type { SessionStartCloudStorageConfiguration } from "../../../sessionsV2/startSessionOptionsV2.types";
+import type { SessionStartDataConnectorConfiguration } from "../../../sessionsV2/startSessionOptionsV2.types";
 
 interface UseDataSourceConfigurationArgs {
   storages: CloudStorageGetV2Read[] | undefined;
@@ -65,7 +65,7 @@ export default function useDataSourceConfiguration({
             };
           });
 
-        const sensitiveFieldValues: SessionStartCloudStorageConfiguration["sensitiveFieldValues"] =
+        const sensitiveFieldValues: SessionStartDataConnectorConfiguration["sensitiveFieldValues"] =
           {};
         configSensitiveFields.forEach((key) => {
           const { name } = defSensitiveFieldsMap[key];

diff --git a/client/src/features/dataConnectorsV2/api/data-connectors.enhanced-api.ts b/client/src/features/dataConnectorsV2/api/data-connectors.enhanced-api.ts
@@ -6,6 +6,8 @@ import { dataConnectorsApi as api } from "./data-connectors.api";
 import type {
   GetDataConnectorsApiArg,
   GetDataConnectorsApiResponse as GetDataConnectorsApiResponseOrig,
+  GetDataConnectorsByDataConnectorIdApiArg,
+  GetDataConnectorsByDataConnectorIdApiResponse,
   GetDataConnectorsByDataConnectorIdSecretsApiArg,
   GetDataConnectorsByDataConnectorIdSecretsApiResponse,
 } from "./data-connectors.api";
@@ -15,6 +17,15 @@ export interface GetDataConnectorsApiResponse
   dataConnectors: GetDataConnectorsApiResponseOrig;
 }
 
+interface GetDataConnectorsListByDataConnectorIdsApiArg {
+  dataConnectorIds: GetDataConnectorsByDataConnectorIdApiArg["dataConnectorId"][];
+}
+
+type GetDataConnectorsListByDataConnectorIdsApiResponse = Record<
+  string,
+  GetDataConnectorsByDataConnectorIdApiResponse
+>;
+
 interface GetDataConnectorListSecretsApiArg {
   dataConnectorIds: GetDataConnectorsByDataConnectorIdSecretsApiArg["dataConnectorId"][];
 }
@@ -58,6 +69,27 @@ const injectedApi = api.injectEndpoints({
         };
       },
     }),
+    getDataConnectorsListByDataConnectorIds: builder.query<
+      GetDataConnectorsListByDataConnectorIdsApiResponse,
+      GetDataConnectorsListByDataConnectorIdsApiArg
+    >({
+      async queryFn(queryArg, _api, _options, fetchWithBQ) {
+        const { dataConnectorIds } = queryArg;
+        const result: GetDataConnectorsListByDataConnectorIdsApiResponse = {};
+        const promises = dataConnectorIds.map((dataConnectorId) =>
+          fetchWithBQ(`/data_connectors/${dataConnectorId}`)
+        );
+        const responses = await Promise.all(promises);
+        for (let i = 0; i < dataConnectorIds.length; i++) {
+          const dataConnectorId = dataConnectorIds[i];
+          const response = responses[i];
+          if (response.error) return response;
+          result[dataConnectorId] =
+            response.data as GetDataConnectorsByDataConnectorIdApiResponse;
+        }
+        return { data: result };
+      },
+    }),
     getDataConnectorsListSecrets: builder.query<
       GetDataConnectorListSecretsApiResponse,
       GetDataConnectorListSecretsApiArg
@@ -104,6 +136,9 @@ const enhancedApi = injectedApi.enhanceEndpoints({
     getDataConnectorsByDataConnectorId: {
       providesTags: ["DataConnectors"],
     },
+    getDataConnectorsListByDataConnectorIds: {
+      providesTags: ["DataConnectors"],
+    },
     getDataConnectorsListSecrets: {
       providesTags: ["DataConnectorSecrets"],
     },
@@ -141,6 +176,7 @@ export const {
   useGetDataConnectorsByDataConnectorIdQuery,
   useGetDataConnectorsByDataConnectorIdProjectLinksQuery,
   useGetDataConnectorsByDataConnectorIdSecretsQuery,
+  useGetDataConnectorsListByDataConnectorIdsQuery,
   useGetDataConnectorsListSecretsQuery,
   useGetNamespacesByNamespaceDataConnectorsAndSlugQuery,
   usePatchDataConnectorsByDataConnectorIdMutation,

diff --git a/client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts b/client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts
@@ -23,10 +23,10 @@ import { RCloneOption } from "../api/data-connectors.api";
 import type { DataConnectorRead } from "../api/data-connectors.api";
 import { useGetDataConnectorsListSecretsQuery } from "../api/data-connectors.enhanced-api";
 
-import type { SessionStartCloudStorageConfiguration } from "../../sessionsV2/startSessionOptionsV2.types";
+import type { SessionStartDataConnectorConfiguration } from "../../sessionsV2/startSessionOptionsV2.types";
 
 export interface DataConnectorConfiguration
-  extends Omit<SessionStartCloudStorageConfiguration, "cloudStorage"> {
+  extends Omit<SessionStartDataConnectorConfiguration, "cloudStorage"> {
   dataConnector: DataConnectorRead;
 }
 
@@ -71,7 +71,7 @@ export default function useDataConnectorConfiguration({
             };
           });
 
-        const sensitiveFieldValues: SessionStartCloudStorageConfiguration["sensitiveFieldValues"] =
+        const sensitiveFieldValues: SessionStartDataConnectorConfiguration["sensitiveFieldValues"] =
           {};
         configSensitiveFields.forEach((key) => {
           const { name } = defSensitiveFieldsMap[key];

diff --git a/client/src/features/dataConnectorsV2/components/useDataConnectorProjects.hook.ts b/client/src/features/dataConnectorsV2/components/useDataConnectorProjects.hook.ts
@@ -23,10 +23,10 @@ import type { DataConnectorRead } from "../api/data-connectors.api";
 import { useGetDataConnectorsByDataConnectorIdProjectLinksQuery } from "../api/data-connectors.enhanced-api";
 import { useGetProjectsByProjectIdsQuery } from "../../projectsV2/api/projectV2.enhanced-api";
 
-import type { SessionStartCloudStorageConfiguration } from "../../sessionsV2/startSessionOptionsV2.types";
+import type { SessionStartDataConnectorConfiguration } from "../../sessionsV2/startSessionOptionsV2.types";
 
 export interface DataConnectorConfiguration
-  extends Omit<SessionStartCloudStorageConfiguration, "cloudStorage"> {
+  extends Omit<SessionStartDataConnectorConfiguration, "cloudStorage"> {
   dataConnector: DataConnectorRead;
 }
 

diff --git a/client/src/features/project/utils/projectCloudStorage.utils.test.ts b/client/src/features/project/utils/projectCloudStorage.utils.test.ts
@@ -4,46 +4,52 @@ describe("storageDefinitionFromConfig", () => {
   it("should return the correct storage definition", () => {
     const config = {
       active: true,
-      cloudStorage: {
+      dataConnector: {
+        id: "ULID-1",
+        etag: "foo",
+        name: "example storage",
+        namespace: "user1-uuid",
+        slug: "example-storage",
         storage: {
+          storage_type: "s3",
           configuration: {
             type: "s3",
             provider: "AWS",
             access_key_id: "<sensitive>",
             secret_access_key: "<sensitive>",
           },
-          name: "example-storage",
-          project_id: "1",
-          readonly: true,
           source_path: "bucket/my-source",
-          storage_id: "2",
-          storage_type: "s3",
           target_path: "external_storage/aws",
+          readonly: true,
+          sensitive_fields: [
+            {
+              name: "access_key_id",
+              help: "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+              provider: "",
+              default: "",
+              default_str: "",
+              required: false,
+              sensitive: true,
+              advanced: false,
+              exclusive: false,
+            },
+            {
+              name: "secret_access_key",
+              help: "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+              provider: "",
+              default: "",
+              default_str: "",
+              required: false,
+              sensitive: true,
+              advanced: false,
+              exclusive: false,
+            },
+          ],
         },
-        sensitive_fields: [
-          {
-            name: "access_key_id",
-            help: "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
-            provider: "",
-            default: "",
-            default_str: "",
-            required: false,
-            sensitive: true,
-            advanced: false,
-            exclusive: false,
-          },
-          {
-            name: "secret_access_key",
-            help: "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
-            provider: "",
-            default: "",
-            default_str: "",
-            required: false,
-            sensitive: true,
-            advanced: false,
-            exclusive: false,
-          },
-        ],
+        creation_date: "2023-11-15T09:55:59Z",
+        created_by: "user1-uuid",
+        visibility: "public" as const,
+        description: "Example storage description",
       },
       sensitiveFieldDefinitions: [
         {
@@ -66,7 +72,7 @@ describe("storageDefinitionFromConfig", () => {
       saveCredentials: false,
       savedCredentialFields: [],
     };
-    const result = storageDefinitionFromConfig(config);
+    const result = storageDefinitionFromConfig(config, "1");
     expect(result).toEqual({
       configuration: {
         type: "s3",
@@ -78,7 +84,7 @@ describe("storageDefinitionFromConfig", () => {
       project_id: "1",
       readonly: true,
       source_path: "bucket/my-source",
-      storage_id: "2",
+      storage_id: "ULID-1",
       storage_type: "s3",
       target_path: "external_storage/aws",
     });

diff --git a/client/src/features/project/utils/projectCloudStorage.utils.ts b/client/src/features/project/utils/projectCloudStorage.utils.ts
@@ -41,7 +41,7 @@ import {
   CloudStorageSchemaOptions,
 } from "../components/cloudStorage/projectCloudStorage.types";
 
-import { SessionStartCloudStorageConfiguration } from "../../sessionsV2/startSessionOptionsV2.types";
+import { SessionStartDataConnectorConfiguration } from "../../sessionsV2/startSessionOptionsV2.types";
 
 const LAST_POSITION = 1000;
 
@@ -429,27 +429,35 @@ export function findSensitive(
 }
 
 export function storageDefinitionAfterSavingCredentialsFromConfig(
-  cs: SessionStartCloudStorageConfiguration
+  cs: SessionStartDataConnectorConfiguration
 ) {
   const newCs = { ...cs, saveCredentials: false };
-  const newStorage = { ...newCs.cloudStorage.storage };
+  const newStorage = { ...newCs.dataConnector.storage };
   // The following two lines remove the sensitive fields from the storage configuration,
   // which should be ok, but isn't; so keep in the sensitive fields.
   // newCs.sensitiveFieldValues = {};
   // newStorage.configuration = {};
-  const newCloudStorage = {
-    ...newCs.cloudStorage,
+  const newDataConnector = {
+    ...newCs.dataConnector,
     storage: newStorage,
   };
-  newCs.cloudStorage = newCloudStorage;
+  newCs.dataConnector = newDataConnector;
   return newCs;
 }
 
 export function storageDefinitionFromConfig(
-  config: SessionStartCloudStorageConfiguration
+  config: SessionStartDataConnectorConfiguration,
+  projectId: string
 ): CloudStorageWithIdRead {
-  const storageDefinition = config.cloudStorage.storage;
-  const newStorageDefinition = { ...storageDefinition };
+  const storageDefinition = config.dataConnector.storage;
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const { sensitive_fields, ...s } = config.dataConnector.storage;
+  const newStorageDefinition = {
+    ...s,
+    name: config.dataConnector.slug,
+    project_id: projectId,
+    storage_id: config.dataConnector.id,
+  };
   newStorageDefinition.configuration = { ...storageDefinition.configuration };
   const sensitiveFieldValues = config.sensitiveFieldValues;
   Object.entries(sensitiveFieldValues).forEach(([name, value]) => {