feature/#392_Unpack_gz_files_and_#391follow_redirects

libs/extensions/std/exec/src/archive-interpreter-executor.ts

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import { strict as assert } from 'assert';
+import * as zlib from 'node:zlib';
 import * as path from 'path';
 
 import * as R from '@jvalue/jayvee-execution';
@@ -20,7 +21,6 @@ import {
 } from '@jvalue/jayvee-execution';
 import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server';
 import * as JSZip from 'jszip';
-import * as zlib from 'node:zlib';
 
 import {
   inferFileExtensionFromFileExtensionString,
@@ -46,76 +46,43 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor<
       'archiveType',
       PrimitiveValuetypes.Text,
     );
+    let fs: R.Result<R.FileSystem>;
+
     if (archiveType === 'zip') {
-      const fs = await this.loadZipFileToInMemoryFileSystem(
-        archiveFile,
-        context,
-      );
-      if (R.isErr(fs)) {
-        return fs;
-      }
-      return R.ok(fs.right);
-    }
-    if (archiveType === 'gz') {
-      const fs = await this.loadGzFileToInMemoryFileSystem(
-        archiveFile,
-        context,
-      );
-      if (R.isErr(fs)) {
-        return fs;
-      }
-      return R.ok(fs.right);
+      fs = await this.loadZipFileToInMemoryFileSystem(archiveFile, context);
+    } else if (archiveType === 'gz') {
+      fs = this.loadGzFileToInMemoryFileSystem(archiveFile, context);
+    } else {
+      return R.err({
+        message: `Archive type is not supported`,
+        diagnostic: { node: context.getCurrentNode(), property: 'name' },
+      });
     }
 
-    return R.err({
-      message: `Archive is not a zip-archive`,
-      diagnostic: { node: context.getCurrentNode(), property: 'name' },
-    });
+    if (R.isErr(fs)) {
+      return fs;
+    }
+    return R.ok(fs.right);
   }
 
-  private async loadGzFileToInMemoryFileSystem(
+  private loadGzFileToInMemoryFileSystem(
     archiveFile: BinaryFile,
     context: ExecutionContext,
-  ): Promise<R.Result<FileSystem>> {
+  ): R.Result<FileSystem> {
     context.logger.logDebug(`Loading gz file from binary content`);
     try {
-
       const fs = new InMemoryFileSystem();
       const archivedObject = zlib.gunzipSync(archiveFile.content);
-      
+
       const extNameArchive = path.extname(archiveFile.name);
-      const fileName = path.basename(archiveFile.name, extNameArchive);
-      const extName = path.extname(fileName);
-
-      const mimeType =
-        inferMimeTypeFromContentTypeString(extName) ||
-        MimeType.APPLICATION_OCTET_STREAM;
-      const fileExtension =
-        inferFileExtensionFromFileExtensionString(extName) ||
-        FileExtension.NONE;
-      const file = new BinaryFile(
-        fileName,
-        fileExtension,
-        mimeType,
-        archivedObject,
-      );
-
-      const addedFile = fs.putFile(
-        InMemoryFileSystem.getPathSeparator() + fileName,
-        file,
-      );
-
-      assert(addedFile != null);
 
-      return R.ok(fs);
-    }
-    catch (error: unknown) {
-      return R.err({
-        message: `Unexpected Error ${
-          error instanceof Error ? error.message : JSON.stringify(err)
-        } occured during processing`,
-        diagnostic: { node: context.getCurrentNode(), property: 'name' },
+      this.createBinaryAndPutFile(fs, archiveFile.name, archivedObject, {
+        extNameArchive: extNameArchive,
       });
+
+      return R.ok(fs);
+    } catch (error: unknown) {
+      return R.err(this.generateErrorObject(context, error));
     }
   }
 
@@ -133,36 +100,47 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor<
       )) {
         if (!archivedObject.dir) {
           const content = await archivedObject.async('arraybuffer');
-          // Ext incl. leading dot
-          const extName = path.extname(archivedObject.name);
-          const fileName = path.basename(archivedObject.name);
-          const mimeType =
-            inferMimeTypeFromContentTypeString(extName) ||
-            MimeType.APPLICATION_OCTET_STREAM;
-          const fileExtension =
-            inferFileExtensionFromFileExtensionString(extName) ||
-            FileExtension.NONE;
-          const file = new BinaryFile(
-            fileName,
-            fileExtension,
-            mimeType,
-            content,
-          );
-          const addedFile = fs.putFile(
-            InMemoryFileSystem.getPathSeparator() + relPath,
-            file,
-          );
-          assert(addedFile != null);
+          this.createBinaryAndPutFile(fs, archivedObject.name, content, {
+            relPath: relPath,
+          });
         }
       }
       return R.ok(fs);
     } catch (error: unknown) {
-      return R.err({
-        message: `Unexpected Error ${
-          error instanceof Error ? error.message : JSON.stringify(err)
-        } occured during processing`,
-        diagnostic: { node: context.getCurrentNode(), property: 'name' },
-      });
+      return R.err(this.generateErrorObject(context, error));
     }
   }
+
+  private createBinaryAndPutFile(
+    fs: InMemoryFileSystem,
+    archiveFileName: string,
+    content: ArrayBuffer,
+    options?: { extNameArchive?: string; relPath?: string },
+  ) {
+    const fileName = path.basename(archiveFileName, options?.extNameArchive);
+    const extName = path.extname(fileName);
+
+    const mimeType =
+      inferMimeTypeFromContentTypeString(extName) ||
+      MimeType.APPLICATION_OCTET_STREAM;
+    const fileExtension =
+      inferFileExtensionFromFileExtensionString(extName) || FileExtension.NONE;
+    const file = new BinaryFile(fileName, fileExtension, mimeType, content);
+
+    const addedFile = fs.putFile(
+      InMemoryFileSystem.getPathSeparator() +
+        String(options?.relPath ?? fileName),
+      file,
+    );
+    assert(addedFile != null);
+  }
+
+  private generateErrorObject(context: ExecutionContext, error: unknown) {
+    return {
+      message: `Unexpected Error ${
+        error instanceof Error ? error.message : JSON.stringify(err)
+      } occured during processing`,
+      diagnostic: { node: context.getCurrentNode(), property: 'name' },
+    };
+  }
 }

libs/extensions/std/exec/src/http-extractor-executor.ts

@@ -4,7 +4,6 @@
 
 import { strict as assert } from 'assert';
 import * as path from 'path';
-import { http, https } from 'follow-redirects';
 
 import * as R from '@jvalue/jayvee-execution';
 import {
@@ -19,6 +18,7 @@ import {
   implementsStatic,
 } from '@jvalue/jayvee-execution';
 import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server';
+import { http, https } from 'follow-redirects';
 import { AstNode } from 'langium';
 
 import {
@@ -105,11 +105,14 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor<
     context.logger.logDebug(`Fetching raw data from ${url}`);
     let httpGetFunction: HttpGetFunction;
     if (url.startsWith('https')) {
-      httpGetFunction = https.get;
+      httpGetFunction = https.get.bind(https);
     } else {
-      httpGetFunction = http.get;
+      httpGetFunction = http.get.bind(http);
     }
-    const followRedirects = context.getPropertyValue('followRedirects', PrimitiveValuetypes.Boolean);
+    const followRedirects = context.getPropertyValue(
+      'followRedirects',
+      PrimitiveValuetypes.Boolean,
+    );
     return new Promise((resolve) => {
       httpGetFunction(url, { followRedirects: followRedirects }, (response) => {
         const responseCode = response.statusCode;
@@ -124,14 +127,10 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor<
               diagnostic: { node: context.getOrFailProperty('url') },
             }),
           );
-        }
-
-        if (responseCode === 302) {
+        } else if (responseCode >= 301 && responseCode < 400) {
           resolve(
             R.err({
-              message: `HTTP fetch was redirected with code ${
-                responseCode ?? 'undefined'
-              }. Redirects are either disabled or maximum number of redirects was exeeded.`,
+              message: `HTTP fetch was redirected with code ${responseCode}. Redirects are either disabled or maximum number of redirects was exeeded.`,
               diagnostic: { node: context.getOrFailProperty('url') },
             }),
           );
@@ -158,10 +157,6 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor<
 
           // Infer FileName and FileExtension from url, if not inferrable, then default to None
           // Get last element of URL assuming this is a filename
-          const urlString = context.getPropertyValue(
-            'url',
-            PrimitiveValuetypes.Text,
-          );
           const url = new URL(response.responseUrl);
           let fileName = url.pathname.split('/').pop();
           if (fileName === undefined) {

libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts

@@ -19,7 +19,8 @@ export class ArchiveInterpreterMetaInformation extends BlockMetaInformation {
         archiveType: {
           type: PrimitiveValuetypes.Text,
           docs: {
-            description: 'The archive type to be interpreted, e.g., `"zip" or "gz`.',
+            description:
+              'The archive type to be interpreted, e.g., "zip" or "gz".',
           },
         },
       },

libs/extensions/std/lang/src/http-extractor-meta-inf.ts

@@ -151,11 +151,13 @@ export class HttpExtractorMetaInformation extends BlockMetaInformation {
           type: PrimitiveValuetypes.Boolean,
           defaultValue: true,
           docs: {
-            description: 'Indicates, whether to follow redirects on get requests. If `false`, Redirects are disabled. Default `true`',
+            description:
+              'Indicates, whether to follow redirects on get requests. If `false`, redirects are not followed. Default `true`',
             examples: [
               {
                 code: 'url: "tinyurl.com/4ub9spwz" \n followRedirects: true',
-                description: 'Specifies the URL to fetch the data from and allows redirects.',
+                description:
+                  'Specifies the URL to fetch the data from and allows redirects.',
               },
             ],
           },

package.json

@@ -9,10 +9,9 @@
     "lint": "nx run-many --target lint --max-warnings 0",
     "test": "nx run-many --target test",
     "generate": "nx run language-server:generate",
-    "example:cars": "nx run interpreter:run -d example/cars.jv",
-    "example:materials": "nx run interpreter:run -d example/materials.jv",
-    "example:gtfs": "nx run interpreter:run -d example/gtfs-static-and-rt.jv",
-    "example:vehicles": "nx run interpreter:run -d -e DB_HOST=localhost -e DB_PORT=5432 -e DB_USERNAME=postgres -e DB_PASSWORD=postgres -e DB_DATABASE=postgres example/electric-vehicles.jv"
+    "example:cars": "nx run interpreter:run -d example/cars.jv -dg peek",
+    "example:gtfs": "nx run interpreter:run -d example/gtfs-static-and-rt.jv -dg peek",
+    "example:vehicles": "nx run interpreter:run -d -e DB_HOST=localhost -e DB_PORT=5432 -e DB_USERNAME=postgres -e DB_PASSWORD=postgres -e DB_DATABASE=postgres example/electric-vehicles.jv -dg peek"
   },
   "private": true,
   "dependencies": {