feature/#392_Unpack_gz_files_and_#391follow_redirects

example/materials.jv

@@ -0,0 +1,56 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+pipeline MaterialsDatabasePipeline {
+
+	block MaterialsDatabaseExtractor oftype HttpExtractor {
+		url: "https://figshare.com/ndownloader/files/31626647";
+		followRedirects: true;
+	}
+
+	block ZipArchiveInterpreter oftype ArchiveInterpreter {
+		archiveType: "zip";
+	}
+
+	block MaterialsDatabaseCSVPicker oftype FilePicker {
+		path: "/Databases/Combined/Combined_YieldStrength_GrainSize_Database.csv";
+	}
+
+	block MaterialsDatabaseTextFileInterpreter oftype TextFileInterpreter {
+
+	}
+
+	block MaterialsDatabaseCSVInterpreter oftype CSVInterpreter {
+		delimiter: ",";
+		enclosing: '"';
+		enclosingEscape: '"';
+	}
+
+	block MaterialsDatabaseTableInterpreter oftype TableInterpreter {
+		header: true;
+		columns: [
+			"Compound" oftype text,
+			"Blacklisted Compound?" oftype text,
+			"Yield Strength Value" oftype text,
+			"Yield Strength Unit" oftype text,
+			"Grain Size Value" oftype text,
+			"Grain Size Unit" oftype text,
+			"DOI" oftype text,
+			"Open Access" oftype text,
+		];
+	}
+
+	block MaterialsDatabaseLoader oftype SQLiteLoader {
+		table: "MaterialsDatabase";
+		file: "./MaterialsDatabase.sqlite";
+	}
+
+	MaterialsDatabaseExtractor
+		-> ZipArchiveInterpreter
+		-> MaterialsDatabaseCSVPicker
+		-> MaterialsDatabaseTextFileInterpreter
+		-> MaterialsDatabaseCSVInterpreter
+		-> MaterialsDatabaseTableInterpreter
+		-> MaterialsDatabaseLoader;
+}

example/thermoelectricMaterials.jv

@@ -0,0 +1,48 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+pipeline ThermoelectricMaterialsPipeline {
+
+	block ThermoelectricMaterialsExtractor oftype HttpExtractor {
+		url: "https://figshare.com/ndownloader/files/28333845";
+		followRedirects: true;
+	}	
+
+	block ThermoelectricMaterialsArchiveInterpreter oftype ArchiveInterpreter {
+		archiveType: "gz";
+	}
+
+	block ThermoelectricMaterialsFilePicker oftype FilePicker {
+		path: "/ucsb_thermoelectrics.json";
+	}
+
+	block ThermoelectricMaterialsTextFileInterpreter oftype TextFileInterpreter {
+
+	}
+
+	// Not working from here on forward, as json file is received and json interpreter is not yet implemented
+	block ThermoelectricMaterialsJSONInterpreter oftype CSVInterpreter {
+		enclosing: '"';
+	}
+
+	block ThermoelectricMaterialsTableInterpreter oftype TableInterpreter {
+		header: true;
+		columns: [
+
+		];
+	}
+
+	block ThermoelectricMaterialsLoader oftype SQLiteLoader {
+		table: "ThermoelectricMaterials";
+		file: "./ThermoelectricMaterials.sqlite";
+	}
+
+	ThermoelectricMaterialsExtractor
+		-> ThermoelectricMaterialsArchiveInterpreter
+		-> ThermoelectricMaterialsFilePicker
+		-> ThermoelectricMaterialsTextFileInterpreter
+		-> ThermoelectricMaterialsJSONInterpreter
+		-> ThermoelectricMaterialsTableInterpreter
+		-> ThermoelectricMaterialsLoader;
+}

libs/extensions/std/exec/src/archive-interpreter-executor.ts

@@ -19,7 +19,8 @@
   implementsStatic,
 } from '@jvalue/jayvee-execution';
 import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server';
 import * as JSZip from 'jszip';
+import * as zlib from 'node:zlib';
 
 import {
   inferFileExtensionFromFileExtensionString,
@@ -55,12 +56,69 @@
       }
       return R.ok(fs.right);
     }
+    if (archiveType === 'gz') {
+      const fs = await this.loadGzFileToInMemoryFileSystem(
+        archiveFile,
+        context,
+      );
+      if (R.isErr(fs)) {
+        return fs;
+      }
+      return R.ok(fs.right);
+    }
+
     return R.err({
       message: `Archive is not a zip-archive`,
       diagnostic: { node: context.getCurrentNode(), property: 'name' },
     });
   }
 
+  private async loadGzFileToInMemoryFileSystem(
+    archiveFile: BinaryFile,
+    context: ExecutionContext,
+  ): Promise<R.Result<FileSystem>> {
+    context.logger.logDebug(`Loading gz file from binary content`);
+    try {
+
+      const fs = new InMemoryFileSystem();
+      const archivedObject = zlib.gunzipSync(archiveFile.content);
+
+      const extNameArchive = path.extname(archiveFile.name);
+      const fileName = path.basename(archiveFile.name, extNameArchive);
+      const extName = path.extname(fileName);
+
+      const mimeType =
+        inferMimeTypeFromContentTypeString(extName) ||
+        MimeType.APPLICATION_OCTET_STREAM;
+      const fileExtension =
+        inferFileExtensionFromFileExtensionString(extName) ||
+        FileExtension.NONE;
+      const file = new BinaryFile(
+        fileName,
+        fileExtension,
+        mimeType,
+        archivedObject,
+      );
+
+      const addedFile = fs.putFile(
+        InMemoryFileSystem.getPathSeparator() + fileName,
+        file,
+      );
+
+      assert(addedFile != null);
+
+      return R.ok(fs);
+    }
+    catch (error: unknown) {
+      return R.err({
+        message: `Unexpected Error ${
+          error instanceof Error ? error.message : JSON.stringify(err)
+        } occured during processing`,
+        diagnostic: { node: context.getCurrentNode(), property: 'name' },
+      });
+    }
+  }
+
   private async loadZipFileToInMemoryFileSystem(
     archiveFile: BinaryFile,
     context: ExecutionContext,

libs/extensions/std/exec/src/http-extractor-executor.ts

@@ -3,9 +3,8 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import { strict as assert } from 'assert';
-import * as http from 'http';
-import * as https from 'https';
 import * as path from 'path';
+import { http, https } from 'follow-redirects';
 
 import * as R from '@jvalue/jayvee-execution';
 import {
@@ -110,8 +109,9 @@
     } else {
       httpGetFunction = http.get;
     }
+    const followRedirects = context.getPropertyValue('followRedirects', PrimitiveValuetypes.Boolean);
     return new Promise((resolve) => {
-      httpGetFunction(url, (response) => {
+      httpGetFunction(url, { followRedirects: followRedirects }, (response) => {
         const responseCode = response.statusCode;
 
         // Catch errors
@@ -126,6 +126,17 @@
           );
         }
 
+        if (responseCode === 302) {
+          resolve(
+            R.err({
+              message: `HTTP fetch was redirected with code ${
+                responseCode ?? 'undefined'
+              }. Redirects are either disabled or maximum number of redirects was exeeded.`,
+              diagnostic: { node: context.getOrFailProperty('url') },
+            }),
+          );
+        }
+
         // Get chunked data and store to ArrayBuffer
         let rawData = new Uint8Array(0);
         response.on('data', (chunk: Buffer) => {
@@ -151,7 +162,7 @@
             'url',
             PrimitiveValuetypes.Text,
           );
-          const url = new URL(urlString);
+          const url = new URL(response.responseUrl);
           let fileName = url.pathname.split('/').pop();
           if (fileName === undefined) {
             fileName = url.pathname.replace('/', '-');

libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts

@@ -19,7 +19,7 @@
         archiveType: {
           type: PrimitiveValuetypes.Text,
           docs: {
-            description: 'The archive type to be interpreted, e.g., `"zip"`.',
+            description: 'The archive type to be interpreted, e.g., `"zip" or "gz`.',
           },
         },
       },

libs/extensions/std/lang/src/http-extractor-meta-inf.ts

@@ -147,6 +147,19 @@
             }
           },
         },
+        followRedirects: {
+          type: PrimitiveValuetypes.Boolean,
+          defaultValue: true,
+          docs: {
+            description: 'Indicates, whether to follow redirects on get requests. If `false`, Redirects are disabled. Default `true`',
+            examples: [
+              {
+                code: 'url: "tinyurl.com/4ub9spwz" \n followRedirects: true',
+                description: 'Specifies the URL to fetch the data from and allows redirects.',
+              },
+            ],
+          },
+        },
       },
 
       // Input type:

package.json

@@ -9,21 +9,24 @@
     "lint": "nx run-many --target lint --max-warnings 0",
     "test": "nx run-many --target test",
     "generate": "nx run language-server:generate",
-    "example:cars": "nx run interpreter:run -d example/cars.jv -dg peek",
-    "example:gtfs": "nx run interpreter:run -d example/gtfs-static-and-rt.jv -dg peek",
-    "example:vehicles": "nx run interpreter:run -d -e DB_HOST=localhost -e DB_PORT=5432 -e DB_USERNAME=postgres -e DB_PASSWORD=postgres -e DB_DATABASE=postgres example/electric-vehicles.jv -dg peek"
+    "example:cars": "nx run interpreter:run -d example/cars.jv",
+    "example:materials": "nx run interpreter:run -d example/materials.jv",
+    "example:gtfs": "nx run interpreter:run -d example/gtfs-static-and-rt.jv",
+    "example:vehicles": "nx run interpreter:run -d -e DB_HOST=localhost -e DB_PORT=5432 -e DB_USERNAME=postgres -e DB_PASSWORD=postgres -e DB_DATABASE=postgres example/electric-vehicles.jv"
   },
   "private": true,
   "dependencies": {
     "@docusaurus/core": "2.4.1",
     "@docusaurus/preset-classic": "2.4.1",
     "@docusaurus/theme-mermaid": "2.4.1",
     "@mdx-js/react": "^1.6.22",
+    "@types/follow-redirects": "^1.14.1",
     "assert": "^2.0.0",
     "chalk": "^4.1.2",
     "clsx": "^1.2.1",
     "commander": "^8.0.0",
     "fast-csv": "^4.3.6",
+    "follow-redirects": "^1.15.2",
     "fp-ts": "^2.13.1",
     "gtfs-realtime-bindings": "^1.1.1",
     "jszip": "^3.10.1",
@@ -94,4 +97,4 @@
       "got": "^11.8.5"
     }
   }
-}
+}