From 71f22c711b3f419160e441cc4e77e41f5925be0c Mon Sep 17 00:00:00 2001 From: Philip Heltweg Date: Wed, 28 Jun 2023 09:53:20 +0200 Subject: [PATCH 01/17] Update docs to be in line with working code relates to #381 --- libs/extensions/std/lang/src/file-picker-meta-inf.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/extensions/std/lang/src/file-picker-meta-inf.ts b/libs/extensions/std/lang/src/file-picker-meta-inf.ts index df53872bd..34e4ad4f2 100644 --- a/libs/extensions/std/lang/src/file-picker-meta-inf.ts +++ b/libs/extensions/std/lang/src/file-picker-meta-inf.ts @@ -36,7 +36,7 @@ export class FilePickerMetaInformation extends BlockMetaInformation { this.docs.examples = [ { code: `block AgencyFilePicker oftype FilePicker { - path: "./agency.txt"; + path: "/agency.txt"; }`, description: 'Tries to pick the file `agency.txt` from the root of the provided `FileSystem`. If `agency.txt` exists it is passed on as `File`, if it does not exist the execution of the pipeline is aborted.', From 3a133031d919e650584f1b77131cf4f01f5989fb Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Mon, 17 Jul 2023 09:30:10 +0200 Subject: [PATCH 02/17] Added 'followRedirects' property to http-extractor-meta and added respective logic to http-extractor-executor. Added jv test file for redirects and added file to automated tests. --- example/materials.jv | 55 +++++++++++++++++++ .../std/exec/src/http-extractor-executor.ts | 25 +++++++-- .../std/lang/src/example-validation.spec.ts | 1 + .../std/lang/src/http-extractor-meta-inf.ts | 14 +++++ package-lock.json | 18 ++++++ package.json | 5 +- 6 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 example/materials.jv diff --git a/example/materials.jv b/example/materials.jv new file mode 100644 index 000000000..a0ee75619 --- /dev/null +++ b/example/materials.jv @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline MaterialsDatabasePipeline { + + block MaterialsDatabaseExtractor oftype HttpExtractor { + url: "https://figshare.com/ndownloader/files/31626647"; + followRedirect: false; + } + + block ZipArchiveInterpreter oftype ArchiveInterpreter { + archiveType: "zip"; + } + + block MaterialsDatabaseCSVPicker oftype FilePicker { + path: "/Databases/Combined/Combined_YieldStrength_GrainSize_Database.csv"; + } + + block MaterialsDatabaseTextFileInterpreter oftype TextFileInterpreter { + + } + + block MaterialsDatabaseCSVInterpreter oftype CSVInterpreter { + delimiter: ","; + enclosing: '"'; + enclosingEscape: '"'; + } + + block MaterialsDatabaseTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "Compound" oftype text, + "Blacklisted Compound?" oftype text, + "Yield Strength Value" oftype text, + "Yield Strength Unit" oftype text, + "Grain Size Value" oftype text, + "Grain Size Unit" oftype text, + "DOI" oftype text, + "Open Access" oftype text, + ]; + } + + block MaterialsDatabaseLoader oftype SQLiteLoader { + table: "MaterialsDatabase"; + file: "./MaterialsDatabase.sqlite"; + } + MaterialsDatabaseExtractor + -> ZipArchiveInterpreter + -> MaterialsDatabaseCSVPicker + -> MaterialsDatabaseTextFileInterpreter + -> MaterialsDatabaseCSVInterpreter + -> MaterialsDatabaseTableInterpreter + -> MaterialsDatabaseLoader; +} \ No newline at end of file diff --git a/libs/extensions/std/exec/src/http-extractor-executor.ts b/libs/extensions/std/exec/src/http-extractor-executor.ts index ba4c96b15..9fa418a16 100644 --- a/libs/extensions/std/exec/src/http-extractor-executor.ts +++ b/libs/extensions/std/exec/src/http-extractor-executor.ts @@ -2,9 +2,10 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import * as http from 'http'; -import * as https from 'https'; +// import * as http from 'http'; +// import * as https from 'https'; import * as path from 'path'; +import { http, https } from 'follow-redirects'; import * as R from '@jvalue/jayvee-execution'; import { @@ -61,17 +62,29 @@ export class HttpExtractorExecutor } else { httpGetFunction = http.get; } + + const followRedirects = context.getPropertyValue('followRedirects', PrimitiveValuetypes.Boolean); + return new Promise((resolve) => { - httpGetFunction(url, (response) => { + httpGetFunction(url, { followRedirects: followRedirects }, (response) => { const responseCode = response.statusCode; // Catch errors if (responseCode === undefined || responseCode >= 400) { resolve( R.err({ - message: `HTTP fetch failed with code ${ - responseCode ?? 'undefined' - }. Please check your connection.`, + message: `HTTP fetch failed with code ${responseCode ?? 'undefined' + }. Please check your connection.`, + diagnostic: { node: context.getOrFailProperty('url') }, + }), + ); + } + + if (responseCode === 302) { + resolve( + R.err({ + message: `HTTP fetch was redirected with code ${responseCode + }. Redirects are either disabled or maximum number of redirects was exeeded.`, diagnostic: { node: context.getOrFailProperty('url') }, }), ); diff --git a/libs/extensions/std/lang/src/example-validation.spec.ts b/libs/extensions/std/lang/src/example-validation.spec.ts index 7a60fbd66..afb4d526d 100644 --- a/libs/extensions/std/lang/src/example-validation.spec.ts +++ b/libs/extensions/std/lang/src/example-validation.spec.ts @@ -37,6 +37,7 @@ describe('jv example tests', () => { it.each([ 'cars.jv', + 'materials.jv', 'electric-vehicles.jv', 'gtfs-rt-simple.jv', 'gtfs-static-and-rt.jv', diff --git a/libs/extensions/std/lang/src/http-extractor-meta-inf.ts b/libs/extensions/std/lang/src/http-extractor-meta-inf.ts index 8b5b8bf89..aa5ba779c 100644 --- a/libs/extensions/std/lang/src/http-extractor-meta-inf.ts +++ b/libs/extensions/std/lang/src/http-extractor-meta-inf.ts @@ -28,7 +28,21 @@ export class HttpExtractorMetaInformation extends BlockMetaInformation { ], }, }, + followRedirects: { + type: PrimitiveValuetypes.Boolean, + defaultValue: true, + docs: { + description: 'Indicates, whether to follow redirects on get requests. If `false`, Redirects are disabled. Default `true`', + examples: [ + { + code: 'url: "tinyurl.com/4ub9spwz" \n followRedirects: true', + description: 'Specifies the URL to fetch the data from and allows redirects.', + }, + ], + }, + }, }, + // Input type: IOType.NONE, diff --git a/package-lock.json b/package-lock.json index 8e124a044..0dbfeda59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,11 +12,13 @@ "@docusaurus/preset-classic": "2.4.1", "@docusaurus/theme-mermaid": "2.4.1", "@mdx-js/react": "^1.6.22", + "@types/follow-redirects": "^1.14.1", "assert": "^2.0.0", "chalk": "^4.1.2", "clsx": "^1.2.1", "commander": "^8.0.0", "fast-csv": "^4.3.6", + "follow-redirects": "^1.15.2", "fp-ts": "^2.13.1", "gtfs-realtime-bindings": "^1.1.1", "jszip": "^3.10.1", @@ -5755,6 +5757,14 @@ "@types/range-parser": "*" } }, + "node_modules/@types/follow-redirects": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@types/follow-redirects/-/follow-redirects-1.14.1.tgz", + "integrity": "sha512-THBEFwqsLuU/K62B5JRwab9NW97cFmL4Iy34NTMX0bMycQVzq2q7PKOkhfivIwxdpa/J72RppgC42vCHfwKJ0Q==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/fs-extra": { "version": "8.1.2", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.2.tgz", @@ -28706,6 +28716,14 @@ "@types/range-parser": "*" } }, + "@types/follow-redirects": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@types/follow-redirects/-/follow-redirects-1.14.1.tgz", + "integrity": "sha512-THBEFwqsLuU/K62B5JRwab9NW97cFmL4Iy34NTMX0bMycQVzq2q7PKOkhfivIwxdpa/J72RppgC42vCHfwKJ0Q==", + "requires": { + "@types/node": "*" + } + }, "@types/fs-extra": { "version": "8.1.2", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.2.tgz", diff --git a/package.json b/package.json index 848f765fe..9c68ca0eb 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "test": "nx run-many --target test", "generate": "nx run language-server:generate", "example:cars": "nx run interpreter:run -d example/cars.jv", + "example:materials": "nx run interpreter:run -d example/materials.jv", "example:gtfs": "nx run interpreter:run -d example/gtfs-static-and-rt.jv", "example:vehicles": "nx run interpreter:run -d -e DB_HOST=localhost -e DB_PORT=5432 -e DB_USERNAME=postgres -e DB_PASSWORD=postgres -e DB_DATABASE=postgres example/electric-vehicles.jv" }, @@ -19,11 +20,13 @@ "@docusaurus/preset-classic": "2.4.1", "@docusaurus/theme-mermaid": "2.4.1", "@mdx-js/react": "^1.6.22", + "@types/follow-redirects": "^1.14.1", "assert": "^2.0.0", "chalk": "^4.1.2", "clsx": "^1.2.1", "commander": "^8.0.0", "fast-csv": "^4.3.6", + "follow-redirects": "^1.15.2", "fp-ts": "^2.13.1", "gtfs-realtime-bindings": "^1.1.1", "jszip": "^3.10.1", @@ -94,4 +97,4 @@ "got": "^11.8.5" } } -} +} \ No newline at end of file From e97b0a68e323f7e240e03cebd4618b3a82007aeb Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Mon, 17 Jul 2023 09:34:46 +0200 Subject: [PATCH 03/17] changed formatting --- example/materials.jv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/example/materials.jv b/example/materials.jv index a0ee75619..16d34f49a 100644 --- a/example/materials.jv +++ b/example/materials.jv @@ -4,7 +4,7 @@ pipeline MaterialsDatabasePipeline { - block MaterialsDatabaseExtractor oftype HttpExtractor { + block MaterialsDatabaseExtractor oftype HttpExtractor { url: "https://figshare.com/ndownloader/files/31626647"; followRedirect: false; } @@ -45,6 +45,7 @@ pipeline MaterialsDatabasePipeline { table: "MaterialsDatabase"; file: "./MaterialsDatabase.sqlite"; } + MaterialsDatabaseExtractor -> ZipArchiveInterpreter -> MaterialsDatabaseCSVPicker From a83e6f6bc4e333477fafde2fd29983aa34aaddfd Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Tue, 18 Jul 2023 08:33:33 +0200 Subject: [PATCH 04/17] initial commit. ExampleJV was only initially set up --- example/thermoelectricMaterials.jv | 89 +++++++++++++++++++ .../exec/src/archive-interpreter-executor.ts | 46 ++++++++++ package.json | 1 + 3 files changed, 136 insertions(+) create mode 100644 example/thermoelectricMaterials.jv diff --git a/example/thermoelectricMaterials.jv b/example/thermoelectricMaterials.jv new file mode 100644 index 000000000..d492010ea --- /dev/null +++ b/example/thermoelectricMaterials.jv @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline ThermoelectricMaterialsPipeline { + + block ThermoelectricMaterialsExtractor oftype HttpExtractor { + url: "https://figshare.com/ndownloader/files/28333845"; + } + + pipe { + from: ThermoelectricMaterialsExtractor; + to: ThermoelectricMaterialsArchiveInterpreter; + } + + block ThermoelectricMaterialsArchiveInterpreter oftype ArchiveInterpreter { + archiveType: "gz"; + } + + pipe { + from: ThermoelectricMaterialsArchiveInterpreter; + to: ThermoelectricMaterialsFilePicker; + } + + block ThermoelectricMaterialsFilePicker oftype FilePicker { + path: "/test"; + } + + pipe { + from: ThermoelectricMaterialsFilePicker; + to: ThermoelectricMaterialsTextFileInterpreter; + } + + block ThermoelectricMaterialsTextFileInterpreter oftype TextFileInterpreter { + + } + + pipe { + from: ThermoelectricMaterialsTextFileInterpreter; + to: ThermoelectricMaterialsCSVInterpreter; + } + + block ThermoelectricMaterialsCSVInterpreter oftype CSVInterpreter { + enclosing: '"'; + } + + pipe { + from: ThermoelectricMaterialsCSVInterpreter; + to: NameHeaderWriter; + } + + block NameHeaderWriter oftype CellWriter { + at: cell A1; + write: ["name"]; + } + + pipe { + from: NameHeaderWriter; + to: ThermoelectricMaterialsTableInterpreter; + } + + block ThermoelectricMaterialsTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "name" oftype text, + "mpg" oftype decimal, + "cyl" oftype integer, + "disp" oftype decimal, + "hp" oftype integer, + "drat" oftype decimal, + "wt" oftype decimal, + "qsec" oftype decimal, + "vs" oftype integer, + "am" oftype integer, + "gear" oftype integer, + "carb" oftype integer + ]; + } + + pipe { + from: ThermoelectricMaterialsTableInterpreter; + to: ThermoelectricMaterialsLoader; + } + + block ThermoelectricMaterialsLoader oftype SQLiteLoader { + table: "ThermoelectricMaterials"; + file: "./ThermoelectricMaterials.sqlite"; + } +} \ No newline at end of file diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index 79f833618..7ca8674d7 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -20,6 +20,10 @@ import { } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; import * as JSZip from 'jszip'; +import * as zlib from 'node:zlib'; +import { createGunzip } from 'node:zlib'; +import { createReadStream, createWriteStream, readFile } from 'fs'; + import { inferFileExtensionFromFileExtensionString, @@ -42,6 +46,7 @@ export class ArchiveInterpreterExecutor 'archiveType', PrimitiveValuetypes.Text, ); + console.log(archiveFile); if (archiveType === 'zip') { const fs = await this.loadZipFileToInMemoryFileSystem( archiveFile, @@ -52,12 +57,53 @@ export class ArchiveInterpreterExecutor } return R.ok(fs.right); } + if (archiveType === 'gz') { + const fs = await this.loadGZipFileToInMemoryFileSystem( + archiveFile, + context, + ); + if (R.isErr(fs)) { + return fs; + } + return R.ok(fs.right); + } + return R.err({ message: `Archive is not a zip-archive`, diagnostic: { node: context.getCurrentNode(), property: 'name' }, }); } + private async loadGZipFileToInMemoryFileSystem( + archiveFile: BinaryFile, + context: ExecutionContext, + ): Promise> { + context.logger.logDebug(`Loading zip file from binary content`); + try { + + const fs = new InMemoryFileSystem(); + console.log(archiveFile); + + zlib.gunzip(archiveFile.content, function(error, result) { + console.log(result); + fs.putFile( + InMemoryFileSystem.getPathSeparator(), + archiveFile, + ); + }); + return R.ok(fs); + } + catch (error: unknown) { + return R.err({ + message: `Unexpected Error ${ + error instanceof Error ? error.message : JSON.stringify(err) + } occured during processing`, + diagnostic: { node: context.getCurrentNode(), property: 'name' }, + }); + } + } + + private async loadZipFileToInMemoryFileSystem( archiveFile: BinaryFile, context: ExecutionContext, diff --git a/package.json b/package.json index 848f765fe..0b59cdff0 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "test": "nx run-many --target test", "generate": "nx run language-server:generate", "example:cars": "nx run interpreter:run -d example/cars.jv", + "example:thermoelectricMaterials": "nx run interpreter:run -d example/thermoelectricMaterials.jv", "example:gtfs": "nx run interpreter:run -d example/gtfs-static-and-rt.jv", "example:vehicles": "nx run interpreter:run -d -e DB_HOST=localhost -e DB_PORT=5432 -e DB_USERNAME=postgres -e DB_PASSWORD=postgres -e DB_DATABASE=postgres example/electric-vehicles.jv" }, From 614901349af081dd5d8e8a5a6003ab65ffe60d71 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Tue, 18 Jul 2023 16:46:47 +0200 Subject: [PATCH 05/17] Added gzip decompress to archive-interpreter and fixed issue with http-extractor on redirects --- example/materials.jv | 2 +- example/thermoelectricMaterials.jv | 67 ++++--------------- .../exec/src/archive-interpreter-executor.ts | 35 +++++++--- .../std/exec/src/http-extractor-executor.ts | 2 +- 4 files changed, 39 insertions(+), 67 deletions(-) diff --git a/example/materials.jv b/example/materials.jv index 16d34f49a..813f02783 100644 --- a/example/materials.jv +++ b/example/materials.jv @@ -6,7 +6,7 @@ pipeline MaterialsDatabasePipeline { block MaterialsDatabaseExtractor oftype HttpExtractor { url: "https://figshare.com/ndownloader/files/31626647"; - followRedirect: false; + followRedirects: true; } block ZipArchiveInterpreter oftype ArchiveInterpreter { diff --git a/example/thermoelectricMaterials.jv b/example/thermoelectricMaterials.jv index d492010ea..d2b1e160c 100644 --- a/example/thermoelectricMaterials.jv +++ b/example/thermoelectricMaterials.jv @@ -6,84 +6,43 @@ pipeline ThermoelectricMaterialsPipeline { block ThermoelectricMaterialsExtractor oftype HttpExtractor { url: "https://figshare.com/ndownloader/files/28333845"; + followRedirects: true; } - - pipe { - from: ThermoelectricMaterialsExtractor; - to: ThermoelectricMaterialsArchiveInterpreter; - } block ThermoelectricMaterialsArchiveInterpreter oftype ArchiveInterpreter { archiveType: "gz"; } - pipe { - from: ThermoelectricMaterialsArchiveInterpreter; - to: ThermoelectricMaterialsFilePicker; - } - block ThermoelectricMaterialsFilePicker oftype FilePicker { - path: "/test"; - } - - pipe { - from: ThermoelectricMaterialsFilePicker; - to: ThermoelectricMaterialsTextFileInterpreter; + path: "/ucsb_thermoelectrics.json"; } block ThermoelectricMaterialsTextFileInterpreter oftype TextFileInterpreter { } - pipe { - from: ThermoelectricMaterialsTextFileInterpreter; - to: ThermoelectricMaterialsCSVInterpreter; - } - - block ThermoelectricMaterialsCSVInterpreter oftype CSVInterpreter { + // Not working from here on forward, as json file is received and json interpreter is not yet implemented + block ThermoelectricMaterialsJSONInterpreter oftype CSVInterpreter { enclosing: '"'; } - pipe { - from: ThermoelectricMaterialsCSVInterpreter; - to: NameHeaderWriter; - } - - block NameHeaderWriter oftype CellWriter { - at: cell A1; - write: ["name"]; - } - - pipe { - from: NameHeaderWriter; - to: ThermoelectricMaterialsTableInterpreter; - } - block ThermoelectricMaterialsTableInterpreter oftype TableInterpreter { header: true; columns: [ - "name" oftype text, - "mpg" oftype decimal, - "cyl" oftype integer, - "disp" oftype decimal, - "hp" oftype integer, - "drat" oftype decimal, - "wt" oftype decimal, - "qsec" oftype decimal, - "vs" oftype integer, - "am" oftype integer, - "gear" oftype integer, - "carb" oftype integer + ]; } - pipe { - from: ThermoelectricMaterialsTableInterpreter; - to: ThermoelectricMaterialsLoader; - } - block ThermoelectricMaterialsLoader oftype SQLiteLoader { table: "ThermoelectricMaterials"; file: "./ThermoelectricMaterials.sqlite"; } + + ThermoelectricMaterialsExtractor + -> ThermoelectricMaterialsArchiveInterpreter + -> ThermoelectricMaterialsFilePicker + -> ThermoelectricMaterialsTextFileInterpreter + -> ThermoelectricMaterialsJSONInterpreter + -> ThermoelectricMaterialsTableInterpreter + -> ThermoelectricMaterialsLoader; } \ No newline at end of file diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index 7ca8674d7..1bb62e57e 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -21,8 +21,6 @@ import { import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; import * as JSZip from 'jszip'; import * as zlib from 'node:zlib'; -import { createGunzip } from 'node:zlib'; -import { createReadStream, createWriteStream, readFile } from 'fs'; import { @@ -46,7 +44,6 @@ export class ArchiveInterpreterExecutor 'archiveType', PrimitiveValuetypes.Text, ); - console.log(archiveFile); if (archiveType === 'zip') { const fs = await this.loadZipFileToInMemoryFileSystem( archiveFile, @@ -82,15 +79,31 @@ export class ArchiveInterpreterExecutor try { const fs = new InMemoryFileSystem(); - console.log(archiveFile); + const archivedObject = zlib.gunzipSync(archiveFile.content); - zlib.gunzip(archiveFile.content, function(error, result) { - console.log(result); - fs.putFile( - InMemoryFileSystem.getPathSeparator(), - archiveFile, - ); - }); + const extNameArchive = path.extname(archiveFile.name); + const fileName = path.basename(archiveFile.name, extNameArchive); + const extName = path.extname(fileName); + + const mimeType = + inferMimeTypeFromContentTypeString(extName) || + MimeType.APPLICATION_OCTET_STREAM; + const fileExtension = + inferFileExtensionFromFileExtensionString(extName) || + FileExtension.NONE; + const file = new BinaryFile( + fileName, + fileExtension, + mimeType, + archivedObject, + ); + const addedFile = fs.putFile( + InMemoryFileSystem.getPathSeparator() + fileName, + file, + ); + + assert(addedFile != null); + return R.ok(fs); } catch (error: unknown) { diff --git a/libs/extensions/std/exec/src/http-extractor-executor.ts b/libs/extensions/std/exec/src/http-extractor-executor.ts index 9fa418a16..a9c7dcb78 100644 --- a/libs/extensions/std/exec/src/http-extractor-executor.ts +++ b/libs/extensions/std/exec/src/http-extractor-executor.ts @@ -116,7 +116,7 @@ export class HttpExtractorExecutor 'url', PrimitiveValuetypes.Text, ); - const url = new URL(urlString); + const url = new URL(response.responseUrl); let fileName = url.pathname.split('/').pop(); if (fileName === undefined) { fileName = url.pathname.replace('/', '-'); From 7b963ad12a810ede6f18a6ee7424685ffcdd89c0 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Tue, 18 Jul 2023 16:53:00 +0200 Subject: [PATCH 06/17] added to description, format and tests --- libs/extensions/std/exec/src/archive-interpreter-executor.ts | 1 + libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts | 2 +- libs/extensions/std/lang/src/example-validation.spec.ts | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index 1bb62e57e..7ab32cc95 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -97,6 +97,7 @@ export class ArchiveInterpreterExecutor mimeType, archivedObject, ); + const addedFile = fs.putFile( InMemoryFileSystem.getPathSeparator() + fileName, file, diff --git a/libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts b/libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts index adf197402..d48290bf1 100644 --- a/libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts +++ b/libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts @@ -19,7 +19,7 @@ export class ArchiveInterpreterMetaInformation extends BlockMetaInformation { archiveType: { type: PrimitiveValuetypes.Text, docs: { - description: 'The archive type to be interpreted, e.g., `"zip"`.', + description: 'The archive type to be interpreted, e.g., `"zip" or "gz`.', }, }, }, diff --git a/libs/extensions/std/lang/src/example-validation.spec.ts b/libs/extensions/std/lang/src/example-validation.spec.ts index afb4d526d..bef2626cf 100644 --- a/libs/extensions/std/lang/src/example-validation.spec.ts +++ b/libs/extensions/std/lang/src/example-validation.spec.ts @@ -38,6 +38,8 @@ describe('jv example tests', () => { it.each([ 'cars.jv', 'materials.jv', + // TODO: implement JSON-Interpreter + 'thermoelectricMaterials.jv', 'electric-vehicles.jv', 'gtfs-rt-simple.jv', 'gtfs-static-and-rt.jv', From a23642af5402a8ba6c37491c04b45fb6c0ee9c41 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Tue, 18 Jul 2023 16:54:39 +0200 Subject: [PATCH 07/17] removed unused imports --- libs/extensions/std/exec/src/http-extractor-executor.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/extensions/std/exec/src/http-extractor-executor.ts b/libs/extensions/std/exec/src/http-extractor-executor.ts index a9c7dcb78..ac375cef0 100644 --- a/libs/extensions/std/exec/src/http-extractor-executor.ts +++ b/libs/extensions/std/exec/src/http-extractor-executor.ts @@ -2,8 +2,6 @@ // // SPDX-License-Identifier: AGPL-3.0-only -// import * as http from 'http'; -// import * as https from 'https'; import * as path from 'path'; import { http, https } from 'follow-redirects'; From 5f55fb9260c183d7b5c0fa7f0e645ecaec712872 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Fri, 21 Jul 2023 09:21:57 +0200 Subject: [PATCH 08/17] formatted code --- example/materials.jv | 17 +++++++++-------- .../exec/src/archive-interpreter-executor.ts | 8 +++----- .../std/exec/src/http-extractor-executor.ts | 12 ++++++------ .../std/lang/src/file-picker-meta-inf.ts | 2 +- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/example/materials.jv b/example/materials.jv index 813f02783..b6988f39c 100644 --- a/example/materials.jv +++ b/example/materials.jv @@ -6,6 +6,7 @@ pipeline MaterialsDatabasePipeline { block MaterialsDatabaseExtractor oftype HttpExtractor { url: "https://figshare.com/ndownloader/files/31626647"; + followRedirects: true; } @@ -30,14 +31,14 @@ pipeline MaterialsDatabasePipeline { block MaterialsDatabaseTableInterpreter oftype TableInterpreter { header: true; columns: [ - "Compound" oftype text, - "Blacklisted Compound?" oftype text, - "Yield Strength Value" oftype text, - "Yield Strength Unit" oftype text, - "Grain Size Value" oftype text, - "Grain Size Unit" oftype text, - "DOI" oftype text, - "Open Access" oftype text, + "Compound" oftype text, + "Blacklisted Compound?" oftype text, + "Yield Strength Value" oftype text, + "Yield Strength Unit" oftype text, + "Grain Size Value" oftype text, + "Grain Size Unit" oftype text, + "DOI" oftype text, + "Open Access" oftype text, ]; } diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index c3a0ceab6..fe2d782f6 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -22,7 +22,6 @@ import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; import * as JSZip from 'jszip'; import * as zlib from 'node:zlib'; - import { inferFileExtensionFromFileExtensionString, inferMimeTypeFromContentTypeString, @@ -58,7 +57,7 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< return R.ok(fs.right); } if (archiveType === 'gz') { - const fs = await this.loadGZipFileToInMemoryFileSystem( + const fs = await this.loadGzFileToInMemoryFileSystem( archiveFile, context, ); @@ -74,11 +73,11 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< }); } - private async loadGZipFileToInMemoryFileSystem( + private async loadGzFileToInMemoryFileSystem( archiveFile: BinaryFile, context: ExecutionContext, ): Promise> { - context.logger.logDebug(`Loading zip file from binary content`); + context.logger.logDebug(`Loading gz file from binary content`); try { const fs = new InMemoryFileSystem(); @@ -120,7 +119,6 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< } } - private async loadZipFileToInMemoryFileSystem( archiveFile: BinaryFile, context: ExecutionContext, diff --git a/libs/extensions/std/exec/src/http-extractor-executor.ts b/libs/extensions/std/exec/src/http-extractor-executor.ts index 439fd8dc3..2b9a6b98f 100644 --- a/libs/extensions/std/exec/src/http-extractor-executor.ts +++ b/libs/extensions/std/exec/src/http-extractor-executor.ts @@ -109,9 +109,7 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor< } else { httpGetFunction = http.get; } - const followRedirects = context.getPropertyValue('followRedirects', PrimitiveValuetypes.Boolean); - return new Promise((resolve) => { httpGetFunction(url, { followRedirects: followRedirects }, (response) => { const responseCode = response.statusCode; @@ -120,8 +118,9 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor< if (responseCode === undefined || responseCode >= 400) { resolve( R.err({ - message: `HTTP fetch failed with code ${responseCode ?? 'undefined' - }. Please check your connection.`, + message: `HTTP fetch failed with code ${ + responseCode ?? 'undefined' + }. Please check your connection.`, diagnostic: { node: context.getOrFailProperty('url') }, }), ); @@ -130,8 +129,9 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor< if (responseCode === 302) { resolve( R.err({ - message: `HTTP fetch was redirected with code ${responseCode - }. Redirects are either disabled or maximum number of redirects was exeeded.`, + message: `HTTP fetch was redirected with code ${ + responseCode ?? 'undefined' + }. Redirects are either disabled or maximum number of redirects was exeeded.`, diagnostic: { node: context.getOrFailProperty('url') }, }), ); diff --git a/libs/extensions/std/lang/src/file-picker-meta-inf.ts b/libs/extensions/std/lang/src/file-picker-meta-inf.ts index 34e4ad4f2..df53872bd 100644 --- a/libs/extensions/std/lang/src/file-picker-meta-inf.ts +++ b/libs/extensions/std/lang/src/file-picker-meta-inf.ts @@ -36,7 +36,7 @@ export class FilePickerMetaInformation extends BlockMetaInformation { this.docs.examples = [ { code: `block AgencyFilePicker oftype FilePicker { - path: "/agency.txt"; + path: "./agency.txt"; }`, description: 'Tries to pick the file `agency.txt` from the root of the provided `FileSystem`. If `agency.txt` exists it is passed on as `File`, if it does not exist the execution of the pipeline is aborted.', From 8d0d1243568642352e5f4a3687ddac5f0ee112fd Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Fri, 21 Jul 2023 09:27:16 +0200 Subject: [PATCH 09/17] format --- example/materials.jv | 7 +++---- example/thermoelectricMaterials.jv | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/example/materials.jv b/example/materials.jv index b6988f39c..da798feeb 100644 --- a/example/materials.jv +++ b/example/materials.jv @@ -6,7 +6,6 @@ pipeline MaterialsDatabasePipeline { block MaterialsDatabaseExtractor oftype HttpExtractor { url: "https://figshare.com/ndownloader/files/31626647"; - followRedirects: true; } @@ -18,14 +17,14 @@ pipeline MaterialsDatabasePipeline { path: "/Databases/Combined/Combined_YieldStrength_GrainSize_Database.csv"; } - block MaterialsDatabaseTextFileInterpreter oftype TextFileInterpreter { + block MaterialsDatabaseTextFileInterpreter oftype TextFileInterpreter { } block MaterialsDatabaseCSVInterpreter oftype CSVInterpreter { delimiter: ","; - enclosing: '"'; - enclosingEscape: '"'; + enclosing: '"'; + enclosingEscape: '"'; } block MaterialsDatabaseTableInterpreter oftype TableInterpreter { diff --git a/example/thermoelectricMaterials.jv b/example/thermoelectricMaterials.jv index d2b1e160c..0d2ae8eeb 100644 --- a/example/thermoelectricMaterials.jv +++ b/example/thermoelectricMaterials.jv @@ -3,7 +3,7 @@ // SPDX-License-Identifier: AGPL-3.0-only pipeline ThermoelectricMaterialsPipeline { - + block ThermoelectricMaterialsExtractor oftype HttpExtractor { url: "https://figshare.com/ndownloader/files/28333845"; followRedirects: true; From 9fe5645958ba8219b5afb430b1893ebfb2207630 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Fri, 21 Jul 2023 11:46:39 +0200 Subject: [PATCH 10/17] removed jv example files --- example/materials.jv | 56 ------------------------------ example/thermoelectricMaterials.jv | 48 ------------------------- 2 files changed, 104 deletions(-) delete mode 100644 example/materials.jv delete mode 100644 example/thermoelectricMaterials.jv diff --git a/example/materials.jv b/example/materials.jv deleted file mode 100644 index da798feeb..000000000 --- a/example/materials.jv +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -pipeline MaterialsDatabasePipeline { - - block MaterialsDatabaseExtractor oftype HttpExtractor { - url: "https://figshare.com/ndownloader/files/31626647"; - followRedirects: true; - } - - block ZipArchiveInterpreter oftype ArchiveInterpreter { - archiveType: "zip"; - } - - block MaterialsDatabaseCSVPicker oftype FilePicker { - path: "/Databases/Combined/Combined_YieldStrength_GrainSize_Database.csv"; - } - - block MaterialsDatabaseTextFileInterpreter oftype TextFileInterpreter { - - } - - block MaterialsDatabaseCSVInterpreter oftype CSVInterpreter { - delimiter: ","; - enclosing: '"'; - enclosingEscape: '"'; - } - - block MaterialsDatabaseTableInterpreter oftype TableInterpreter { - header: true; - columns: [ - "Compound" oftype text, - "Blacklisted Compound?" oftype text, - "Yield Strength Value" oftype text, - "Yield Strength Unit" oftype text, - "Grain Size Value" oftype text, - "Grain Size Unit" oftype text, - "DOI" oftype text, - "Open Access" oftype text, - ]; - } - - block MaterialsDatabaseLoader oftype SQLiteLoader { - table: "MaterialsDatabase"; - file: "./MaterialsDatabase.sqlite"; - } - - MaterialsDatabaseExtractor - -> ZipArchiveInterpreter - -> MaterialsDatabaseCSVPicker - -> MaterialsDatabaseTextFileInterpreter - -> MaterialsDatabaseCSVInterpreter - -> MaterialsDatabaseTableInterpreter - -> MaterialsDatabaseLoader; -} \ No newline at end of file diff --git a/example/thermoelectricMaterials.jv b/example/thermoelectricMaterials.jv deleted file mode 100644 index 0d2ae8eeb..000000000 --- a/example/thermoelectricMaterials.jv +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -pipeline ThermoelectricMaterialsPipeline { - - block ThermoelectricMaterialsExtractor oftype HttpExtractor { - url: "https://figshare.com/ndownloader/files/28333845"; - followRedirects: true; - } - - block ThermoelectricMaterialsArchiveInterpreter oftype ArchiveInterpreter { - archiveType: "gz"; - } - - block ThermoelectricMaterialsFilePicker oftype FilePicker { - path: "/ucsb_thermoelectrics.json"; - } - - block ThermoelectricMaterialsTextFileInterpreter oftype TextFileInterpreter { - - } - - // Not working from here on forward, as json file is received and json interpreter is not yet implemented - block ThermoelectricMaterialsJSONInterpreter oftype CSVInterpreter { - enclosing: '"'; - } - - block ThermoelectricMaterialsTableInterpreter oftype TableInterpreter { - header: true; - columns: [ - - ]; - } - - block ThermoelectricMaterialsLoader oftype SQLiteLoader { - table: "ThermoelectricMaterials"; - file: "./ThermoelectricMaterials.sqlite"; - } - - ThermoelectricMaterialsExtractor - -> ThermoelectricMaterialsArchiveInterpreter - -> ThermoelectricMaterialsFilePicker - -> ThermoelectricMaterialsTextFileInterpreter - -> ThermoelectricMaterialsJSONInterpreter - -> ThermoelectricMaterialsTableInterpreter - -> ThermoelectricMaterialsLoader; -} \ No newline at end of file From 88d480ab1123a35806dd8259b6b4f035f4092b83 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Wed, 26 Jul 2023 14:19:02 +0200 Subject: [PATCH 11/17] formatted and refactored code incorporated feedback from PR --- .../exec/src/archive-interpreter-executor.ts | 126 ++++++++---------- .../std/exec/src/http-extractor-executor.ts | 23 ++-- .../lang/src/archive-interpreter-meta-inf.ts | 3 +- .../std/lang/src/http-extractor-meta-inf.ts | 6 +- 4 files changed, 72 insertions(+), 86 deletions(-) diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index fe2d782f6..225ec1d0d 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -3,6 +3,7 @@ // SPDX-License-Identifier: AGPL-3.0-only import { strict as assert } from 'assert'; +import * as zlib from 'node:zlib'; import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; @@ -20,7 +21,6 @@ import { } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; import * as JSZip from 'jszip'; -import * as zlib from 'node:zlib'; import { inferFileExtensionFromFileExtensionString, @@ -46,76 +46,52 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< 'archiveType', PrimitiveValuetypes.Text, ); + let fs; + if (archiveType === 'zip') { - const fs = await this.loadZipFileToInMemoryFileSystem( - archiveFile, - context, - ); - if (R.isErr(fs)) { - return fs; - } - return R.ok(fs.right); - } - if (archiveType === 'gz') { - const fs = await this.loadGzFileToInMemoryFileSystem( - archiveFile, - context, - ); - if (R.isErr(fs)) { - return fs; - } - return R.ok(fs.right); + fs = await this.loadZipFileToInMemoryFileSystem(archiveFile, context); + } else if (archiveType === 'gz') { + fs = this.loadGzFileToInMemoryFileSystem(archiveFile, context); + } else { + return R.err({ + message: `Archive type is not supported`, + diagnostic: { node: context.getCurrentNode(), property: 'name' }, + }); } - return R.err({ - message: `Archive is not a zip-archive`, - diagnostic: { node: context.getCurrentNode(), property: 'name' }, - }); + if (R.isErr(fs)) { + return fs; + } + return R.ok(fs.right); } - private async loadGzFileToInMemoryFileSystem( + private loadGzFileToInMemoryFileSystem( archiveFile: BinaryFile, context: ExecutionContext, - ): Promise> { + ): R.Result { context.logger.logDebug(`Loading gz file from binary content`); try { - const fs = new InMemoryFileSystem(); const archivedObject = zlib.gunzipSync(archiveFile.content); - + const extNameArchive = path.extname(archiveFile.name); - const fileName = path.basename(archiveFile.name, extNameArchive); - const extName = path.extname(fileName); - - const mimeType = - inferMimeTypeFromContentTypeString(extName) || - MimeType.APPLICATION_OCTET_STREAM; - const fileExtension = - inferFileExtensionFromFileExtensionString(extName) || - FileExtension.NONE; - const file = new BinaryFile( - fileName, - fileExtension, - mimeType, + + const file = this.createBinaryFromArchiveFile( + archiveFile.name, archivedObject, + extNameArchive, ); - + const addedFile = fs.putFile( - InMemoryFileSystem.getPathSeparator() + fileName, + InMemoryFileSystem.getPathSeparator() + file.name, file, ); assert(addedFile != null); return R.ok(fs); - } - catch (error: unknown) { - return R.err({ - message: `Unexpected Error ${ - error instanceof Error ? error.message : JSON.stringify(err) - } occured during processing`, - diagnostic: { node: context.getCurrentNode(), property: 'name' }, - }); + } catch (error: unknown) { + return R.err(this.generateErrorObject(context, error)); } } @@ -133,21 +109,11 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< )) { if (!archivedObject.dir) { const content = await archivedObject.async('arraybuffer'); - // Ext incl. leading dot - const extName = path.extname(archivedObject.name); - const fileName = path.basename(archivedObject.name); - const mimeType = - inferMimeTypeFromContentTypeString(extName) || - MimeType.APPLICATION_OCTET_STREAM; - const fileExtension = - inferFileExtensionFromFileExtensionString(extName) || - FileExtension.NONE; - const file = new BinaryFile( - fileName, - fileExtension, - mimeType, + const file = this.createBinaryFromArchiveFile( + archivedObject.name, content, ); + // Ext incl. leading dot const addedFile = fs.putFile( InMemoryFileSystem.getPathSeparator() + relPath, file, @@ -157,12 +123,34 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< } return R.ok(fs); } catch (error: unknown) { - return R.err({ - message: `Unexpected Error ${ - error instanceof Error ? error.message : JSON.stringify(err) - } occured during processing`, - diagnostic: { node: context.getCurrentNode(), property: 'name' }, - }); + return R.err(this.generateErrorObject(context, error)); } } + + private createBinaryFromArchiveFile( + archiveFileName: string, + content: ArrayBuffer, + extNameArchive?: string, + ): BinaryFile { + const fileName = path.basename(archiveFileName, extNameArchive); + const extName = path.extname(fileName); + + const mimeType = + inferMimeTypeFromContentTypeString(extName) || + MimeType.APPLICATION_OCTET_STREAM; + const fileExtension = + inferFileExtensionFromFileExtensionString(extName) || FileExtension.NONE; + const file = new BinaryFile(fileName, fileExtension, mimeType, content); + + return file; + } + + private generateErrorObject(context: ExecutionContext, error: unknown) { + return { + message: `Unexpected Error ${ + error instanceof Error ? error.message : JSON.stringify(err) + } occured during processing`, + diagnostic: { node: context.getCurrentNode(), property: 'name' }, + }; + } } diff --git a/libs/extensions/std/exec/src/http-extractor-executor.ts b/libs/extensions/std/exec/src/http-extractor-executor.ts index 2b9a6b98f..03534cfcb 100644 --- a/libs/extensions/std/exec/src/http-extractor-executor.ts +++ b/libs/extensions/std/exec/src/http-extractor-executor.ts @@ -4,7 +4,6 @@ import { strict as assert } from 'assert'; import * as path from 'path'; -import { http, https } from 'follow-redirects'; import * as R from '@jvalue/jayvee-execution'; import { @@ -19,6 +18,7 @@ import { implementsStatic, } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; +import { http, https } from 'follow-redirects'; import { AstNode } from 'langium'; import { @@ -105,11 +105,14 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor< context.logger.logDebug(`Fetching raw data from ${url}`); let httpGetFunction: HttpGetFunction; if (url.startsWith('https')) { - httpGetFunction = https.get; + httpGetFunction = https.get.bind(https); } else { - httpGetFunction = http.get; + httpGetFunction = http.get.bind(http); } - const followRedirects = context.getPropertyValue('followRedirects', PrimitiveValuetypes.Boolean); + const followRedirects = context.getPropertyValue( + 'followRedirects', + PrimitiveValuetypes.Boolean, + ); return new Promise((resolve) => { httpGetFunction(url, { followRedirects: followRedirects }, (response) => { const responseCode = response.statusCode; @@ -124,14 +127,10 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor< diagnostic: { node: context.getOrFailProperty('url') }, }), ); - } - - if (responseCode === 302) { + } else if (responseCode >= 301 && responseCode < 400) { resolve( R.err({ - message: `HTTP fetch was redirected with code ${ - responseCode ?? 'undefined' - }. Redirects are either disabled or maximum number of redirects was exeeded.`, + message: `HTTP fetch was redirected with code ${responseCode}. Redirects are either disabled or maximum number of redirects was exeeded.`, diagnostic: { node: context.getOrFailProperty('url') }, }), ); @@ -158,10 +157,6 @@ export class HttpExtractorExecutor extends AbstractBlockExecutor< // Infer FileName and FileExtension from url, if not inferrable, then default to None // Get last element of URL assuming this is a filename - const urlString = context.getPropertyValue( - 'url', - PrimitiveValuetypes.Text, - ); const url = new URL(response.responseUrl); let fileName = url.pathname.split('/').pop(); if (fileName === undefined) { diff --git a/libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts b/libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts index d48290bf1..7339fbd2a 100644 --- a/libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts +++ b/libs/extensions/std/lang/src/archive-interpreter-meta-inf.ts @@ -19,7 +19,8 @@ export class ArchiveInterpreterMetaInformation extends BlockMetaInformation { archiveType: { type: PrimitiveValuetypes.Text, docs: { - description: 'The archive type to be interpreted, e.g., `"zip" or "gz`.', + description: + 'The archive type to be interpreted, e.g., "zip" or "gz".', }, }, }, diff --git a/libs/extensions/std/lang/src/http-extractor-meta-inf.ts b/libs/extensions/std/lang/src/http-extractor-meta-inf.ts index 9724b4edc..1ac27dc34 100644 --- a/libs/extensions/std/lang/src/http-extractor-meta-inf.ts +++ b/libs/extensions/std/lang/src/http-extractor-meta-inf.ts @@ -151,11 +151,13 @@ export class HttpExtractorMetaInformation extends BlockMetaInformation { type: PrimitiveValuetypes.Boolean, defaultValue: true, docs: { - description: 'Indicates, whether to follow redirects on get requests. If `false`, Redirects are disabled. Default `true`', + description: + 'Indicates, whether to follow redirects on get requests. If `false`, redirects are not followed. Default `true`', examples: [ { code: 'url: "tinyurl.com/4ub9spwz" \n followRedirects: true', - description: 'Specifies the URL to fetch the data from and allows redirects.', + description: + 'Specifies the URL to fetch the data from and allows redirects.', }, ], }, From 35aef3fdaf023f0db2bb9b236cfcbccdf7349991 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Wed, 26 Jul 2023 15:02:13 +0200 Subject: [PATCH 12/17] refactored to reduce repetition in code --- .../exec/src/archive-interpreter-executor.ts | 49 ++++++++----------- package.json | 1 - 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index 225ec1d0d..841449fd6 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -76,18 +76,9 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< const extNameArchive = path.extname(archiveFile.name); - const file = this.createBinaryFromArchiveFile( - archiveFile.name, - archivedObject, - extNameArchive, - ); - - const addedFile = fs.putFile( - InMemoryFileSystem.getPathSeparator() + file.name, - file, - ); - - assert(addedFile != null); + this.createBinaryAndPutFile(fs, archiveFile.name, archivedObject, { + extNameArchive: extNameArchive, + }); return R.ok(fs); } catch (error: unknown) { @@ -109,16 +100,9 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< )) { if (!archivedObject.dir) { const content = await archivedObject.async('arraybuffer'); - const file = this.createBinaryFromArchiveFile( - archivedObject.name, - content, - ); - // Ext incl. leading dot - const addedFile = fs.putFile( - InMemoryFileSystem.getPathSeparator() + relPath, - file, - ); - assert(addedFile != null); + this.createBinaryAndPutFile(fs, archivedObject.name, content, { + relPath: relPath, + }); } } return R.ok(fs); @@ -127,12 +111,13 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< } } - private createBinaryFromArchiveFile( + private createBinaryAndPutFile( + fs: InMemoryFileSystem, archiveFileName: string, content: ArrayBuffer, - extNameArchive?: string, - ): BinaryFile { - const fileName = path.basename(archiveFileName, extNameArchive); + options?: { extNameArchive?: string; relPath?: string }, + ) { + const fileName = path.basename(archiveFileName, options?.extNameArchive); const extName = path.extname(fileName); const mimeType = @@ -141,8 +126,16 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< const fileExtension = inferFileExtensionFromFileExtensionString(extName) || FileExtension.NONE; const file = new BinaryFile(fileName, fileExtension, mimeType, content); - - return file; + console.log( + InMemoryFileSystem.getPathSeparator() + + String(options?.relPath ?? 'Teeeeeeeeest'), + ); + const addedFile = fs.putFile( + InMemoryFileSystem.getPathSeparator() + + String(options?.relPath ?? fileName), + file, + ); + assert(addedFile != null); } private generateErrorObject(context: ExecutionContext, error: unknown) { diff --git a/package.json b/package.json index 9c68ca0eb..3be1a706a 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,6 @@ "test": "nx run-many --target test", "generate": "nx run language-server:generate", "example:cars": "nx run interpreter:run -d example/cars.jv", - "example:materials": "nx run interpreter:run -d example/materials.jv", "example:gtfs": "nx run interpreter:run -d example/gtfs-static-and-rt.jv", "example:vehicles": "nx run interpreter:run -d -e DB_HOST=localhost -e DB_PORT=5432 -e DB_USERNAME=postgres -e DB_PASSWORD=postgres -e DB_DATABASE=postgres example/electric-vehicles.jv" }, From b1f1b0db4dfdc75bc4de3fa4428f0a3b1e8590f8 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Wed, 26 Jul 2023 15:07:11 +0200 Subject: [PATCH 13/17] fixed error in previous commit --- libs/extensions/std/exec/src/archive-interpreter-executor.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index 841449fd6..537f67b72 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -126,10 +126,7 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< const fileExtension = inferFileExtensionFromFileExtensionString(extName) || FileExtension.NONE; const file = new BinaryFile(fileName, fileExtension, mimeType, content); - console.log( - InMemoryFileSystem.getPathSeparator() + - String(options?.relPath ?? 'Teeeeeeeeest'), - ); + const addedFile = fs.putFile( InMemoryFileSystem.getPathSeparator() + String(options?.relPath ?? fileName), From 5194dbe8076d266eb599632ecf3d4a2256f62479 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Wed, 26 Jul 2023 16:17:19 +0200 Subject: [PATCH 14/17] . --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 3be1a706a..1edf42734 100644 --- a/package.json +++ b/package.json @@ -9,9 +9,9 @@ "lint": "nx run-many --target lint --max-warnings 0", "test": "nx run-many --target test", "generate": "nx run language-server:generate", - "example:cars": "nx run interpreter:run -d example/cars.jv", - "example:gtfs": "nx run interpreter:run -d example/gtfs-static-and-rt.jv", - "example:vehicles": "nx run interpreter:run -d -e DB_HOST=localhost -e DB_PORT=5432 -e DB_USERNAME=postgres -e DB_PASSWORD=postgres -e DB_DATABASE=postgres example/electric-vehicles.jv" + "example:cars": "nx run interpreter:run -d example/cars.jv -dg peek", + "example:gtfs": "nx run interpreter:run -d example/gtfs-static-and-rt.jv -dg peek", + "example:vehicles": "nx run interpreter:run -d -e DB_HOST=localhost -e DB_PORT=5432 -e DB_USERNAME=postgres -e DB_PASSWORD=postgres -e DB_DATABASE=postgres example/electric-vehicles.jv -dg peek" }, "private": true, "dependencies": { From be91edae95ef8bae999f037b32bda10aa62fe308 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Wed, 26 Jul 2023 16:33:07 +0200 Subject: [PATCH 15/17] added missing type to variable --- libs/extensions/std/exec/src/archive-interpreter-executor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index 537f67b72..88040b091 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -46,7 +46,7 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< 'archiveType', PrimitiveValuetypes.Text, ); - let fs; + let fs: R.Result; if (archiveType === 'zip') { fs = await this.loadZipFileToInMemoryFileSystem(archiveFile, context); From 729788b8d5bd2fe735b73c9b3a852cc914545e52 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Fri, 28 Jul 2023 12:46:01 +0200 Subject: [PATCH 16/17] added pr feedback - simplified createAndPutFile to createFileFromArchive. Moved putFile logic outside --- .../exec/src/archive-interpreter-executor.ts | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index 88040b091..7d6f6b214 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -76,9 +76,17 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< const extNameArchive = path.extname(archiveFile.name); - this.createBinaryAndPutFile(fs, archiveFile.name, archivedObject, { - extNameArchive: extNameArchive, - }); + const file = this.createFileFromArchive( + archiveFile.name, + archivedObject, + extNameArchive, + ); + + const addedFile = fs.putFile( + InMemoryFileSystem.getPathSeparator() + file.name, + file, + ); + assert(addedFile != null); return R.ok(fs); } catch (error: unknown) { @@ -100,9 +108,14 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< )) { if (!archivedObject.dir) { const content = await archivedObject.async('arraybuffer'); - this.createBinaryAndPutFile(fs, archivedObject.name, content, { - relPath: relPath, - }); + + const file = this.createFileFromArchive(archivedObject.name, content); + + const addedFile = fs.putFile( + InMemoryFileSystem.getPathSeparator() + relPath, + file, + ); + assert(addedFile != null); } } return R.ok(fs); @@ -111,13 +124,12 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< } } - private createBinaryAndPutFile( - fs: InMemoryFileSystem, + private createFileFromArchive( archiveFileName: string, content: ArrayBuffer, - options?: { extNameArchive?: string; relPath?: string }, + extNameArchive?: string, ) { - const fileName = path.basename(archiveFileName, options?.extNameArchive); + const fileName = path.basename(archiveFileName, extNameArchive); const extName = path.extname(fileName); const mimeType = @@ -125,14 +137,8 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< MimeType.APPLICATION_OCTET_STREAM; const fileExtension = inferFileExtensionFromFileExtensionString(extName) || FileExtension.NONE; - const file = new BinaryFile(fileName, fileExtension, mimeType, content); - const addedFile = fs.putFile( - InMemoryFileSystem.getPathSeparator() + - String(options?.relPath ?? fileName), - file, - ); - assert(addedFile != null); + return new BinaryFile(fileName, fileExtension, mimeType, content); } private generateErrorObject(context: ExecutionContext, error: unknown) { From 9499a52d701ba9742f5deb132a711314be1c1d87 Mon Sep 17 00:00:00 2001 From: A-M-A-X Date: Fri, 28 Jul 2023 12:58:29 +0200 Subject: [PATCH 17/17] Removed whitespaces --- libs/extensions/std/exec/src/archive-interpreter-executor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index 7d6f6b214..bf78fe65d 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -108,7 +108,7 @@ export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< )) { if (!archivedObject.dir) { const content = await archivedObject.async('arraybuffer'); - + const file = this.createFileFromArchive(archivedObject.name, content); const addedFile = fs.putFile(