Merge pull request #14 from Jisco/progress

add possibility to show progress during file parsing
Jisco · Feb 6, 2022 · 35fde02 · 35fde02
2 parents 4cebb54 + 5dac524
commit 35fde02
Show file tree

Hide file tree

Showing 8 changed files with 110 additions and 11 deletions.
diff --git a/README.md b/README.md
@@ -31,7 +31,7 @@ I am aware that there are already several parsers for the gedcom format. However
 
 ##### Via commandline
 
-Just run npx ts-node src/index.ts  with the wanted flags. Eg if you run "npm run demo:JSON" it will execute "ts-node src/index.ts --path 'examples/simpsons.get'" and will print out the Simpsons GEDCOM examplke file as JSON object in the console. With "npm run demoFile:JSON" it will do the same but prints the JSON object in a 'test.json' file.
+Just run npx ts-node src/console.ts  with the wanted flags. Eg if you run "npm run demo:JSON" it will execute "ts-node src/console.ts --path 'examples/simpsons.get'" and will print out the Simpsons GEDCOM examplke file as JSON object in the console. With "npm run demoFile:JSON" it will do the same but prints the JSON object in a 'test.json' file.
 
 | Flag             | Description                                                  |
 | ---------------- | ------------------------------------------------------------ |
@@ -40,6 +40,7 @@ Just run npx ts-node src/index.ts  with the wanted flags. Eg if you run "npm run
 | --out *xxx.json* | File path to print into                                      |
 | --path *xxx.ged* | Set the path to the GEDCOM file                              |
 | --silent         | Don't print anything to the console                          |
+| --showProgress   | Print the progress during processing the file                |
 
 ##### Via Node or JS
 

diff --git a/package.json b/package.json
@@ -8,6 +8,7 @@
     "clean": "tsc --build ./tsconfig.json --clean",
     "demo:JSON": "ts-node src/console.ts --path 'examples/simpsons.ged'",
     "demoFile:JSON": "ts-node src/console.ts --path 'examples/simpsons.ged' --out 'test.json'",
+    "demoFileWithProgress:JSON": "ts-node src/console.ts --path 'examples/royal92.ged' --out 'test.json' --showProgress",
     "test": "nyc mocha",
     "test:watch": "mocha --watch --watch-files src, tests/**/*.ts",
     "doc": "typedoc --out docs src",

diff --git a/src/ToJSON/console.ts b/src/ToJSON/console.ts
@@ -12,6 +12,12 @@ export function Convert(argv:any) {
         options.SetConfigFile(argv.opt as string);
     }
 
+    if (argv.showProgress) {
+        options.SetProgressFunction((linesCount: number, lineNumber:number) => {
+            process.stdout.write(`\rProgress: parsing line ${lineNumber} from ${linesCount}`);
+        });
+    }
+
     let silent = false;
     if (argv.silent) {
         silent = true;

diff --git a/src/ToJSON/models/Parsing.ts b/src/ToJSON/models/Parsing.ts
@@ -18,7 +18,7 @@ export default class Parsing {
             return new ParsingResult({});
         }
 
-        return ParseText(this.options.GetText(), this.options.GetConfig());
+        return ParseText(this.options.GetText(), this.options.GetConfig(), this.options.GetProgressFunction());
     }
 
     ParseTextAsync(): Promise<ParsingResult> {
@@ -29,7 +29,7 @@ export default class Parsing {
         }
 
         return new Promise<ParsingResult>((resolve, reject) => {
-            resolve(ParseText(this.options.GetText(), this.options.GetConfig()));
+            resolve(ParseText(this.options.GetText(), this.options.GetConfig(), this.options.GetProgressFunction()));
         });
     }
 
@@ -39,7 +39,7 @@ export default class Parsing {
             return;
         }
 
-        ParseFile(filePath, this.options.GetConfig(), doneCallback, errorCallback);
+        ParseFile(filePath, this.options.GetConfig(), doneCallback, errorCallback, this.options.GetProgressFunction());
     }
 
     ParseFileAsync(): Promise<ParsingResult> {

diff --git a/src/ToJSON/models/ParsingOptions.ts b/src/ToJSON/models/ParsingOptions.ts
@@ -2,6 +2,7 @@ export default class ParsingOptions {
     private text?: string;
     private filePath?: string;
     private config?: string;
+    private progressFunction?: (linesCount: number, actualLine: number) => void;
 
     SetText(text: string) {
         this.text = text;
@@ -30,4 +31,12 @@ export default class ParsingOptions {
     GetConfig() {
         return this.config ?? require('fs').readFileSync('options/version551.yaml', 'utf8');
     }
+
+    SetProgressFunction(func: (linesCount: number, actualLine: number) => void) {
+        this.progressFunction = func;
+    }
+
+    GetProgressFunction() : ((linesCount: number, actualLine: number) => void) | undefined {
+        return this.progressFunction;
+    }
 }
diff --git a/src/ToJSON/parsing/parsing.ts b/src/ToJSON/parsing/parsing.ts
@@ -20,9 +20,14 @@ let stats = new Statistics();
  *
  * @param text - The text
  * @param parsingOptions - The parsing options
+ * @param invokeProgressFunction - Set function that is called before each line, to show progress in some way
  * @returns An object which includes the parsed object and parsing statistics
 */
-export function ParseText(text?: string, parsingOptions?: string): ParsingResult {
+export function ParseText(
+    text?: string, 
+    parsingOptions?: string, 
+    invokeProgressFunction?: (linesCount: number, actualLine: number) => void): ParsingResult {
+
     if (!text || !parsingOptions){
         return new ParsingResult({});
     }
@@ -51,8 +56,12 @@ export function ParseText(text?: string, parsingOptions?: string): ParsingResult
         lineNumber++;
     };
 
-    forEach(lines, line => {
-        ProcessNewLine(lastLevel, lineNumber, line, nextLine);
+    forEach(lines, (line, index) => {
+      if (invokeProgressFunction) {
+        invokeProgressFunction(lines.length, index);
+      }
+
+      ProcessNewLine(lastLevel, lineNumber, line, nextLine);
     });
 
     let result = GetResult();
@@ -67,11 +76,47 @@ export function ParseText(text?: string, parsingOptions?: string): ParsingResult
  * @param parsingOptions - The parsing options
  * @param doneCallback - Returns the resulting object when file is readed completly
  * @param errorCallback - Returns file reading errors
+ * @param invokeProgressFunction - Set function that is called before each line, to show progress in some way
  * @returns An object which includes the parsed object and parsing statistics
 */
+
 /* istanbul ignore next */ // maybe later ;)
-export function ParseFile(path: string, parsingOptions: string, doneCallback: (result: ParsingResult) => void, errorCallback: any) {
-    let lr = new LineByLineReader(path);    
+export function ParseFile(
+  path: string, 
+  parsingOptions: string, 
+  doneCallback: (result: ParsingResult) => void, 
+  errorCallback: any,
+  invokeProgressFunction?: ((linesCount: number, actualLine: number) => void) | undefined) {
+
+    // if no progress should be shown, it is not necessary to get the line count of the file at first
+    if (!invokeProgressFunction) {
+      ExecuteParseFile(path, parsingOptions, doneCallback, errorCallback, 0);
+      return;
+    }
+
+    // read first time to get lines count
+    let linesCountLr = new LineByLineReader(path);
+    let linesCount = 0;
+    linesCountLr.on('line', function (line: any) {
+      linesCountLr.pause();
+      linesCount++;
+      linesCountLr.resume();
+    });
+
+    linesCountLr.on('end', function () {
+      ExecuteParseFile(path, parsingOptions, doneCallback, errorCallback, linesCount, invokeProgressFunction);
+    });
+}
+
+/* istanbul ignore next */ // maybe later ;)
+function ExecuteParseFile(
+  path: string, 
+  parsingOptions: string, 
+  doneCallback: (result: ParsingResult) => void, 
+  errorCallback: any,
+  linesCount: number,
+  invokeProgressFunction?: ((linesCount: number, actualLine: number) => void) | undefined) {
+    let lr = new LineByLineReader(path);
     let lastLevel = 0;
     let lineNumber = 1;
     let yamlOptions: string | object | undefined = {};
@@ -93,6 +138,10 @@ export function ParseFile(path: string, parsingOptions: string, doneCallback: (r
             lastLevel = lastLine.Level;
         }
 
+        if (invokeProgressFunction) {
+          invokeProgressFunction(linesCount, lineNumber);
+        }
+
         lineNumber++;
         lr.resume();
     };
@@ -115,7 +164,7 @@ export function ParseFile(path: string, parsingOptions: string, doneCallback: (r
         // All lines are read, file is closed now.
         doneCallback(new ParsingResult(result, stats));
     });
-}
+  }
 
 /**
  * Function that processes a text line

diff --git a/src/ToJSON/parsing/processLine.ts b/src/ToJSON/parsing/processLine.ts
@@ -15,6 +15,7 @@ import ParsingPath from '../models/ParsingPath';
 import { ClearDateTimeMergingInfos } from './parseDate';
 
 const eachDeep = require('deepdash/eachDeep');
+let parsingOptions: any;
 
 /**
  * Store for file\text processing
@@ -30,7 +31,6 @@ export function ResetProcessing() {
     parsingOptions = {};
 }
 
-let parsingOptions: any;
 export function SetParsingOptions(options: any) {
     parsingOptions = options;
 }

diff --git a/tests/ToJSON/Parsing.tests.ts b/tests/ToJSON/Parsing.tests.ts
@@ -28,4 +28,37 @@ describe('Parsing text', () => {
             expect(stats.NotParsedLinesWithoutGEDCOMTag).lengthOf(1);
         });
     });
+
+    it('With Progress', () => {
+        let testData = `
+        0 @N00010@ NOTE
+        0 TRLR`;
+
+        let options = `
+        Definition:
+        - Tag: NOTE
+          CollectAs: Notes
+          Property: Id
+        `;
+
+        let allLinesCount = 0;
+        let invokeCounter = 0;
+        let lineNumberArray: number[] = [];
+        let progressFunc = (linesCount: number, lineNumber:number) => {
+            allLinesCount = linesCount;
+            invokeCounter++;
+            lineNumberArray.push(lineNumber);
+        };
+
+        expect(ParseText(testData, options, progressFunc).Object).to.deep.equal({
+          Notes:
+          {
+            Id: "@N00010@"
+          }
+        });
+
+        expect(allLinesCount).to.equal(3);
+        expect(invokeCounter).to.equal(3);
+        expect(lineNumberArray).to.deep.equal([ 0, 1, 2 ]);
+    });
 });