NiFi: cohort workflow updated for large files.

CogStack · Feb 25, 2024 · c46b2a1 · c46b2a1
1 parent 0e4bcc5
commit c46b2a1
Show file tree

Hide file tree

Showing 2 changed files with 3,857 additions and 1,276 deletions.
diff --git a/nifi/user-scripts/cogstack_cohort_generate_data.py b/nifi/user-scripts/cogstack_cohort_generate_data.py
@@ -230,5 +230,17 @@ def process_annotation_records(annotation_records: list, _doc2ptt: dict):
 dict2json_file(ptt2dob, os.path.join(OUTPUT_FOLDER_PATH, "ptt2dob.json"))
 dict2json_file(ptt2dod, os.path.join(OUTPUT_FOLDER_PATH, "ptt2dod.json"))
 dict2json_file(ptt2age, os.path.join(OUTPUT_FOLDER_PATH, "ptt2age.json"))
-dict2json_file(cui2ptt_pos, os.path.join(OUTPUT_FOLDER_PATH, "cui2ptt_pos.jsonl"))
-dict2json_file(cui2ptt_tsp, os.path.join(OUTPUT_FOLDER_PATH, "cui2ptt_tsp.jsonl"))
+
+with open('cui2ptt_pos.jsonl', 'a', encoding='utf-8') as outfile:
+    for k,v in cui2ptt_pos.items():
+        o = {k: v}
+        json_obj = json.loads(json.dumps(o))
+        json.dump(json_obj, outfile, ensure_ascii=False, indent=None, separators=(',',':'))
+        print('', file = outfile)
+
+with open('cui2ptt_tsp.jsonl', 'a', encoding='utf-8') as outfile:
+    for k,v in cui2ptt_tsp.items():
+        o = {k: v}
+        json_obj = json.loads(json.dumps(o))
+        json.dump(json_obj, outfile, ensure_ascii=False, indent=None, separators=(',',':'))
+        print('', file = outfile)