diff --git a/nifi/user-scripts/cogstack_cohort_generate_data.py b/nifi/user-scripts/cogstack_cohort_generate_data.py
index 5760d146..22612219 100644
--- a/nifi/user-scripts/cogstack_cohort_generate_data.py
+++ b/nifi/user-scripts/cogstack_cohort_generate_data.py
@@ -230,5 +230,17 @@ def process_annotation_records(annotation_records: list, _doc2ptt: dict):
dict2json_file(ptt2dob, os.path.join(OUTPUT_FOLDER_PATH, "ptt2dob.json"))
dict2json_file(ptt2dod, os.path.join(OUTPUT_FOLDER_PATH, "ptt2dod.json"))
dict2json_file(ptt2age, os.path.join(OUTPUT_FOLDER_PATH, "ptt2age.json"))
-dict2json_file(cui2ptt_pos, os.path.join(OUTPUT_FOLDER_PATH, "cui2ptt_pos.jsonl"))
-dict2json_file(cui2ptt_tsp, os.path.join(OUTPUT_FOLDER_PATH, "cui2ptt_tsp.jsonl"))
+
+with open('cui2ptt_pos.jsonl', 'a', encoding='utf-8') as outfile:
+ for k,v in cui2ptt_pos.items():
+ o = {k: v}
+ json_obj = json.loads(json.dumps(o))
+ json.dump(json_obj, outfile, ensure_ascii=False, indent=None, separators=(',',':'))
+ print('', file = outfile)
+
+with open('cui2ptt_tsp.jsonl', 'a', encoding='utf-8') as outfile:
+ for k,v in cui2ptt_tsp.items():
+ o = {k: v}
+ json_obj = json.loads(json.dumps(o))
+ json.dump(json_obj, outfile, ensure_ascii=False, indent=None, separators=(',',':'))
+ print('', file = outfile)
diff --git a/nifi/user-templates/CogStack_Cohort_create_source_docs.xml b/nifi/user-templates/CogStack_Cohort_create_source_docs.xml
index 2e92c4b4..80ea1680 100644
--- a/nifi/user-templates/CogStack_Cohort_create_source_docs.xml
+++ b/nifi/user-templates/CogStack_Cohort_create_source_docs.xml
@@ -391,16 +391,14 @@
Exports annotations and records that are used by CogStack-Cohort.
- 24d61ddf-c4f6-3a92-0000-000000000000
+ 0191cd63-24f0-337b-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 24d61ddf-c4f6-3a92-970f-008a4fa5fbbe
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- d1e53967-c2ec-394e-0000-000000000000
- FUNNEL
- d1e53967-c2ec-394e-8527-eee1e3c8a36e
+ eb4eeb18-315c-334c-0000-000000000000
+ PROCESSOR
0 sec
1
@@ -409,7 +407,7 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- unmatched
+ matched
ba40cb68-0d02-3fdf-0000-000000000000
a4d63379-4aa7-3bb7-0000-000000000000
@@ -419,16 +417,14 @@
0
- 25971974-d027-318c-0000-000000000000
+ 0ed672f6-fca3-3ce5-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 25971974-d027-318c-8010-84551db9fa16
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- 26255a3f-b4e9-384f-0000-000000000000
+ 72a4a83e-464d-30e0-0000-000000000000
FUNNEL
- 26255a3f-b4e9-384f-9c30-d7e472d7455f
0 sec
1
@@ -437,26 +433,24 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- success
+ failure
+ unmatched
ba40cb68-0d02-3fdf-0000-000000000000
- f710ad2f-4da0-36c0-0000-000000000000
+ e9507246-cbf9-3c4a-0000-000000000000
PROCESSOR
- f710ad2f-4da0-36c0-8894-020161ab03c1
0
- 4eeb5b2f-8b38-3229-0000-000000000000
+ 0f6087cd-84dd-3777-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 4eeb5b2f-8b38-3229-a70c-b1b53f739441
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- da0f9bf1-0677-36c9-0000-000000000000
- FUNNEL
- da0f9bf1-0677-36c9-acaa-409d8700cf23
+ bfe47c8f-331e-39a4-0000-000000000000
+ PROCESSOR
0 sec
1
@@ -465,26 +459,23 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- failure
+ unmatched
ba40cb68-0d02-3fdf-0000-000000000000
- 57a57c00-01c7-32f1-0000-000000000000
+ eb4eeb18-315c-334c-0000-000000000000
PROCESSOR
- 57a57c00-01c7-32f1-9f43-abaf1f0b54e0
0
- 5b67bfec-d7ce-3981-0000-000000000000
+ 120b19fd-0c23-3f07-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 5b67bfec-d7ce-3981-af71-d4d63b079e5b
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- 5be72191-ed94-3f80-0000-000000000000
+ 1b0dc321-d6a7-3bfa-0000-000000000000
PROCESSOR
- 5be72191-ed94-3f80-90c9-1b661fd2ec2a
0 sec
1
@@ -493,26 +484,23 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- hits
+ matched
ba40cb68-0d02-3fdf-0000-000000000000
- af4ad2b1-2e82-37bf-0000-000000000000
+ e9507246-cbf9-3c4a-0000-000000000000
PROCESSOR
- af4ad2b1-2e82-37bf-a78c-dc2e865265c1
0
- 605ad7df-4119-3ed7-0000-000000000000
+ 15105083-2b5c-3325-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 605ad7df-4119-3ed7-a775-9a3bffb5ee59
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- 7dd60db1-2d31-351c-0000-000000000000
+ 6da089fa-8b72-3452-0000-000000000000
PROCESSOR
- 7dd60db1-2d31-351c-89c1-c2ed37eb4567
0 sec
1
@@ -521,19 +509,17 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- success
+ hits
ba40cb68-0d02-3fdf-0000-000000000000
- e4ad572d-99c6-335d-0000-000000000000
+ aba517db-6b48-3fa5-0000-000000000000
PROCESSOR
- e4ad572d-99c6-335d-8fee-db4e0d05ec2d
0
- 7a2d1623-f147-331b-0000-000000000000
+ 1fd47d94-a973-370b-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 7a2d1623-f147-331b-803d-de9940687cab
1 GB
10000
@@ -559,16 +545,14 @@
0
- 8401294e-2b6e-3d95-0000-000000000000
+ 205b293e-eb38-351b-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 8401294e-2b6e-3d95-9fed-90ed413856b6
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- 71bde74b-e1b9-38af-0000-000000000000
+ aba517db-6b48-3fa5-0000-000000000000
PROCESSOR
- 71bde74b-e1b9-38af-a977-c3b470f7866b
0 sec
1
@@ -580,23 +564,22 @@
matched
ba40cb68-0d02-3fdf-0000-000000000000
- a4d63379-4aa7-3bb7-0000-000000000000
+ b3ec8bc1-7402-3635-0000-000000000000
PROCESSOR
- a4d63379-4aa7-3bb7-a224-e3a995a30469
0
- 854832f1-0bf5-3903-0000-000000000000
+ 24d61ddf-c4f6-3a92-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 854832f1-0bf5-3903-9900-5d03b4057cb3
+ 24d61ddf-c4f6-3a92-970f-008a4fa5fbbe
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- a5436bd7-26bb-30bb-0000-000000000000
+ d1e53967-c2ec-394e-0000-000000000000
FUNNEL
- a5436bd7-26bb-30bb-90c8-3fff128b63fb
+ d1e53967-c2ec-394e-8527-eee1e3c8a36e
0 sec
1
@@ -608,23 +591,23 @@
unmatched
ba40cb68-0d02-3fdf-0000-000000000000
- d0734b2f-aba4-385a-0000-000000000000
+ a4d63379-4aa7-3bb7-0000-000000000000
PROCESSOR
- d0734b2f-aba4-385a-a9ac-0ef232d90725
+ a4d63379-4aa7-3bb7-a224-e3a995a30469
0
- 892dcee5-4536-3434-0000-000000000000
+ 25971974-d027-318c-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 892dcee5-4536-3434-ad34-7cb5899e0851
+ 25971974-d027-318c-8010-84551db9fa16
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- 57a57c00-01c7-32f1-0000-000000000000
- PROCESSOR
- 57a57c00-01c7-32f1-9f43-abaf1f0b54e0
+ 26255a3f-b4e9-384f-0000-000000000000
+ FUNNEL
+ 26255a3f-b4e9-384f-9c30-d7e472d7455f
0 sec
1
@@ -633,26 +616,24 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- hits
+ success
ba40cb68-0d02-3fdf-0000-000000000000
- 7eeac5f1-3266-32b5-0000-000000000000
+ f710ad2f-4da0-36c0-0000-000000000000
PROCESSOR
- 7eeac5f1-3266-32b5-b125-2e22bd9e375a
+ f710ad2f-4da0-36c0-8894-020161ab03c1
0
- ae182ee1-9da8-3034-0000-000000000000
+ 2a8ade4a-e612-379e-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- ae182ee1-9da8-3034-8ba1-36462fac9b49
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- dac27e7d-95cc-355e-0000-000000000000
+ 26be5a0a-df8b-3b57-0000-000000000000
FUNNEL
- dac27e7d-95cc-355e-bd0e-0302a8aeb005
0 sec
1
@@ -664,23 +645,22 @@
failure
ba40cb68-0d02-3fdf-0000-000000000000
- a4d63379-4aa7-3bb7-0000-000000000000
+ 19f6fabd-eb97-3345-0000-000000000000
PROCESSOR
- a4d63379-4aa7-3bb7-a224-e3a995a30469
0
- b60c6078-06b8-34de-0000-000000000000
+ 4eeb5b2f-8b38-3229-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- b60c6078-06b8-34de-b2ea-bf29c190aca2
+ 4eeb5b2f-8b38-3229-a70c-b1b53f739441
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- f055188a-7690-3670-0000-000000000000
+ da0f9bf1-0677-36c9-0000-000000000000
FUNNEL
- f055188a-7690-3670-a19f-5f66540f17bd
+ da0f9bf1-0677-36c9-acaa-409d8700cf23
0 sec
1
@@ -689,26 +669,25 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- aggregations
+ failure
ba40cb68-0d02-3fdf-0000-000000000000
- af4ad2b1-2e82-37bf-0000-000000000000
+ 57a57c00-01c7-32f1-0000-000000000000
PROCESSOR
- af4ad2b1-2e82-37bf-a78c-dc2e865265c1
+ 57a57c00-01c7-32f1-9f43-abaf1f0b54e0
0
- cd5d0e70-0fe1-3cc3-0000-000000000000
+ 5212726f-f8c7-3c44-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- cd5d0e70-0fe1-3cc3-8570-074b9dd7d325
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- a4d63379-4aa7-3bb7-0000-000000000000
+ d0734b2f-aba4-385a-0000-000000000000
PROCESSOR
- a4d63379-4aa7-3bb7-a224-e3a995a30469
+ d0734b2f-aba4-385a-a9ac-0ef232d90725
0 sec
1
@@ -717,26 +696,23 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- matched
+ unmatched
ba40cb68-0d02-3fdf-0000-000000000000
- d0734b2f-aba4-385a-0000-000000000000
+ 56582913-7c21-31da-0000-000000000000
PROCESSOR
- d0734b2f-aba4-385a-a9ac-0ef232d90725
0
- dc7a538f-8bba-31fd-0000-000000000000
+ 5568e7ae-0b0f-3338-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- dc7a538f-8bba-31fd-922a-fee798634dad
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- 3d9a51b4-f3ec-3b80-0000-000000000000
+ c87baaa6-745a-3011-0000-000000000000
FUNNEL
- 3d9a51b4-f3ec-3b80-a4df-bafa1a13dcc3
0 sec
1
@@ -747,24 +723,74 @@
failure
+ ba40cb68-0d02-3fdf-0000-000000000000
+ bfe47c8f-331e-39a4-0000-000000000000
+ PROCESSOR
+
+ 0
+
+
+ 5f12c6b3-cc4d-3157-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ b3ec8bc1-7402-3635-0000-000000000000
+ PROCESSOR
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 19f6fabd-eb97-3345-0000-000000000000
+ PROCESSOR
+
+ 0
+
+
+ 605ad7df-4119-3ed7-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 605ad7df-4119-3ed7-a775-9a3bffb5ee59
+ 1 GB
+ 10000
+
ba40cb68-0d02-3fdf-0000-000000000000
7dd60db1-2d31-351c-0000-000000000000
PROCESSOR
7dd60db1-2d31-351c-89c1-c2ed37eb4567
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ e4ad572d-99c6-335d-0000-000000000000
+ PROCESSOR
+ e4ad572d-99c6-335d-8fee-db4e0d05ec2d
0
- e2a0e013-2d90-3db8-0000-000000000000
+ 6085e270-8efd-30c6-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- e2a0e013-2d90-3db8-a5d5-3290998675ce
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- f055188a-7690-3670-0000-000000000000
- FUNNEL
- f055188a-7690-3670-a19f-5f66540f17bd
+ 19f6fabd-eb97-3345-0000-000000000000
+ PROCESSOR
0 sec
1
@@ -773,26 +799,25 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- aggregations
+ splits
ba40cb68-0d02-3fdf-0000-000000000000
- 7eeac5f1-3266-32b5-0000-000000000000
+ 1b0dc321-d6a7-3bfa-0000-000000000000
PROCESSOR
- 7eeac5f1-3266-32b5-b125-2e22bd9e375a
0
- e640123e-240d-3ef8-0000-000000000000
+ 67b00272-ece3-3efa-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- e640123e-240d-3ef8-b803-ac51b27d2d8e
+ 67b00272-ece3-3efa-a5d2-dcc53eab1215
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- da0f9bf1-0677-36c9-0000-000000000000
+ fffda609-5de0-3e17-0000-000000000000
FUNNEL
- da0f9bf1-0677-36c9-acaa-409d8700cf23
+ fffda609-5de0-3e17-8d7b-c0ce44492285
0 sec
1
@@ -804,23 +829,21 @@
failure
ba40cb68-0d02-3fdf-0000-000000000000
- 5be72191-ed94-3f80-0000-000000000000
+ c08f004a-29a3-33ef-0000-000000000000
PROCESSOR
- 5be72191-ed94-3f80-90c9-1b661fd2ec2a
+ c08f004a-29a3-33ef-8c29-9b70540173cc
0
- ebd4ce54-a855-3e29-0000-000000000000
+ 79364360-9ad9-3049-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- ebd4ce54-a855-3e29-88b3-32c18f7df925
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- d0734b2f-aba4-385a-0000-000000000000
+ bfe47c8f-331e-39a4-0000-000000000000
PROCESSOR
- d0734b2f-aba4-385a-a9ac-0ef232d90725
0 sec
1
@@ -829,26 +852,49 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- merged
ba40cb68-0d02-3fdf-0000-000000000000
- 57a57c00-01c7-32f1-0000-000000000000
+ 3d9a51b4-f3ec-3b80-0000-000000000000
+ FUNNEL
+ 3d9a51b4-f3ec-3b80-a4df-bafa1a13dcc3
+
+ 0
+
+
+ 79b60de4-815c-3c57-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ a4d63379-4aa7-3bb7-0000-000000000000
+ PROCESSOR
+ a4d63379-4aa7-3bb7-a224-e3a995a30469
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 3a440e2f-7973-36a0-0000-000000000000
PROCESSOR
- 57a57c00-01c7-32f1-9f43-abaf1f0b54e0
0
- f6e673fc-fc44-3d5f-0000-000000000000
+ 7aa3f1f6-cdb2-3347-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- f6e673fc-fc44-3d5f-b27a-c3cb53a3beb7
1 GB
10000
ba40cb68-0d02-3fdf-0000-000000000000
- eea2cd33-d1a9-37a1-0000-000000000000
+ a481ce78-a522-3484-0000-000000000000
FUNNEL
- eea2cd33-d1a9-37a1-bcd8-87b2c2f027a4
0 sec
1
@@ -860,23 +906,49 @@
failure
ba40cb68-0d02-3fdf-0000-000000000000
- d0734b2f-aba4-385a-0000-000000000000
+ 2c150f1f-1273-3eab-0000-000000000000
PROCESSOR
- d0734b2f-aba4-385a-a9ac-0ef232d90725
0
- f8310f1a-40c4-39c7-0000-000000000000
+ 854832f1-0bf5-3903-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- f8310f1a-40c4-39c7-a9c1-7921e62f6044
+ 854832f1-0bf5-3903-9900-5d03b4057cb3
1 GB
10000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ a5436bd7-26bb-30bb-0000-000000000000
+ FUNNEL
+ a5436bd7-26bb-30bb-90c8-3fff128b63fb
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ unmatched
+
ba40cb68-0d02-3fdf-0000-000000000000
d0734b2f-aba4-385a-0000-000000000000
PROCESSOR
d0734b2f-aba4-385a-a9ac-0ef232d90725
+
+ 0
+
+
+ 866bd4e1-ecc6-3fb4-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 57a57c00-01c7-32f1-0000-000000000000
+ PROCESSOR
+ 57a57c00-01c7-32f1-9f43-abaf1f0b54e0
0 sec
1
@@ -885,501 +957,2934 @@
LOAD_BALANCE_NOT_CONFIGURED
DO_NOT_LOAD_BALANCE
- merged
+ success
ba40cb68-0d02-3fdf-0000-000000000000
- 5be72191-ed94-3f80-0000-000000000000
+ 6da089fa-8b72-3452-0000-000000000000
PROCESSOR
- 5be72191-ed94-3f80-90c9-1b661fd2ec2a
0
-
- 63f66b1f-f920-3c02-0000-000000000000
+
+ 8b0255b5-7983-38db-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- 63f66b1f-f920-3c02-97b9-15ff08868d7a
- WARN
-
- nifi-record-serialization-services-nar
- org.apache.nifi
- 1.24.0
-
-
-
- Schema Write Strategy
-
-Schema Write Strategy
-
-
-
- schema-cache
-
-org.apache.nifi.serialization.RecordSchemaCacheService
-schema-cache
-
-
-
- schema-protocol-version
-
-
- hwx-schema-ref-attributes
- hwx-content-encoded-schema
- Schema Write Strategy
-
-schema-protocol-version
-
-
-
- schema-access-strategy
-
-schema-access-strategy
-
-
-
- schema-registry
-
-
- confluent-encoded
- schema-name
- hwx-schema-ref-attributes
- hwx-content-encoded-schema
- schema-access-strategy
-
-org.apache.nifi.schemaregistry.services.SchemaRegistry
-schema-registry
-
-
-
- schema-name
-
-
- schema-name
- schema-access-strategy
-
-schema-name
-
-
-
- schema-version
-
-
- schema-name
- schema-access-strategy
-
-schema-version
-
-
-
- schema-branch
-
-
- schema-name
- schema-access-strategy
-
-schema-branch
-
-
-
- schema-text
-
-
- schema-text-property
- schema-access-strategy
-
-schema-text
-
-
-
- Date Format
-
-Date Format
-
-
-
- Time Format
-
-Time Format
-
-
-
- Timestamp Format
-
-Timestamp Format
-
-
-
- Pretty Print JSON
-
-Pretty Print JSON
-
-
-
- suppress-nulls
-
-suppress-nulls
-
-
-
- output-grouping
-
-output-grouping
-
-
-
- compression-format
-
-compression-format
-
-
-
- compression-level
-
-
- gzip
- compression-format
-
-compression-level
-
-
-
- JsonRecordSetWriter
- false
-
-
- Schema Write Strategy
- no-schema
-
-
- schema-cache
-
-
- schema-protocol-version
- 1
-
-
- schema-access-strategy
- inherit-record-schema
-
-
- schema-registry
-
-
- schema-name
- ${schema.name}
-
-
- schema-version
-
-
- schema-branch
-
-
- schema-text
- ${avro.schema}
-
-
- Date Format
-
-
- Time Format
-
-
- Timestamp Format
-
-
- Pretty Print JSON
- false
-
-
- suppress-nulls
- never-suppress
-
-
- output-grouping
- output-array
-
-
- compression-format
- none
-
-
- compression-level
- 1
-
-
- ENABLED
- org.apache.nifi.json.JsonRecordSetWriter
-
-
- 26255a3f-b4e9-384f-0000-000000000000
- ba40cb68-0d02-3fdf-0000-000000000000
-
- -2400.0
- 1384.0
-
- 26255a3f-b4e9-384f-9c30-d7e472d7455f
-
-
- 3d9a51b4-f3ec-3b80-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 56582913-7c21-31da-0000-000000000000
+ PROCESSOR
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ c08f004a-29a3-33ef-0000-000000000000
+ PROCESSOR
+ c08f004a-29a3-33ef-8c29-9b70540173cc
+
+ 0
+
+
+ 8bddb25e-705f-36c5-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -760.0
- 2040.0
-
- 3d9a51b4-f3ec-3b80-a4df-bafa1a13dcc3
-
-
- a5436bd7-26bb-30bb-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ c02113ea-4915-39a6-0000-000000000000
+ FUNNEL
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ failure
+ unmatched
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ b3ec8bc1-7402-3635-0000-000000000000
+ PROCESSOR
+
+ 0
+
+
+ 94d35ecb-f4f5-390e-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -808.0
- 1280.0
-
- a5436bd7-26bb-30bb-90c8-3fff128b63fb
-
-
- d1e53967-c2ec-394e-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 7dd60db1-2d31-351c-0000-000000000000
+ PROCESSOR
+ 7dd60db1-2d31-351c-89c1-c2ed37eb4567
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 8990354d-6606-3294-0000-000000000000
+ PROCESSOR
+
+ 0
+
+
+ a5d103fd-4503-304b-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -808.0
- 1424.0
-
- d1e53967-c2ec-394e-8527-eee1e3c8a36e
-
-
- da0f9bf1-0677-36c9-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ e7943295-e173-3870-0000-000000000000
+ PROCESSOR
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ bfe47c8f-331e-39a4-0000-000000000000
+ PROCESSOR
+
+ 0
+
+
+ ae182ee1-9da8-3034-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -1240.0
- 952.0
-
- da0f9bf1-0677-36c9-acaa-409d8700cf23
-
-
- dac27e7d-95cc-355e-0000-000000000000
+ ae182ee1-9da8-3034-8ba1-36462fac9b49
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ dac27e7d-95cc-355e-0000-000000000000
+ FUNNEL
+ dac27e7d-95cc-355e-bd0e-0302a8aeb005
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ failure
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ a4d63379-4aa7-3bb7-0000-000000000000
+ PROCESSOR
+ a4d63379-4aa7-3bb7-a224-e3a995a30469
+
+ 0
+
+
+ b38a3841-5a8e-375d-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -808.0
- 1496.0
-
- dac27e7d-95cc-355e-bd0e-0302a8aeb005
-
-
- eea2cd33-d1a9-37a1-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 2c150f1f-1273-3eab-0000-000000000000
+ PROCESSOR
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ ef208086-8b96-38e5-0000-000000000000
+ PROCESSOR
+
+ 0
+
+
+ b60c6078-06b8-34de-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -808.0
- 1216.0
-
- eea2cd33-d1a9-37a1-bcd8-87b2c2f027a4
-
-
- f055188a-7690-3670-0000-000000000000
+ b60c6078-06b8-34de-b2ea-bf29c190aca2
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ f055188a-7690-3670-0000-000000000000
+ FUNNEL
+ f055188a-7690-3670-a19f-5f66540f17bd
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ aggregations
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ af4ad2b1-2e82-37bf-0000-000000000000
+ PROCESSOR
+ af4ad2b1-2e82-37bf-a78c-dc2e865265c1
+
+ 0
+
+
+ bae81a0d-f0f6-3d62-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -1240.0
- 720.0
-
- f055188a-7690-3670-a19f-5f66540f17bd
-
-
- 025c1003-a0a3-3908-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 5be72191-ed94-3f80-0000-000000000000
+ PROCESSOR
+ 5be72191-ed94-3f80-90c9-1b661fd2ec2a
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ success
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ f0f64682-161c-38cd-0000-000000000000
+ PROCESSOR
+
+ 0
+
+
+ c3b3f49f-2cc1-35e6-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -888.0
- 640.0
-
- 025c1003-a0a3-3908-a1ef-dbf84e18a100
- 32.0
-
-
- 352.0
- 4
-
-
- 0f4b7612-1522-301e-0000-000000000000
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 2fe3e4dc-fc87-36bb-0000-000000000000
+ FUNNEL
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ failure
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ aba517db-6b48-3fa5-0000-000000000000
+ PROCESSOR
+
+ 0
+
+
+ c531e317-4a31-35c2-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -2016.0
- 464.0
-
- 0f4b7612-1522-301e-9075-c7591900c8fe
- 1704.0
-
- 1696.0
+ 1 GB
+ 10000
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ f7e901b4-6170-3e9e-0000-000000000000
+ FUNNEL
+
+ 0 sec
+ 1
+ DO_NOT_COMPRESS
+
+ LOAD_BALANCE_NOT_CONFIGURED
+ DO_NOT_LOAD_BALANCE
+
+ failure
+
+ ba40cb68-0d02-3fdf-0000-000000000000
+ 8990354d-6606-3294-0000-000000000000
+ PROCESSOR
+
0
-
-
- 21212aa8-6a49-330c-0000-000000000000
+
+
+ c915c671-9dff-3ee1-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-
- -2688.0
- 496.0
-
- 21212aa8-6a49-330c-8a6c-42f462837323
- 128.0
-
-
+ 576.0
+ 14
+
+
+ 4723675e-3311-3e39-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -888.0
+ 840.0
+
+ 4723675e-3311-3e39-aea3-9582ffcd43ff
+ 64.0
+
+
+ 352.0
+ 10
+
+
+ 50ad6304-25f5-3270-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1920.0
+ 864.0
+
+ 40.0
+
+
+ 352.0
+ 0
+
+
+ 5477f249-906f-3ca1-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -3552.0
+ 1264.0
+
+ 5477f249-906f-3ca1-a05c-caeb00e05e3e
+ 352.0
+
+
- 576.0
- 6
+ 352.0
+ 15
- 4723675e-3311-3e39-0000-000000000000
+ 635d2ddc-3a65-3ee6-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1432.0
+ 2240.0
+
+ 40.0
+
+
+ 208.0
+ 0
+
+
+ 728caedf-c2f0-38b7-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -496.0
+ 1368.0
+
+ 48.0
+
+
+ 296.0
+ 0
+
+
+ 860711d2-08bc-37b8-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1920.0
+ 608.0
+
+ 860711d2-08bc-37b8-8b6d-b8c1ec2bd092
+ 64.0
+
+
+ 352.0
+ 12
+
+
+ d02a7343-481b-3067-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ 168.0
+ 2096.0
+
+ 128.0
+
+
+ 192.0
+ 0
+
+
+ df5b654c-1c81-3d42-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -2808.0
+ 2296.0
+
+ 48.0
+
+
+ 216.0
+ 0
+
+
+ fdc8b666-d08d-3712-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1688.0
+ 456.0
+
+ fdc8b666-d08d-3712-80d2-163e3c2f2a05
+ 40.0
+
+
+ 904.0
+ 13
+
+
+ 19f6fabd-eb97-3345-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -192.0
+ 1704.0
+
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Replacement Strategy
+
+ Replacement Strategy
+
+
+
+Regular Expression
+
+
+ Regex Replace
+ Literal Replace
+ Replacement Strategy
+
+ Regular Expression
+
+
+
+Replacement Value
+
+
+ Prepend
+ Regex Replace
+ Always Replace
+ Append
+ Literal Replace
+ Replacement Strategy
+
+ Replacement Value
+
+
+
+Text to Prepend
+
+
+ Surround
+ Replacement Strategy
+
+ Text to Prepend
+
+
+
+Text to Append
+
+
+ Surround
+ Replacement Strategy
+
+ Text to Append
+
+
+
+Character Set
+
+ Character Set
+
+
+
+Maximum Buffer Size
+
+ Maximum Buffer Size
+
+
+
+Evaluation Mode
+
+ Evaluation Mode
+
+
+
+Line-by-Line Evaluation Mode
+
+ Line-by-Line Evaluation Mode
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Replacement Strategy
+Regex Replace
+
+
+Regular Expression
+(\]|\[)
+
+
+Replacement Value
+
+
+
+Text to Prepend
+
+
+Text to Append
+
+
+Character Set
+UTF-8
+
+
+Maximum Buffer Size
+1 MB
+
+
+Evaluation Mode
+Entire text
+
+
+Line-by-Line Evaluation Mode
+All
+
+
+ 10
+ 25
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ ReplaceText-removeBrackets
+
+ false
+ failure
+ false
+
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.ReplaceText
+
+
+ 1b0dc321-d6a7-3bfa-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -192.0
+ 1912.0
+
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Byte Sequence Format
+
+ Byte Sequence Format
+
+
+
+Byte Sequence
+
+ Byte Sequence
+
+
+
+Keep Byte Sequence
+
+ Keep Byte Sequence
+
+
+
+Byte Sequence Location
+
+ Byte Sequence Location
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Byte Sequence Format
+Text
+
+
+Byte Sequence
+,
+
+
+Keep Byte Sequence
+false
+
+
+Byte Sequence Location
+Trailing
+
+
+ 10
+ 0
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ SplitContent
+
+ true
+ original
+ false
+
+
+ false
+ splits
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.SplitContent
+
+
+ 2c150f1f-1273-3eab-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -2584.0
+ 2296.0
+
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Merge Strategy
+
+ Merge Strategy
+
+
+
+Merge Format
+
+ Merge Format
+
+
+
+Attribute Strategy
+
+ Attribute Strategy
+
+
+
+Correlation Attribute Name
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Correlation Attribute Name
+
+
+
+mergecontent-metadata-strategy
+
+
+ Avro
+ Merge Format
+
+ mergecontent-metadata-strategy
+
+
+
+Minimum Number of Entries
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Minimum Number of Entries
+
+
+
+Maximum Number of Entries
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Maximum Number of Entries
+
+
+
+Minimum Group Size
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Minimum Group Size
+
+
+
+Maximum Group Size
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Maximum Group Size
+
+
+
+Max Bin Age
+
+ Max Bin Age
+
+
+
+Maximum number of Bins
+
+ Maximum number of Bins
+
+
+
+Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Delimiter Strategy
+
+
+
+Header File
+
+
+ Filename
+ Text
+ Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Header File
+
+
+
+Footer File
+
+
+ Filename
+ Text
+ Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Footer File
+
+
+
+Demarcator File
+
+
+ Filename
+ Text
+ Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Demarcator File
+
+
+
+Compression Level
+
+
+ ZIP
+ Merge Format
+
+ Compression Level
+
+
+
+Keep Path
+
+
+ ZIP
+ TAR
+ Merge Format
+
+ Keep Path
+
+
+
+Tar Modified Time
+
+
+ TAR
+ Merge Format
+
+ Tar Modified Time
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Merge Strategy
+Bin-Packing Algorithm
+
+
+Merge Format
+Binary Concatenation
+
+
+Attribute Strategy
+Keep All Unique Attributes
+
+
+Correlation Attribute Name
+timestamp
+
+
+mergecontent-metadata-strategy
+Do Not Merge Uncommon Metadata
+
+
+Minimum Number of Entries
+10
+
+
+Maximum Number of Entries
+100
+
+
+Minimum Group Size
+0 B
+
+
+Maximum Group Size
+
+
+Max Bin Age
+
+
+Maximum number of Bins
+1
+
+
+Delimiter Strategy
+Do Not Use Delimiters
+
+
+Header File
+
+
+Footer File
+
+
+Demarcator File
+
+
+Compression Level
+1
+
+
+Keep Path
+false
+
+
+Tar Modified Time
+${file.lastModifiedTime}
+
+
+ 10
+ 0
+ 1 min
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ MergeContent
+
+ false
+ failure
+ false
+
+
+ false
+ merged
+ false
+
+
+ true
+ original
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.MergeContent
+
+
+ 3a440e2f-7973-36a0-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1400.0
+ 1656.0
+
+
+ nifi-update-attribute-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Delete Attributes Expression
+
+ Delete Attributes Expression
+
+
+
+Store State
+
+ Store State
+
+
+
+Stateful Variables Initial Value
+
+ Stateful Variables Initial Value
+
+
+
+canonical-value-lookup-cache-size
+
+ canonical-value-lookup-cache-size
+
+
+
+index_name
+
+ index_name
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Delete Attributes Expression
+
+
+Store State
+Do not store state
+
+
+Stateful Variables Initial Value
+
+
+canonical-value-lookup-cache-size
+100
+
+
+index_name
+${output_filename}
+
+
+ 10
+ 25
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ UpdateAttribute-addIndexName
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.attributes.UpdateAttribute
+
+
+ 56582913-7c21-31da-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1400.0
+ 1408.0
+
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Routing Strategy
+
+ Routing Strategy
+
+
+
+is_annotation_index
+
+ is_annotation_index
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Routing Strategy
+Route to 'match' if any matches
+
+
+is_annotation_index
+${is_annotation_index:notNull()}
+
+
+ 10
+ 25
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ RouteOnAttribute-checkIfFlowFileIsAnnotation
+
+ false
+ matched
+ false
+
+
+ false
+ unmatched
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.RouteOnAttribute
+
+
+ 57a57c00-01c7-32f1-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -888.0
+ 912.0
+
+ 57a57c00-01c7-32f1-9f43-abaf1f0b54e0
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Merge Strategy
+
+ Merge Strategy
+
+
+
+Merge Format
+
+ Merge Format
+
+
+
+Attribute Strategy
+
+ Attribute Strategy
+
+
+
+Correlation Attribute Name
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Correlation Attribute Name
+
+
+
+mergecontent-metadata-strategy
+
+
+ Avro
+ Merge Format
+
+ mergecontent-metadata-strategy
+
+
+
+Minimum Number of Entries
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Minimum Number of Entries
+
+
+
+Maximum Number of Entries
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Maximum Number of Entries
+
+
+
+Minimum Group Size
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Minimum Group Size
+
+
+
+Maximum Group Size
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Maximum Group Size
+
+
+
+Max Bin Age
+
+ Max Bin Age
+
+
+
+Maximum number of Bins
+
+ Maximum number of Bins
+
+
+
+Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Delimiter Strategy
+
+
+
+Header File
+
+
+ Filename
+ Text
+ Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Header File
+
+
+
+Footer File
+
+
+ Filename
+ Text
+ Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Footer File
+
+
+
+Demarcator File
+
+
+ Filename
+ Text
+ Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Demarcator File
+
+
+
+Compression Level
+
+
+ ZIP
+ Merge Format
+
+ Compression Level
+
+
+
+Keep Path
+
+
+ ZIP
+ TAR
+ Merge Format
+
+ Keep Path
+
+
+
+Tar Modified Time
+
+
+ TAR
+ Merge Format
+
+ Tar Modified Time
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Merge Strategy
+Bin-Packing Algorithm
+
+
+Merge Format
+Binary Concatenation
+
+
+Attribute Strategy
+Keep All Unique Attributes
+
+
+Correlation Attribute Name
+timestamp
+
+
+mergecontent-metadata-strategy
+Do Not Merge Uncommon Metadata
+
+
+Minimum Number of Entries
+1
+
+
+Maximum Number of Entries
+1000
+
+
+Minimum Group Size
+0 B
+
+
+Maximum Group Size
+
+
+Max Bin Age
+
+
+Maximum number of Bins
+5
+
+
+Delimiter Strategy
+Do Not Use Delimiters
+
+
+Header File
+
+
+Footer File
+
+
+Demarcator File
+
+
+Compression Level
+1
+
+
+Keep Path
+false
+
+
+Tar Modified Time
+${file.lastModifiedTime}
+
+
+ 10
+ 0
+ 10 min
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ MergeContent
+
+ false
+ failure
+ false
+
+
+ false
+ merged
+ false
+
+
+ true
+ original
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.MergeContent
+
+
+ 5be72191-ed94-3f80-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1920.0
+ 912.0
+
+ 5be72191-ed94-3f80-90c9-1b661fd2ec2a
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Merge Strategy
+
+ Merge Strategy
+
+
+
+Merge Format
+
+ Merge Format
+
+
+
+Attribute Strategy
+
+ Attribute Strategy
+
+
+
+Correlation Attribute Name
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Correlation Attribute Name
+
+
+
+mergecontent-metadata-strategy
+
+
+ Avro
+ Merge Format
+
+ mergecontent-metadata-strategy
+
+
+
+Minimum Number of Entries
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Minimum Number of Entries
+
+
+
+Maximum Number of Entries
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Maximum Number of Entries
+
+
+
+Minimum Group Size
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Minimum Group Size
+
+
+
+Maximum Group Size
+
+
+ Bin-Packing Algorithm
+ Merge Strategy
+
+ Maximum Group Size
+
+
+
+Max Bin Age
+
+ Max Bin Age
+
+
+
+Maximum number of Bins
+
+ Maximum number of Bins
+
+
+
+Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Delimiter Strategy
+
+
+
+Header File
+
+
+ Filename
+ Text
+ Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Header File
+
+
+
+Footer File
+
+
+ Filename
+ Text
+ Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Footer File
+
+
+
+Demarcator File
+
+
+ Filename
+ Text
+ Delimiter Strategy
+
+
+ Binary Concatenation
+ Merge Format
+
+ Demarcator File
+
+
+
+Compression Level
+
+
+ ZIP
+ Merge Format
+
+ Compression Level
+
+
+
+Keep Path
+
+
+ ZIP
+ TAR
+ Merge Format
+
+ Keep Path
+
+
+
+Tar Modified Time
+
+
+ TAR
+ Merge Format
+
+ Tar Modified Time
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Merge Strategy
+Bin-Packing Algorithm
+
+
+Merge Format
+Binary Concatenation
+
+
+Attribute Strategy
+Keep Only Common Attributes
+
+
+Correlation Attribute Name
+
+
+mergecontent-metadata-strategy
+Do Not Merge Uncommon Metadata
+
+
+Minimum Number of Entries
+1
+
+
+Maximum Number of Entries
+1
+
+
+Minimum Group Size
+0 B
+
+
+Maximum Group Size
+
+
+Max Bin Age
+
+
+Maximum number of Bins
+1
+
+
+Delimiter Strategy
+Do Not Use Delimiters
+
+
+Header File
+
+
+Footer File
+
+
+Demarcator File
+
+
+Compression Level
+1
+
+
+Keep Path
+false
+
+
+Tar Modified Time
+${file.lastModifiedTime}
+
+
+ 10
+ 0
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ MergeContent
+
+ false
+ failure
+ false
+
+
+ false
+ merged
+ false
+
+
+ true
+ original
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.MergeContent
+
+
+ 6da089fa-8b72-3452-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -528.0
+ -192.0
912.0
- 4723675e-3311-3e39-aea3-9582ffcd43ff
- 128.0
-
-
- 176.0
- 3
-
-
- 5477f249-906f-3ca1-0000-000000000000
- ba40cb68-0d02-3fdf-0000-000000000000
-
- -2552.0
- 816.0
-
- 5477f249-906f-3ca1-a05c-caeb00e05e3e
- 352.0
-
-
- 352.0
- 0
-
-
- 860711d2-08bc-37b8-0000-000000000000
+
+ nifi-update-attribute-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Delete Attributes Expression
+
+ Delete Attributes Expression
+
+
+
+Store State
+
+ Store State
+
+
+
+Stateful Variables Initial Value
+
+ Stateful Variables Initial Value
+
+
+
+canonical-value-lookup-cache-size
+
+ canonical-value-lookup-cache-size
+
+
+
+is_annotation_index
+
+ is_annotation_index
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Delete Attributes Expression
+
+
+Store State
+Do not store state
+
+
+Stateful Variables Initial Value
+
+
+canonical-value-lookup-cache-size
+100
+
+
+is_annotation_index
+True
+
+
+ 10
+ 25
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ UpdateAttribute-addAnnIndexFlag
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.attributes.UpdateAttribute
+
+
+ 71bde74b-e1b9-38af-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1920.0
- 608.0
+ -792.0
+ 2304.0
- 860711d2-08bc-37b8-8b6d-b8c1ec2bd092
- 64.0
-
-
- 352.0
- 1
-
-
- eba8b945-5e84-301e-0000-000000000000
+ 71bde74b-e1b9-38af-a977-c3b470f7866b
+
+ nifi-update-attribute-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Delete Attributes Expression
+
+ Delete Attributes Expression
+
+
+
+Store State
+
+ Store State
+
+
+
+Stateful Variables Initial Value
+
+ Stateful Variables Initial Value
+
+
+
+canonical-value-lookup-cache-size
+
+ canonical-value-lookup-cache-size
+
+
+
+timestamp
+
+ timestamp
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Delete Attributes Expression
+
+
+Store State
+Do not store state
+
+
+Stateful Variables Initial Value
+
+
+canonical-value-lookup-cache-size
+100
+
+
+timestamp
+${now():format('yyyy_MM_dd_HH_mm_ss_SSS')}
+
+
+ 10
+ 25
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ UpdateAttribute-addTimestamp
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.attributes.UpdateAttribute
+
+
+ 7dd60db1-2d31-351c-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -2792.0
- 464.0
+ -1984.0
+ 2488.0
- eba8b945-5e84-301e-afff-d3142434c8fa
- 1704.0
-
- 760.0
- 0
-
-
- fdc8b666-d08d-3712-0000-000000000000
+ 7dd60db1-2d31-351c-89c1-c2ed37eb4567
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Directory
+
+ Directory
+
+
+
+Conflict Resolution Strategy
+
+ Conflict Resolution Strategy
+
+
+
+Create Missing Directories
+
+ Create Missing Directories
+
+
+
+Maximum File Count
+
+ Maximum File Count
+
+
+
+Last Modified Time
+
+ Last Modified Time
+
+
+
+Permissions
+
+ Permissions
+
+
+
+Owner
+
+ Owner
+
+
+
+Group
+
+ Group
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Directory
+/opt/data/cogstack-cohort/
+
+
+Conflict Resolution Strategy
+fail
+
+
+Create Missing Directories
+true
+
+
+Maximum File Count
+
+
+Last Modified Time
+
+
+Permissions
+
+
+Owner
+
+
+Group
+
+
+ 10
+ 0
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ PutFile
+
+ false
+ failure
+ false
+
+
+ true
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.PutFile
+
+
+ 8990354d-6606-3294-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1640.0
- 504.0
+ -2584.0
+ 2488.0
- fdc8b666-d08d-3712-80d2-163e3c2f2a05
- 40.0
-
-
- 904.0
- 5
-
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Replacement Strategy
+
+ Replacement Strategy
+
+
+
+Regular Expression
+
+
+ Regex Replace
+ Literal Replace
+ Replacement Strategy
+
+ Regular Expression
+
+
+
+Replacement Value
+
+
+ Prepend
+ Regex Replace
+ Always Replace
+ Append
+ Literal Replace
+ Replacement Strategy
+
+ Replacement Value
+
+
+
+Text to Prepend
+
+
+ Surround
+ Replacement Strategy
+
+ Text to Prepend
+
+
+
+Text to Append
+
+
+ Surround
+ Replacement Strategy
+
+ Text to Append
+
+
+
+Character Set
+
+ Character Set
+
+
+
+Maximum Buffer Size
+
+ Maximum Buffer Size
+
+
+
+Evaluation Mode
+
+ Evaluation Mode
+
+
+
+Line-by-Line Evaluation Mode
+
+ Line-by-Line Evaluation Mode
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Replacement Strategy
+Literal Replace
+
+
+Regular Expression
+][
+
+
+Replacement Value
+,
+
+
+Text to Prepend
+
+
+Text to Append
+
+
+Character Set
+UTF-8
+
+
+Maximum Buffer Size
+1024 MB
+
+
+Evaluation Mode
+Line-by-Line
+
+
+Line-by-Line Evaluation Mode
+All
+
+
+ 10
+ 25
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ ReplaceText
+
+ false
+ failure
+ false
+
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.ReplaceText
+
- 57a57c00-01c7-32f1-0000-000000000000
+ a4d63379-4aa7-3bb7-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -888.0
- 912.0
+ -1400.0
+ 1880.0
- 57a57c00-01c7-32f1-9f43-abaf1f0b54e0
+ a4d63379-4aa7-3bb7-a224-e3a995a30469
nifi-standard-nar
org.apache.nifi
@@ -1392,179 +3897,246 @@ Also the change index name.
1
-Merge Strategy
+Destination
- Merge Strategy
+ Destination
-Merge Format
+Return Type
- Merge Format
+ Return Type
-Attribute Strategy
+Path Not Found Behavior
- Attribute Strategy
+ Path Not Found Behavior
-Correlation Attribute Name
+Null Value Representation
+
+ Null Value Representation
+
+
+
+Max String Length
+
+ Max String Length
+
+
+
+*._source
+
+ *._source
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Destination
+flowfile-content
+
+
+Return Type
+json
+
+
+Path Not Found Behavior
+warn
+
+
+Null Value Representation
+empty string
+
+
+Max String Length
+100 MB
+
+
+*._source
+*._source
+
+
+ 10
+ 0
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ EvaluateJsonPath-grabSourceField
+
+ false
+ failure
+ false
+
+
+ false
+ matched
+ false
+
+
+ false
+ unmatched
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.EvaluateJsonPath
+
+
+ aba517db-6b48-3fa5-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -192.0
+ 1328.0
+
+
+ nifi-elasticsearch-restapi-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+el-rest-query-definition-style
+
+ el-rest-query-definition-style
+
+
+
+el-rest-query
- Bin-Packing Algorithm
- Merge Strategy
+ full
+ el-rest-query-definition-style
- Correlation Attribute Name
+ el-rest-query
-mergecontent-metadata-strategy
+el-rest-query-clause
- Avro
- Merge Format
+ build
+ el-rest-query-definition-style
- mergecontent-metadata-strategy
+ el-rest-query-clause
-Minimum Number of Entries
+es-rest-size
- Bin-Packing Algorithm
- Merge Strategy
+ build
+ el-rest-query-definition-style
- Minimum Number of Entries
+ es-rest-size
-Maximum Number of Entries
+es-rest-query-sort
- Bin-Packing Algorithm
- Merge Strategy
+ build
+ el-rest-query-definition-style
- Maximum Number of Entries
+ es-rest-query-sort
-Minimum Group Size
+es-rest-query-aggs
- Bin-Packing Algorithm
- Merge Strategy
+ build
+ el-rest-query-definition-style
- Minimum Group Size
+ es-rest-query-aggs
-Maximum Group Size
+es-rest-query-fields
- Bin-Packing Algorithm
- Merge Strategy
+ build
+ el-rest-query-definition-style
- Maximum Group Size
+ es-rest-query-fields
-Max Bin Age
+es-rest-query-script-fields
- Max Bin Age
+
+ build
+ el-rest-query-definition-style
+
+ es-rest-query-script-fields
-Maximum number of Bins
+el-query-attribute
- Maximum number of Bins
+ el-query-attribute
-Delimiter Strategy
+el-rest-fetch-index
-
- Binary Concatenation
- Merge Format
-
- Delimiter Strategy
+ el-rest-fetch-index
-Header File
+el-rest-type
-
- Filename
- Text
- Delimiter Strategy
-
-
- Binary Concatenation
- Merge Format
-
- Header File
+ el-rest-type
-Footer File
+el-rest-client-service
-
- Filename
- Text
- Delimiter Strategy
-
-
- Binary Concatenation
- Merge Format
-
- Footer File
+ org.apache.nifi.elasticsearch.ElasticSearchClientService
+ el-rest-client-service
-Demarcator File
+el-rest-split-up-hits
-
- Filename
- Text
- Delimiter Strategy
-
-
- Binary Concatenation
- Merge Format
-
- Demarcator File
+ el-rest-split-up-hits
-Compression Level
+el-rest-format-hits
-
- ZIP
- Merge Format
-
- Compression Level
+ el-rest-format-hits
-Keep Path
+el-rest-split-up-aggregations
-
- ZIP
- TAR
- Merge Format
-
- Keep Path
+ el-rest-split-up-aggregations
-Tar Modified Time
+el-rest-format-aggregations
-
- TAR
- Merge Format
-
- Tar Modified Time
+ el-rest-format-aggregations
+
+
+
+el-rest-output-no-hits
+
+ el-rest-output-no-hits
@@ -1574,70 +4146,69 @@ Also the change index name.
30 sec
-Merge Strategy
-Bin-Packing Algorithm
-
-
-Merge Format
-Binary Concatenation
+el-rest-query-definition-style
+full
-Attribute Strategy
-Keep All Unique Attributes
+el-rest-query
+{
+ "size": 10000,
+ "query": { "match": { "${document_id_field_name:prepend("meta."):toString()}": "${document_id:toString()}" } },
+ "_source": ["meta.docid", "nlp.cui", "nlp.meta_anns.*"],
+ "sort": [ { "timestamp": "asc" } ]
+}
-Correlation Attribute Name
+el-rest-query-clause
-mergecontent-metadata-strategy
-Do Not Merge Uncommon Metadata
+es-rest-size
-Minimum Number of Entries
-50
+es-rest-query-sort
-Maximum Number of Entries
-100
+es-rest-query-aggs
-Minimum Group Size
-0 B
+es-rest-query-fields
-Maximum Group Size
+es-rest-query-script-fields
-Max Bin Age
+el-query-attribute
-Maximum number of Bins
-1
+el-rest-fetch-index
+${annotations_index_name}
-Delimiter Strategy
-Do Not Use Delimiters
+el-rest-type
-Header File
+el-rest-client-service
+944263c2-7443-35ce-0000-000000000000
-Footer File
+el-rest-split-up-hits
+splitUp-no
-Demarcator File
+el-rest-format-hits
+FULL
-Compression Level
-1
+el-rest-split-up-aggregations
+splitUp-no
-Keep Path
-false
+el-rest-format-aggregations
+FULL
-Tar Modified Time
-${file.lastModifiedTime}
+el-rest-output-no-hits
+false
10
@@ -1647,7 +4218,12 @@ Also the change index name.
1 sec
false
- MergeContent
+ JsonQueryElasticsearch
+
+ true
+ aggregations
+ false
+
false
failure
@@ -1655,7 +4231,7 @@ Also the change index name.
false
- merged
+ hits
false
@@ -1665,18 +4241,18 @@ Also the change index name.
STOPPED
- org.apache.nifi.processors.standard.MergeContent
+ org.apache.nifi.processors.elasticsearch.JsonQueryElasticsearch
- 5be72191-ed94-3f80-0000-000000000000
+ af4ad2b1-2e82-37bf-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
-1920.0
- 912.0
+ 680.0
- 5be72191-ed94-3f80-90c9-1b661fd2ec2a
+ af4ad2b1-2e82-37bf-a78c-dc2e865265c1
- nifi-standard-nar
+ nifi-elasticsearch-restapi-nar
org.apache.nifi
1.24.0
@@ -1687,291 +4263,259 @@ Also the change index name.
1
-Merge Strategy
+el-rest-query-definition-style
- Merge Strategy
+ el-rest-query-definition-style
-Merge Format
+el-rest-query
- Merge Format
+
+ full
+ el-rest-query-definition-style
+
+ el-rest-query
-Attribute Strategy
+el-rest-query-clause
- Attribute Strategy
+
+ build
+ el-rest-query-definition-style
+
+ el-rest-query-clause
-Correlation Attribute Name
+es-rest-size
- Bin-Packing Algorithm
- Merge Strategy
+ build
+ el-rest-query-definition-style
- Correlation Attribute Name
+ es-rest-size
-mergecontent-metadata-strategy
+es-rest-query-sort
- Avro
- Merge Format
+ build
+ el-rest-query-definition-style
- mergecontent-metadata-strategy
+ es-rest-query-sort
-Minimum Number of Entries
+es-rest-query-aggs
- Bin-Packing Algorithm
- Merge Strategy
+ build
+ el-rest-query-definition-style
- Minimum Number of Entries
+ es-rest-query-aggs
-Maximum Number of Entries
+es-rest-query-fields
- Bin-Packing Algorithm
- Merge Strategy
+ build
+ el-rest-query-definition-style
- Maximum Number of Entries
+ es-rest-query-fields
-Minimum Group Size
+es-rest-query-script-fields
- Bin-Packing Algorithm
- Merge Strategy
+ build
+ el-rest-query-definition-style
- Minimum Group Size
+ es-rest-query-script-fields
-Maximum Group Size
+el-query-attribute
-
- Bin-Packing Algorithm
- Merge Strategy
-
- Maximum Group Size
+ el-query-attribute
-Max Bin Age
+el-rest-fetch-index
- Max Bin Age
+ el-rest-fetch-index
-Maximum number of Bins
+el-rest-type
- Maximum number of Bins
+ el-rest-type
-Delimiter Strategy
+el-rest-client-service
-
- Binary Concatenation
- Merge Format
-
- Delimiter Strategy
+ org.apache.nifi.elasticsearch.ElasticSearchClientService
+ el-rest-client-service
-Header File
+el-rest-split-up-hits
-
- Filename
- Text
- Delimiter Strategy
-
-
- Binary Concatenation
- Merge Format
-
- Header File
+ el-rest-split-up-hits
-Footer File
+el-rest-format-hits
-
- Filename
- Text
- Delimiter Strategy
-
-
- Binary Concatenation
- Merge Format
-
- Footer File
+ el-rest-format-hits
-Demarcator File
+el-rest-split-up-aggregations
-
- Filename
- Text
- Delimiter Strategy
-
-
- Binary Concatenation
- Merge Format
-
- Demarcator File
+ el-rest-split-up-aggregations
-Compression Level
+el-rest-format-aggregations
-
- ZIP
- Merge Format
-
- Compression Level
+ el-rest-format-aggregations
-Keep Path
+el-rest-output-no-hits
-
- ZIP
- TAR
- Merge Format
-
- Keep Path
+ el-rest-output-no-hits
-Tar Modified Time
+el-rest-pagination-type
-
- TAR
- Merge Format
-
- Tar Modified Time
+ el-rest-pagination-type
+
+
+
+el-rest-pagination-keep-alive
+
+ el-rest-pagination-keep-alive
- ALL
+ PRIMARY
false
10 mins
30 sec
-Merge Strategy
-Bin-Packing Algorithm
+el-rest-query-definition-style
+full
-Merge Format
-Binary Concatenation
+el-rest-query
+{
+ "size": 10000,
+ "query": { "match_all": {} },
+ "_source": ["_id","docid","patient","ethnicity","gender","birthdate","deathdate"],
+ "sort": [ { "_id": "asc" } ]
+}
-Attribute Strategy
-Keep All Unique Attributes
+el-rest-query-clause
-Correlation Attribute Name
+es-rest-size
-mergecontent-metadata-strategy
-Do Not Merge Uncommon Metadata
+es-rest-query-sort
-Minimum Number of Entries
-50
+es-rest-query-aggs
-Maximum Number of Entries
-100
+es-rest-query-fields
-Minimum Group Size
-0 B
+es-rest-query-script-fields
-Maximum Group Size
+el-query-attribute
+
+
+el-rest-fetch-index
+medical_reports_text
-Max Bin Age
+el-rest-type
-Maximum number of Bins
-1
+el-rest-client-service
+944263c2-7443-35ce-0000-000000000000
-Delimiter Strategy
-Do Not Use Delimiters
+el-rest-split-up-hits
+splitUp-no
-Header File
+el-rest-format-hits
+FULL
-Footer File
+el-rest-split-up-aggregations
+splitUp-no
-Demarcator File
+el-rest-format-aggregations
+FULL
-Compression Level
-1
+el-rest-output-no-hits
+true
-Keep Path
-false
+el-rest-pagination-type
+pagination-search_after
-Tar Modified Time
-${file.lastModifiedTime}
+el-rest-pagination-keep-alive
+60 minutes
10
0
- 0 sec
+ 10 sec
TIMER_DRIVEN
1 sec
- false
- MergeContent
+ true
+ SearchElasticsearch-patient_records-medical_reports_text
false
- failure
+ aggregations
false
false
- merged
- false
-
-
- true
- original
+ hits
false
STOPPED
- org.apache.nifi.processors.standard.MergeContent
+ org.apache.nifi.processors.elasticsearch.SearchElasticsearch
- 71bde74b-e1b9-38af-0000-000000000000
+ b3ec8bc1-7402-3635-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1400.0
- 1616.0
+ -192.0
+ 1520.0
- 71bde74b-e1b9-38af-a977-c3b470f7866b
- nifi-update-attribute-nar
+ nifi-standard-nar
org.apache.nifi
1.24.0
@@ -1982,33 +4526,39 @@ Also the change index name.
1
-Delete Attributes Expression
+Destination
- Delete Attributes Expression
+ Destination
-Store State
+Return Type
- Store State
+ Return Type
-Stateful Variables Initial Value
+Path Not Found Behavior
- Stateful Variables Initial Value
+ Path Not Found Behavior
-canonical-value-lookup-cache-size
+Null Value Representation
- canonical-value-lookup-cache-size
+ Null Value Representation
-timestamp
+Max String Length
- timestamp
+ Max String Length
+
+
+
+document_id
+
+ document_id
@@ -2018,49 +4568,64 @@ Also the change index name.
30 sec
-Delete Attributes Expression
+Destination
+flowfile-attribute
-Store State
-Do not store state
+Return Type
+json
-Stateful Variables Initial Value
+Path Not Found Behavior
+ignore
-canonical-value-lookup-cache-size
-100
+Null Value Representation
+empty string
-timestamp
-${now():format('yyyy_MM_dd_HH_mm_SS')}
+Max String Length
+2048 MB
+
+
+document_id
+$
10
- 25
+ 0
0 sec
TIMER_DRIVEN
1 sec
false
- UpdateAttribute-addTimestamp
+ EvaluateJsonPath-extractDocumentIds
false
- success
+ failure
+ false
+
+
+ false
+ matched
+ false
+
+
+ false
+ unmatched
false
STOPPED
- org.apache.nifi.processors.attributes.UpdateAttribute
+ org.apache.nifi.processors.standard.EvaluateJsonPath
- 7dd60db1-2d31-351c-0000-000000000000
+ bfe47c8f-331e-39a4-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1400.0
- 2000.0
+ -1392.0
+ 2096.0
- 7dd60db1-2d31-351c-89c1-c2ed37eb4567
nifi-standard-nar
org.apache.nifi
@@ -2073,51 +4638,63 @@ Also the change index name.
1
-Directory
-
- Directory
-
-
-
-Conflict Resolution Strategy
+Rate Control Criteria
- Conflict Resolution Strategy
+ Rate Control Criteria
-Create Missing Directories
+Time Duration
- Create Missing Directories
+ Time Duration
-Maximum File Count
+Maximum Rate
- Maximum File Count
+
+ data rate
+ attribute value
+ flowfile count
+ Rate Control Criteria
+
+ Maximum Rate
-Last Modified Time
+Maximum Data Rate
- Last Modified Time
+
+ data rate or flowfile count
+ Rate Control Criteria
+
+ Maximum Data Rate
-Permissions
+Maximum FlowFile Rate
- Permissions
+
+ data rate or flowfile count
+ Rate Control Criteria
+
+ Maximum FlowFile Rate
-Owner
+Rate Controlled Attribute
- Owner
+
+ attribute value
+ Rate Control Criteria
+
+ Rate Controlled Attribute
-Group
+Grouping Attribute
- Group
+ Grouping Attribute
@@ -2127,31 +4704,28 @@ Also the change index name.
30 sec
-Directory
-/opt/data/cogstack-cohort/
-
-
-Conflict Resolution Strategy
-fail
+Rate Control Criteria
+flowfile count
-Create Missing Directories
-true
+Time Duration
+1 sec
-Maximum File Count
+Maximum Rate
+1
-Last Modified Time
+Maximum Data Rate
-Permissions
+Maximum FlowFile Rate
-Owner
+Rate Controlled Attribute
-Group
+Grouping Attribute
10
@@ -2161,31 +4735,31 @@ Also the change index name.
1 sec
false
- PutFile
+ ControlRate
false
failure
false
- true
+ false
success
false
STOPPED
- org.apache.nifi.processors.standard.PutFile
+ org.apache.nifi.processors.standard.ControlRate
- 7eeac5f1-3266-32b5-0000-000000000000
+ c08f004a-29a3-33ef-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -888.0
- 680.0
+ -1400.0
+ 1080.0
- 7eeac5f1-3266-32b5-b125-2e22bd9e375a
+ c08f004a-29a3-33ef-8c29-9b70540173cc
- nifi-elasticsearch-restapi-nar
+ nifi-standard-nar
org.apache.nifi
1.24.0
@@ -2196,257 +4770,151 @@ Also the change index name.
1
-el-rest-query-definition-style
-
- el-rest-query-definition-style
-
-
-
-el-rest-query
-
-
- full
- el-rest-query-definition-style
-
- el-rest-query
-
-
-
-el-rest-query-clause
-
-
- build
- el-rest-query-definition-style
-
- el-rest-query-clause
-
-
-
-es-rest-size
-
-
- build
- el-rest-query-definition-style
-
- es-rest-size
-
-
-
-es-rest-query-sort
-
-
- build
- el-rest-query-definition-style
-
- es-rest-query-sort
-
-
-
-es-rest-query-aggs
-
-
- build
- el-rest-query-definition-style
-
- es-rest-query-aggs
-
-
-
-es-rest-query-fields
-
-
- build
- el-rest-query-definition-style
-
- es-rest-query-fields
-
-
-
-es-rest-query-script-fields
-
-
- build
- el-rest-query-definition-style
-
- es-rest-query-script-fields
-
-
-
-el-query-attribute
-
- el-query-attribute
-
-
-
-el-rest-fetch-index
-
- el-rest-fetch-index
-
-
-
-el-rest-type
+Replacement Strategy
- el-rest-type
+ Replacement Strategy
-el-rest-client-service
+Regular Expression
- org.apache.nifi.elasticsearch.ElasticSearchClientService
- el-rest-client-service
+
+ Regex Replace
+ Literal Replace
+ Replacement Strategy
+
+ Regular Expression
-el-rest-split-up-hits
+Replacement Value
- el-rest-split-up-hits
+
+ Prepend
+ Regex Replace
+ Always Replace
+ Append
+ Literal Replace
+ Replacement Strategy
+
+ Replacement Value
-el-rest-format-hits
+Text to Prepend
- el-rest-format-hits
+
+ Surround
+ Replacement Strategy
+
+ Text to Prepend
-el-rest-split-up-aggregations
+Text to Append
- el-rest-split-up-aggregations
+
+ Surround
+ Replacement Strategy
+
+ Text to Append
-el-rest-format-aggregations
+Character Set
- el-rest-format-aggregations
+ Character Set
-el-rest-output-no-hits
+Maximum Buffer Size
- el-rest-output-no-hits
+ Maximum Buffer Size
-el-rest-pagination-type
+Evaluation Mode
- el-rest-pagination-type
+ Evaluation Mode
-el-rest-pagination-keep-alive
+Line-by-Line Evaluation Mode
- el-rest-pagination-keep-alive
+ Line-by-Line Evaluation Mode
- PRIMARY
+ ALL
false
10 mins
30 sec
-el-rest-query-definition-style
-full
-
-
-el-rest-query
-{
- "size": 10000,
- "query": { "match_all": {} },
- "sort": [ { "timestamp": "asc" } ]
-}
-
-
-el-rest-query-clause
-
-
-es-rest-size
-
-
-es-rest-query-sort
-
-
-es-rest-query-aggs
-
-
-es-rest-query-fields
-
-
-es-rest-query-script-fields
-
-
-el-query-attribute
-
-
-el-rest-fetch-index
-medical_reports_anns_medcat_medmen
-
-
-el-rest-type
+Replacement Strategy
+Literal Replace
-el-rest-client-service
-944263c2-7443-35ce-0000-000000000000
+Regular Expression
+][
-el-rest-split-up-hits
-splitUp-no
+Replacement Value
+,
-el-rest-format-hits
-FULL
+Text to Prepend
-el-rest-split-up-aggregations
-splitUp-no
+Text to Append
-el-rest-format-aggregations
-FULL
+Character Set
+UTF-8
-el-rest-output-no-hits
-false
+Maximum Buffer Size
+1024 MB
-el-rest-pagination-type
-pagination-search_after
+Evaluation Mode
+Line-by-Line
-el-rest-pagination-keep-alive
-60 minutes
+Line-by-Line Evaluation Mode
+All
10
- 0
- 1sec
+ 25
+ 0 sec
TIMER_DRIVEN
1 sec
- true
- SearchElasticsearch-medical_reports_text_annotations
+ false
+ ReplaceText
false
- aggregations
+ failure
false
false
- hits
+ success
false
STOPPED
- org.apache.nifi.processors.elasticsearch.SearchElasticsearch
+ org.apache.nifi.processors.standard.ReplaceText
- a4d63379-4aa7-3bb7-0000-000000000000
+ d0734b2f-aba4-385a-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1400.0
- 1416.0
+ -2216.0
+ 1408.0
- a4d63379-4aa7-3bb7-a224-e3a995a30469
+ d0734b2f-aba4-385a-a9ac-0ef232d90725
nifi-standard-nar
org.apache.nifi
@@ -2489,9 +4957,9 @@ Also the change index name.
-*._source
+output_filename
- *._source
+ output_filename
@@ -2502,7 +4970,7 @@ Also the change index name.
Destination
-flowfile-content
+flowfile-attribute
Return Type
@@ -2518,11 +4986,11 @@ Also the change index name.
Max String Length
-100 MB
+1024 MB
-*._source
-*._source
+output_filename
+$.[0]._index
10
@@ -2532,7 +5000,7 @@ Also the change index name.
1 sec
false
- EvaluateJsonPath-grabSourceField
+ EvaluateJsonPath-grabFileName
false
failure
@@ -2553,15 +5021,15 @@ Also the change index name.
org.apache.nifi.processors.standard.EvaluateJsonPath
- af4ad2b1-2e82-37bf-0000-000000000000
+ e4ad572d-99c6-335d-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1920.0
- 680.0
+ -792.0
+ 2488.0
- af4ad2b1-2e82-37bf-a78c-dc2e865265c1
+ e4ad572d-99c6-335d-8fee-db4e0d05ec2d
- nifi-elasticsearch-restapi-nar
+ nifi-update-attribute-nar
org.apache.nifi
1.24.0
@@ -2572,259 +5040,184 @@ Also the change index name.
1
-el-rest-query-definition-style
-
- el-rest-query-definition-style
-
-
-
-el-rest-query
-
-
- full
- el-rest-query-definition-style
-
- el-rest-query
-
-
-
-el-rest-query-clause
+Delete Attributes Expression
-
- build
- el-rest-query-definition-style
-
- el-rest-query-clause
+ Delete Attributes Expression
-es-rest-size
+Store State
-
- build
- el-rest-query-definition-style
-
- es-rest-size
+ Store State
-es-rest-query-sort
+Stateful Variables Initial Value
-
- build
- el-rest-query-definition-style
-
- es-rest-query-sort
+ Stateful Variables Initial Value
-es-rest-query-aggs
+canonical-value-lookup-cache-size
-
- build
- el-rest-query-definition-style
-
- es-rest-query-aggs
+ canonical-value-lookup-cache-size
-es-rest-query-fields
+filename
-
- build
- el-rest-query-definition-style
-
- es-rest-query-fields
+ filename
-es-rest-query-script-fields
+output_filename
-
- build
- el-rest-query-definition-style
-
- es-rest-query-script-fields
+ output_filename
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
-el-query-attribute
-
- el-query-attribute
-
+Delete Attributes Expression
-el-rest-fetch-index
-
- el-rest-fetch-index
-
+Store State
+Do not store state
-el-rest-type
-
- el-rest-type
-
+Stateful Variables Initial Value
-el-rest-client-service
-
- org.apache.nifi.elasticsearch.ElasticSearchClientService
- el-rest-client-service
-
+canonical-value-lookup-cache-size
+100
-el-rest-split-up-hits
-
- el-rest-split-up-hits
-
+filename
+${output_filename:append("__"):append(${timestamp}):append(".json")}
-el-rest-format-hits
-
- el-rest-format-hits
-
+output_filename
+${output_filename:append("__"):append(${timestamp}):append(".json")}
+
+ 10
+ 25
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ UpdateAttribute-filename
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.attributes.UpdateAttribute
+
+
+ e7943295-e173-3870-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1984.0
+ 2088.0
+
+
+ nifi-update-attribute-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
-el-rest-split-up-aggregations
+Delete Attributes Expression
- el-rest-split-up-aggregations
+ Delete Attributes Expression
-el-rest-format-aggregations
+Store State
- el-rest-format-aggregations
+ Store State
-el-rest-output-no-hits
+Stateful Variables Initial Value
- el-rest-output-no-hits
+ Stateful Variables Initial Value
-el-rest-pagination-type
+canonical-value-lookup-cache-size
- el-rest-pagination-type
+ canonical-value-lookup-cache-size
-el-rest-pagination-keep-alive
+ann_timestamp
- el-rest-pagination-keep-alive
+ ann_timestamp
- PRIMARY
+ ALL
false
10 mins
30 sec
-el-rest-query-definition-style
-full
-
-
-el-rest-query
-{
- "size": 10000,
- "query": { "match_all": {} },"_source": {
- "exclude": [ "document" ]
- },
- "sort": [ { "_id": "asc" } ]
-}
-
-
-el-rest-query-clause
-
-
-es-rest-size
-
-
-es-rest-query-sort
-
-
-es-rest-query-aggs
-
-
-es-rest-query-fields
-
-
-es-rest-query-script-fields
-
-
-el-query-attribute
-
-
-el-rest-fetch-index
-medical_reports_text
-
-
-el-rest-type
-
-
-el-rest-client-service
-944263c2-7443-35ce-0000-000000000000
-
-
-el-rest-split-up-hits
-splitUp-no
-
-
-el-rest-format-hits
-FULL
-
-
-el-rest-split-up-aggregations
-splitUp-no
+Delete Attributes Expression
-el-rest-format-aggregations
-FULL
+Store State
+Do not store state
-el-rest-output-no-hits
-false
+Stateful Variables Initial Value
-el-rest-pagination-type
-pagination-search_after
+canonical-value-lookup-cache-size
+1
-el-rest-pagination-keep-alive
-60 minutes
+ann_timestamp
+${now():format('yyyy_MM_dd_HH_mm_ss_SSS')}
10
0
- 10 sec
+ 1 sec
TIMER_DRIVEN
1 sec
- true
- SearchElasticsearch-patient_records-medical_reports_text
-
- false
- aggregations
- false
-
+ false
+ UpdateAttribute-addTimestampAnn
false
- hits
+ success
false
STOPPED
- org.apache.nifi.processors.elasticsearch.SearchElasticsearch
+ org.apache.nifi.processors.attributes.UpdateAttribute
- d0734b2f-aba4-385a-0000-000000000000
+ e9507246-cbf9-3c4a-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1400.0
- 1216.0
+ -192.0
+ 2096.0
- d0734b2f-aba4-385a-a9ac-0ef232d90725
nifi-standard-nar
org.apache.nifi
@@ -2867,9 +5260,9 @@ Also the change index name.
-filename
+doc_ids
- filename
+ doc_ids
@@ -2880,11 +5273,11 @@ Also the change index name.
Destination
-flowfile-attribute
+flowfile-content
Return Type
-auto-detect
+json
Path Not Found Behavior
@@ -2896,11 +5289,11 @@ Also the change index name.
Max String Length
-100 MB
+1000 MB
-filename
-$.[0]._index
+doc_ids
+*.docid
10
@@ -2910,7 +5303,7 @@ Also the change index name.
1 sec
false
- EvaluateJsonPath-grabFileName
+ EvaluateJsonPath-extractDocumentIds
false
failure
@@ -2931,13 +5324,79 @@ Also the change index name.
org.apache.nifi.processors.standard.EvaluateJsonPath
- e4ad572d-99c6-335d-0000-000000000000
+ eb4eeb18-315c-334c-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -1400.0
- 1816.0
+ -792.0
+ 2096.0
+
+
+ nifi-standard-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Routing Strategy
+
+ Routing Strategy
+
+
+
+is_annotation_index
+
+ is_annotation_index
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Routing Strategy
+Route to 'match' if any matches
+
+
+is_annotation_index
+${is_annotation_index:isNull():or(${is_annotation_index:equals("False")})}
+
+
+ 10
+ 25
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ RouteOnAttribute-checkIfAnnotationIndex
+
+ false
+ matched
+ false
+
+
+ false
+ unmatched
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.standard.RouteOnAttribute
+
+
+ ef208086-8b96-38e5-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -1984.0
+ 2296.0
- e4ad572d-99c6-335d-8fee-db4e0d05ec2d
nifi-update-attribute-nar
org.apache.nifi
@@ -2977,6 +5436,12 @@ Also the change index name.
filename
filename
+
+
+
+output_filename
+
+ output_filename
@@ -3001,7 +5466,11 @@ Also the change index name.
filename
-${filename:append("_"):append(${timestamp}):append(".json")}
+${annotations_index_name:append("__"):append(${timestamp}):append("__"):append(${ann_timestamp}):append(".json")}
+
+
+output_filename
+${annotations_index_name:append("__"):append(${timestamp}):append("__"):append(${ann_timestamp}):append(".json")}
10
@@ -3011,7 +5480,107 @@ Also the change index name.
1 sec
false
- UpdateAttribute-filename
+ UpdateAttribute-filename-annotations
+
+ false
+ success
+ false
+
+ STOPPED
+
+ org.apache.nifi.processors.attributes.UpdateAttribute
+
+
+ f0f64682-161c-38cd-0000-000000000000
+ ba40cb68-0d02-3fdf-0000-000000000000
+
+ -2536.0
+ 792.0
+
+
+ nifi-update-attribute-nar
+ org.apache.nifi
+ 1.24.0
+
+
+ PENALIZE_FLOWFILE
+ WARN
+
+ 1
+
+
+Delete Attributes Expression
+
+ Delete Attributes Expression
+
+
+
+Store State
+
+ Store State
+
+
+
+Stateful Variables Initial Value
+
+ Stateful Variables Initial Value
+
+
+
+canonical-value-lookup-cache-size
+
+ canonical-value-lookup-cache-size
+
+
+
+annotations_index_name
+
+ annotations_index_name
+
+
+
+document_id_field_name
+
+ document_id_field_name
+
+
+
+ ALL
+ false
+ 10 mins
+ 30 sec
+
+
+Delete Attributes Expression
+
+
+Store State
+Do not store state
+
+
+Stateful Variables Initial Value
+
+
+canonical-value-lookup-cache-size
+100
+
+
+annotations_index_name
+medical_reports_anns_medcat_medmen
+
+
+document_id_field_name
+docid
+
+
+ 10
+ 25
+ 0 sec
+ TIMER_DRIVEN
+ 1 sec
+
+ false
+ UpdateAttribute-docIdFieldName_and_ann_idx_name
false
success
@@ -3025,8 +5594,8 @@ Also the change index name.
f710ad2f-4da0-36c0-0000-000000000000
ba40cb68-0d02-3fdf-0000-000000000000
- -2552.0
- 1176.0
+ -3552.0
+ 1624.0
f710ad2f-4da0-36c0-8894-020161ab03c1
@@ -3094,14 +5663,14 @@ Also the change index name.
Command Arguments
-/opt/nifi/user-scripts/cogstack_cohort_generate_data.py;input_annotations_records_path=/opt/data/cogstack-cohort/medical_reports_anns_medcat_medmen.json;input_patient_records_path=/opt/data/cogstack-cohort/medical_reports_text.json;annotation_document_id_field_name="meta.docid";
+/opt/nifi/user-scripts/cogstack_cohort_generate_data.py;document_id_field_name="docid";patient_id_field_name="patient";input_annotations_records_path=/opt/data/cogstack-cohort/medical_reports_anns_medcat_medmen_2024_02_23_15_28_599.json;input_patient_records_path=/opt/data/cogstack-cohort/medical_reports_text_2024_02_23_16_28_541.json;annotation_document_id_field_name="meta.docid";output_folder_path="/opt/data/cogstack-cohort/"
Batch Duration
Redirect Error Stream
-false
+true
Working Directory
@@ -3129,7 +5698,7 @@ Also the change index name.
success
false
- STOPPED
+ DISABLED
org.apache.nifi.processors.standard.ExecuteProcess
@@ -3144,5 +5713,5 @@ Also the change index name.
- 02/20/2024 17:31:20 GMT
+ 02/25/2024 12:35:50 GMT