Feature: Ignore Empty HL7 Segments

TEMPLATING.md

@@ -14,6 +14,7 @@ A HL7 message template maps one or more HL7 segments to a FHIR resource using th
       resourcePath: [REQUIRED]
       repeats:  [DEFAULT false]
       isReferenced: [DEFAULT false]
+      ignoreEmpty: [DEFAULT false]
       additionalSegments: [DEFAULT empty]
 ```
 
@@ -24,6 +25,7 @@ A HL7 message template maps one or more HL7 segments to a FHIR resource using th
 | resourcePath       | Required         | Relative path to the resource template. Example: resource/Patient                                                                                                                                    |
 | repeats            | Default: false   | Indicates if a repeating HL7 segment will generate multiple FHIR resources.                                                                                                                          |
 | isReferenced       | Default: false   | Indicates if the FHIR Resource is referenced by other FHIR resources.                                                                                                                                |
+| ignoreEmpty        | Default: false   | Indicates if an empty HL7 segment will NOT generate the matching (almost empty) FHIR resource   |
 | group | Default: empty   | Base group from which the segment and additionalSegments are specified. 
 | additionalSegments | Default: empty   | List of additional HL7 segment names required to complete the FHIR resource mapping.                                                                                                                 |
 
@@ -61,6 +63,7 @@ resources:
       segment: AL1
       resourcePath: resource/AllergyIntolerance
       repeats: true
+      ignoreEmpty: true   ## Sometimes AL1 segments arrive empty
       additionalSegments:
 
 

src/main/java/io/github/linuxforhealth/api/FHIRResourceTemplate.java

@@ -49,6 +49,10 @@ public interface FHIRResourceTemplate {
    */
   boolean isReferenced();
 
-
-
+ /**
+  * If this resource is to ignore empty source segments
+  * 
+  * @return True/False
+  */
+  boolean ignoreEmpty();
 }

src/main/java/io/github/linuxforhealth/core/message/AbstractFHIRResourceTemplate.java

@@ -17,22 +17,25 @@ public abstract class AbstractFHIRResourceTemplate implements FHIRResourceTempla
   private boolean repeats;
   private String resourcePath;
   private boolean isReferenced;
+  private boolean ignoreEmpty;
 
 
   @JsonCreator
   public AbstractFHIRResourceTemplate(@JsonProperty("resourceName") String resourceName,
       @JsonProperty("resourcePath") String resourcePath,
       @JsonProperty("isReferenced") boolean isReferenced,
-      @JsonProperty("repeats") boolean repeats) {
+      @JsonProperty("repeats") boolean repeats,
+      @JsonProperty("ignoreEmpty") boolean ignoreEmpty) {
     this.resourceName = resourceName;
     this.resourcePath = resourcePath;
     this.repeats = repeats;
     this.isReferenced = isReferenced;
+    this.ignoreEmpty = ignoreEmpty;
   }
 
 
   public AbstractFHIRResourceTemplate(String resourceName, String resourcePath) {
-    this(resourceName, resourcePath, false, false);
+    this(resourceName, resourcePath, false, false, false);
   }
 
   @Override
@@ -64,6 +67,11 @@ public boolean isReferenced() {
     return isReferenced;
   }
 
+  @Override
+  public boolean ignoreEmpty() {
+    return ignoreEmpty;
+  }
+
 
 
 

src/main/java/io/github/linuxforhealth/hl7/message/HL7FHIRResourceTemplate.java

@@ -52,6 +52,10 @@ public boolean isReferenced() {
     return this.attributes.isReferenced();
   }
 
+  @Override
+  public boolean ignoreEmpty() {
+    return this.attributes.ignoreEmpty();
+  }
 
 
 }

src/main/java/io/github/linuxforhealth/hl7/message/HL7FHIRResourceTemplateAttributes.java

@@ -20,13 +20,13 @@ public class HL7FHIRResourceTemplateAttributes {
   private boolean repeats;
   private String resourcePath;
   private boolean isReferenced;
+  private boolean ignoreEmpty;
   private HL7Segment segment;// primary segment
   private List<HL7Segment> additionalSegments;
   private ResourceModel resource;
   private List<String> group;
 
 
-
   public HL7FHIRResourceTemplateAttributes(Builder builder) {
     Preconditions.checkArgument(StringUtils.isNotBlank(builder.resourceName),
         "resourceName cannot be null");
@@ -35,6 +35,7 @@ public HL7FHIRResourceTemplateAttributes(Builder builder) {
     this.resourcePath = builder.resourcePath;
     this.repeats = builder.repeats;
     this.isReferenced = builder.isReferenced;
+    this.ignoreEmpty = builder.ignoreEmpty;
     additionalSegments = new ArrayList<>();
     builder.rawAdditionalSegments
         .forEach(e -> additionalSegments.add(HL7Segment.parse(e, builder.group)));
@@ -85,7 +86,9 @@ public boolean isReferenced() {
     return isReferenced;
   }
 
-
+  public boolean ignoreEmpty() {
+    return ignoreEmpty;
+  }
 
   private static ResourceModel generateResourceModel(String resourcePath) {
     return ResourceReader.getInstance().generateResourceModel(resourcePath);
@@ -102,6 +105,7 @@ public static class Builder {
     private String resourcePath;
     private String group;
     private boolean isReferenced;
+    private boolean ignoreEmpty;
     private boolean repeats;
     private ResourceModel resourceModel;
 
@@ -156,6 +160,11 @@ public Builder withIsReferenced(boolean isReferenced) {
       return this;
     }
 
+    public Builder withignoreEmpty(boolean ignoreEmpty) {
+      this.ignoreEmpty = ignoreEmpty;
+      return this;
+    }
+
     public Builder withResourceModel(ResourceModel resourceModel) {
       this.resourceModel = resourceModel;
       return this;

src/main/java/io/github/linuxforhealth/hl7/message/HL7MessageEngine.java

@@ -27,7 +27,9 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 
+import ca.uhn.hl7v2.HL7Exception;
 import ca.uhn.hl7v2.model.Structure;
+import ca.uhn.hl7v2.model.Visitable;
 import io.github.linuxforhealth.api.EvaluationResult;
 import io.github.linuxforhealth.api.FHIRResourceTemplate;
 import io.github.linuxforhealth.api.InputDataExtractor;
@@ -189,7 +191,7 @@ private List<ResourceResult> generateResources(HL7MessageData hl7DataInput,
         if (!multipleSegments.isEmpty()) {
 
             resourceResults = generateMultipleResources(hl7DataInput, resourceModel, contextValues,
-                    multipleSegments, template.isGenerateMultiple());
+                    multipleSegments, template.isGenerateMultiple(), template.ignoreEmpty());
         }
         return resourceResults;
     }
@@ -284,7 +286,7 @@ private static List<SegmentGroup> getMultipleSegments(final HL7MessageData hl7Da
 
     private static List<ResourceResult> generateMultipleResources(final HL7MessageData hl7DataInput,
             final ResourceModel rs, final Map<String, EvaluationResult> contextValues,
-            final List<SegmentGroup> multipleSegments, boolean generateMultiple) {
+            final List<SegmentGroup> multipleSegments, boolean generateMultiple, boolean ignoreEmpty) {
         List<ResourceResult> resourceResults = new ArrayList<>();
         for (SegmentGroup currentGroup : multipleSegments) {
 
@@ -300,16 +302,24 @@ private static List<ResourceResult> generateMultipleResources(final HL7MessageDa
 
             for (EvaluationResult baseValue : baseValues) {
                 try {
-                    ResourceResult result = rs.evaluate(hl7DataInput, ImmutableMap.copyOf(localContextValues),
-                            baseValue);
-                    if (result != null && result.getValue() != null) {
-                        resourceResults.add(result);
-                        if (!generateMultiple) {
-                            // If only single resource needs to be generated then return.
-                            return resourceResults;
+                    // We might need to check if the baseValue is empty
+                    Visitable vs = baseValue.getValue();
+                    if((vs != null && ! vs.isEmpty()) || ! ignoreEmpty) {
+
+                        // baseValue is either not empty or we're not allowed to ignore empty segments
+                        ResourceResult result = rs.evaluate(hl7DataInput, ImmutableMap.copyOf(localContextValues),
+                                baseValue);
+
+                        // We can't rely on empty segment giving us an empty resource; as common templates populate Resource.meta fields
+                        if (result != null && result.getValue() != null) {
+                            resourceResults.add(result);
+                            if (!generateMultiple) {
+                                // If only single resource needs to be generated then return.
+                                return resourceResults;
+                            }
                         }
                     }
-                } catch (RequiredConstraintFailureException | IllegalArgumentException
+                } catch (RequiredConstraintFailureException | IllegalArgumentException | HL7Exception
                         | IllegalStateException e) {
                     LOGGER.warn("generateMultipleResources - Exception encountered");
                     LOGGER.debug("generateMultipleResources - Exception encountered", e);

src/test/java/io/github/linuxforhealth/hl7/message/Hl7CustomMessageTest.java

@@ -81,7 +81,6 @@ static void reloadPreviousConfigurations() {
     @Test
     void testCustomPatMessage() throws IOException {
 
-        // Set up the config file
         commonConfigFileSetup();
 
         String hl7message = "MSH|^~\\&|||||20211005105125||CUSTOM^PAT|1a3952f1-38fe-4d55-95c6-ce58ebfc7f10|P|2.6\n"
@@ -113,7 +112,8 @@ private static void commonConfigFileSetup() throws IOException {
         prop.put("base.path.resource", "src/main/resources");
         prop.put("supported.hl7.messages", "*"); // Must use wild card so the custom resources are found.
         prop.put("default.zoneid", "+08:00");
-        prop.put("additional.resources.location", "src/test/resources/additional_custom_resources"); // Location of custom resources
+        // Location of custom (or merely additional) resources
+        prop.put("additional.resources.location",  "src/test/resources/additional_custom_resources");
         prop.store(new FileOutputStream(configFile), null);
         System.setProperty(CONF_PROP_HOME, configFile.getParent());
     }

src/test/java/io/github/linuxforhealth/hl7/message/Hl7IgnoreEmptyTest.java

@@ -0,0 +1,157 @@
+/*
+ * (c) Te Whatu Ora, Health New Zealand, 2023
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * 
+ * @author Stuart McGrigor
+ */
+
+package io.github.linuxforhealth.hl7.message;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.r4.model.AllergyIntolerance;
+import org.hl7.fhir.r4.model.Bundle;
+import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
+import org.hl7.fhir.r4.model.Resource;
+import org.hl7.fhir.r4.model.ResourceType;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import io.github.linuxforhealth.core.config.ConverterConfiguration;
+import io.github.linuxforhealth.fhir.FHIRContext;
+import io.github.linuxforhealth.hl7.ConverterOptions;
+import io.github.linuxforhealth.hl7.ConverterOptions.Builder;
+import io.github.linuxforhealth.hl7.HL7ToFHIRConverter;
+import io.github.linuxforhealth.hl7.resource.ResourceReader;
+import io.github.linuxforhealth.hl7.segments.util.ResourceUtils;
+
+// This class uses the ability to create ADDITIONAL HL7 messages to convert weird HL7 messages
+// that exercise the new ignoreEmpty Resource Template functionality
+//
+// In these tests, the additional message definitions for (entirely ficticious) ADT^A09 and ^A10 messages
+// are placed in src/test/resources/additional_resources/hl7/message/ADT_A09.yml and ADT_A10.yml
+
+class Hl7IgnoreEmptyTest {
+
+    // # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+    // NOTE VALIDATION IS INTENTIONALLY NOT USED BECAUSE WE ARE CREATING RESOURCES THAT ARE NOT STANDARD
+    private static final ConverterOptions OPTIONS = new Builder().withPrettyPrint().build();
+    // # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+    private static final String CONF_PROP_HOME = "hl7converter.config.home";
+
+    @TempDir
+    static File folder;
+
+    static String originalConfigHome;
+
+    @BeforeAll
+    static void saveConfigHomeProperty() {
+        originalConfigHome = System.getProperty(CONF_PROP_HOME);
+        ConverterConfiguration.reset();
+        ResourceReader.reset();
+        folder.setWritable(true);
+    }
+
+    @AfterEach
+    void reset() {
+        System.clearProperty(CONF_PROP_HOME);
+        ConverterConfiguration.reset();
+        ResourceReader.reset();
+    }
+
+    @AfterAll
+    static void reloadPreviousConfigurations() {
+        if (originalConfigHome != null)
+            System.setProperty(CONF_PROP_HOME, originalConfigHome);
+        else
+            System.clearProperty(CONF_PROP_HOME);
+        folder.setWritable(true);
+    }
+
+    // ADT_A09 has ignoreEmpty = true; ADT_A10 doesn't
+    @ParameterizedTest
+    @CsvSource({ "ADT^A09,ZAL|\r,0", "ADT^A10,ZAL|\r,1",     // Custom ZAL segment
+                 "ADT^A09,AL1|\r,0", "ADT^A10,AL1|\r,1",     // Standard AL1 segment
+                 "ADT^A09,ZAL|||||||||||||||\r,0",           // ZAL segment with all fields being empty
+                 "ADT^A09,ZAL|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"|\"\"\r,1"    // So-called EMPTY ZAL segment is not actually empty
+               })
+    void testIgnoreEmptySegment(String messageType, String emptySegment, int aiCount) throws IOException {
+
+        // Set up the config file
+        commonConfigFileSetup();
+
+        // An empty AL1 and ZAL Segment...
+        String hl7message = "MSH|^~\\&|TestSystem||TestTransformationAgent||20150502090000||" + messageType + "|controlID|P|2.6\r"
+                + "EVN|A01|20150502090000|\r"
+                + "PID|||1234^^^^MR||DOE^JANE^|||F||||||||||||||||||||||\r"
+                + "PV1||I||||||||SUR||||||||S|VisitNumber^^^ACME|A||||||||||||||||||||||||20150502090000|\r"
+                + emptySegment;
+
+        List<BundleEntryComponent> e = getBundleEntryFromHL7Message(hl7message);
+
+        List<Resource> patientResource = ResourceUtils.getResourceList(e, ResourceType.Patient);
+        assertThat(patientResource).hasSize(1); // from PID
+
+        List<Resource> encounterResource = ResourceUtils.getResourceList(e, ResourceType.Encounter);
+        assertThat(encounterResource).hasSize(1); // from EVN, PV1
+
+        List<Resource> allergyIntoleranceResource = ResourceUtils.getResourceList(e, ResourceType.AllergyIntolerance);
+        assertThat(allergyIntoleranceResource).hasSize(aiCount); // empty AL1 and ZAL Segments
+
+        // Confirm that there are no extra resources
+        assertThat(e).hasSize(2 + aiCount);
+
+        // We might be done
+        if(aiCount == 0)
+            return;
+
+        // Let's take a peek at the 'empty' ZAL Segment
+        assertThat(allergyIntoleranceResource).allSatisfy(rs -> {
+
+                // Only some of the fields
+                AllergyIntolerance ai = (AllergyIntolerance) rs;
+                assert(ai.hasId());
+                assert(ai.hasClinicalStatus());
+                assert(ai.hasVerificationStatus());
+            });
+    }
+
+
+    private static void commonConfigFileSetup() throws IOException {
+        File configFile = new File(folder, "config.properties");
+        Properties prop = new Properties();
+        prop.put("base.path.resource", "src/main/resources");
+        prop.put("supported.hl7.messages", "ADT_A09, ADT_A10, ADT_A11"); // We're using weird ADT messages
+        prop.put("default.zoneid", "+08:00");
+        // Location of additional resources
+        prop.put("additional.resources.location",  "src/test/resources/additional_resources");
+        prop.store(new FileOutputStream(configFile), null);
+        System.setProperty(CONF_PROP_HOME, configFile.getParent());
+    }
+
+    // Need custom convert sequence with options that turn off FHIR validation.
+    private static List<BundleEntryComponent> getBundleEntryFromHL7Message(String hl7message) {
+        HL7ToFHIRConverter ftv = new HL7ToFHIRConverter(); // Testing loading of config which happens once per instantiation
+        String json = ftv.convert(hl7message, OPTIONS); // Need custom options that turn off FHIR validation.
+        assertThat(json).isNotNull();
+        FHIRContext context = new FHIRContext();
+        IBaseResource bundleResource = context.getParser().parseResource(json);
+        assertThat(bundleResource).isNotNull();
+        Bundle b = (Bundle) bundleResource;
+        return b.getEntry();
+    }
+
+}

src/test/resources/additional_resources/hl7/message/ADT_A09.yml

@@ -39,3 +39,19 @@ resources:
         - EVN
         - MSH
         - DG1
+
+    - resourceName: AllergyIntolerance
+      segment: AL1
+      resourcePath: resource/AllergyIntolerance
+      repeats: true
+      ignoreEmpty: true      ## We want to test our new ignoreEmpty flag
+      additionalSegments:
+        - MSH
+
+    - resourceName: AllergyIntolerance
+      segment: ZAL
+      resourcePath: resource/AllergyIntolerance
+      repeats: true
+      ignoreEmpty: true      ## We want to test our new ignoreEmpty flag on 'Z' segments
+      additionalSegments:
+        - MSH