feat: allow remote resources in scripted content

Message changes: - message `OPF-018` (`WARNING`) is only reported when a Content Doc is declared in the Package Document with the property `remote-resource`, and neither remote references nor scripts are found. - new message `OPF-018b` (`USAGE`) reported when a Content Doc is declared in the Package Document with the property `remote-resource`, and no remote references are found but the Content Doc has scripted content. - message `RSC-006` (`ERROR`) is only reported when a remote publication resource is declared in the Package Document with a type that doesn't specifically allow it to be located outside the container, and when it is directly referenced from a Content Document, or used in the spine, or else if there is no scripted content in the publication. - new message `RSC-006b` (`USAGE`) is reported when a remote publication resource is declared in the Package Document with a type that doesn't specifically allow it to be located outside the container, and scripted content is found. Internal changes: - new utility static method `OPFChecker.isScriptType(String)` tells if a media type is a JavaScript MIME type essence match (see https://mimesniff.spec.whatwg.org/#javascript-mime-type-essence-match) - better identify scripted content: from event handler attributes, or script element with a type matching a JavaScript MIME type essence. - add a couple tests and fix existing tests Fix #869. fixup: better check remote resources
w3c · Mar 10, 2019 · 1c90ae9 · 1c90ae9
1 parent 4d5a5a9
commit 1c90ae9
Show file tree

Hide file tree

Showing 41 changed files with 449 additions and 51 deletions.
diff --git a/src/main/java/com/adobe/epubcheck/messages/DefaultSeverities.java b/src/main/java/com/adobe/epubcheck/messages/DefaultSeverities.java
@@ -191,6 +191,7 @@ private void initialize()
     severities.put(MessageId.OPF_016, Severity.ERROR);
     severities.put(MessageId.OPF_017, Severity.ERROR);
     severities.put(MessageId.OPF_018, Severity.WARNING);
+    severities.put(MessageId.OPF_018b, Severity.USAGE);
     severities.put(MessageId.OPF_019, Severity.FATAL);
     severities.put(MessageId.OPF_020, Severity.SUPPRESSED);
     severities.put(MessageId.OPF_021, Severity.WARNING);
@@ -291,6 +292,7 @@ private void initialize()
     severities.put(MessageId.RSC_004, Severity.ERROR);
     severities.put(MessageId.RSC_005, Severity.ERROR);
     severities.put(MessageId.RSC_006, Severity.ERROR);
+    severities.put(MessageId.RSC_006b, Severity.USAGE);
     severities.put(MessageId.RSC_007, Severity.ERROR);
     severities.put(MessageId.RSC_007w, Severity.WARNING);
     severities.put(MessageId.RSC_008, Severity.ERROR);

diff --git a/src/main/java/com/adobe/epubcheck/messages/MessageId.java b/src/main/java/com/adobe/epubcheck/messages/MessageId.java
@@ -184,6 +184,7 @@ public enum MessageId implements Comparable<MessageId>
   OPF_016("OPF-016"),
   OPF_017("OPF-017"),
   OPF_018("OPF-018"),
+  OPF_018b("OPF-018b"),
   OPF_019("OPF-019"),
   OPF_020("OPF-020"),
   OPF_021("OPF-021"),
@@ -284,6 +285,7 @@ public enum MessageId implements Comparable<MessageId>
   RSC_004("RSC-004"),
   RSC_005("RSC-005"),
   RSC_006("RSC-006"),
+  RSC_006b("RSC-006b"),
   RSC_007("RSC-007"),
   RSC_007w("RSC-007w"),
   RSC_008("RSC-008"),

diff --git a/src/main/java/com/adobe/epubcheck/opf/OPFChecker.java b/src/main/java/com/adobe/epubcheck/opf/OPFChecker.java
@@ -27,6 +27,7 @@
 import java.util.HashSet;
 import java.util.Hashtable;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 
 import com.adobe.epubcheck.api.EPUBLocation;
@@ -331,6 +332,27 @@ public static boolean isBlessedFontMimetype20(String mime)
     return mime != null && (mime.startsWith("font/") || mime.startsWith("application/font")
         || mime.startsWith("application/x-font") || "application/vnd.ms-opentype".equals(mime));
   }
+
+  public static boolean isScriptType(String type)
+  {
+    type = (type == null)? null : type.toLowerCase(Locale.ENGLISH);
+    return "application/javascript".equals(type)
+        || "text/javascript".equals(type)
+        || "application/ecmascript".equals(type)
+        || "application/x-ecmascript".equals(type)
+        || "application/x-javascript".equals(type)
+        || "text/ecmascript".equals(type)
+        || "text/javascript1.0".equals(type)
+        || "text/javascript1.1".equals(type)
+        || "text/javascript1.2".equals(type)
+        || "text/javascript1.3".equals(type)
+        || "text/javascript1.4".equals(type)
+        || "text/javascript1.5".equals(type)
+        || "text/jscript".equals(type)
+        || "text/livescript".equals(type)
+        || "text/x-ecmascript".equals(type)
+        || "text/x-javascript".equals(type);
+  }
 
   protected void checkItem(OPFItem item, OPFHandler opfHandler)
   {

diff --git a/src/main/java/com/adobe/epubcheck/opf/OPFChecker30.java b/src/main/java/com/adobe/epubcheck/opf/OPFChecker30.java
@@ -44,6 +44,7 @@
 import com.adobe.epubcheck.overlay.OverlayCheckerFactory;
 import com.adobe.epubcheck.util.EPUBVersion;
 import com.adobe.epubcheck.util.FeatureEnum;
+import com.adobe.epubcheck.util.PathUtil;
 import com.adobe.epubcheck.vocab.DCMESVocab;
 import com.adobe.epubcheck.vocab.PackageVocabs;
 import com.google.common.base.Optional;
@@ -142,17 +143,36 @@ protected void checkItemAfterResourceValidation(OPFItem item)
   {
     XRefChecker xrefChecker = context.xrefChecker.get();
 
-    // Report remote resources when not allowed
+    // Check remote resources
     String mediatype = item.getMimeType();
-    if (item.getPath().matches("^[^:/?#]+://.*") 
+    if (PathUtil.isRemote(item.getPath())
+        // audio, video, and fonts can be remote resources
         && !(isAudioType(mediatype)
         || isVideoType(mediatype)
         || "application/x-shockwave-flash".equals(mediatype)
-        || isFontType(mediatype)
-        || xrefChecker.getTypes(item.getPath()).equals(EnumSet.of(Type.FONT))))
+        || isFontType(mediatype)))
     {
-      report.message(MessageId.RSC_006,
-          EPUBLocation.create(path, item.getLineNumber(), item.getColumnNumber()), item.getPath());
+      // spine items cannot be remote resources
+      // (except, theoretically, for video/audio/fonts)
+      if (item.isInSpine())
+      {
+        report.message(MessageId.RSC_006,
+            EPUBLocation.create(path, item.getLineNumber(), item.getColumnNumber()), item.getPath());
+      }
+      // if no direct reference to the resource was found,
+      else if (xrefChecker.getTypes(item.getPath()).isEmpty())
+      {
+        // if may be allowed when if the resource is retrieved from a script
+        if (context.featureReport.hasFeature(FeatureEnum.HAS_SCRIPTS)) {
+          report.message(MessageId.RSC_006b,
+              EPUBLocation.create(path, item.getLineNumber(), item.getColumnNumber()), item.getPath());
+        }
+        // otherwise, still report it as an error, even if not used
+        else {
+          report.message(MessageId.RSC_006,
+              EPUBLocation.create(path, item.getLineNumber(), item.getColumnNumber()), item.getPath());
+        }
+      }
     }
   }
 

diff --git a/src/main/java/com/adobe/epubcheck/opf/XRefChecker.java b/src/main/java/com/adobe/epubcheck/opf/XRefChecker.java
@@ -40,7 +40,6 @@
 import com.adobe.epubcheck.vocab.PackageVocabs;
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 
 public class XRefChecker
@@ -262,37 +261,48 @@ private void checkReference(Reference ref)
   {
     Resource res = resources.get(ref.refResource);
     Resource host = resources.get(ref.source);
+
+    // Check remote resources
+    if (PathUtil.isRemote(ref.refResource)
+        // remote links and hyperlinks are not Publication Resources
+        && !EnumSet.of(Type.LINK, Type.HYPERLINK).contains(ref.type)
+        // spine items are checked in OPFChecker30
+        && !(version == EPUBVersion.VERSION_3
+             && res != null && res.item.isInSpine())
+        // audio, video, and fonts can be remote resources in EPUB 3
+        && !(version == EPUBVersion.VERSION_3
+             && EnumSet.of(Type.AUDIO, Type.VIDEO, Type.FONT).contains(ref.type)))
+    {
+      report.message(MessageId.RSC_006,
+          EPUBLocation.create(ref.source, ref.lineNumber, ref.columnNumber, ref.refResource));
+      return;
+    }
 
     // Check undeclared resources
     if (res == null)
     {
-      if (version == EPUBVersion.VERSION_3 && ref.type == Type.LINK)
+      // Report references to missing local resources
+      if (!ocf.hasEntry(ref.refResource) && !PathUtil.isRemote(ref.refResource))
       {
-        if (PathUtil.isRemote(ref.refResource) || ocf.hasEntry(ref.refResource))
-        {
-          return;
+        // only as a WARNING for 'link' references in EPUB 3
+        if (version == EPUBVersion.VERSION_3 && ref.type == Type.LINK) {
+          report.message(MessageId.RSC_007w,
+              EPUBLocation.create(ref.source, ref.lineNumber, ref.columnNumber, ref.refResource),
+              ref.refResource);
         }
+        // by default as an ERROR
         else
         {
-          report.message(MessageId.RSC_007w,
+          report.message(MessageId.RSC_007,
               EPUBLocation.create(ref.source, ref.lineNumber, ref.columnNumber, ref.refResource),
               ref.refResource);
         }
       }
-      else if (PathUtil.isRemote(ref.refResource) && !(version == EPUBVersion.VERSION_3
-          && (ref.type == Type.AUDIO || ref.type == Type.VIDEO || ref.type == Type.FONT)))
-      {
-        report.message(MessageId.RSC_006,
-            EPUBLocation.create(ref.source, ref.lineNumber, ref.columnNumber, ref.refResource));
-      }
-      else if (!ocf.hasEntry(ref.refResource) && !PathUtil.isRemote(ref.refResource))
-      {
-        report.message(MessageId.RSC_007,
-            EPUBLocation.create(ref.source, ref.lineNumber, ref.columnNumber, ref.refResource),
-            ref.refResource);
-
-      }
-      else if (!undeclared.contains(ref.refResource))
+      // Report undeclared Publication Resources (once)
+      else if (!undeclared.contains(ref.refResource)
+          // links and remote hyperlinks are not Publication Resources
+          && !(ref.type == Type.LINK
+               ||  PathUtil.isRemote(ref.refResource) && ref.type == Type.HYPERLINK))
       {
         undeclared.add(ref.refResource);
         report.message(MessageId.RSC_008,

diff --git a/src/main/java/com/adobe/epubcheck/ops/OPSHandler.java b/src/main/java/com/adobe/epubcheck/ops/OPSHandler.java
@@ -360,6 +360,10 @@ else if (name.equals("font-face-uri"))
         {
           checkSVGFontFaceURI(e, "http://www.w3.org/1999/xlink", "href");
         }
+        else if (name.equals("script"))
+        {
+          checkScript(e);
+        }
         checkPaint(e, "fill");
         checkPaint(e, "stroke");
       }
@@ -415,6 +419,10 @@ else if (name.equals("i") || name.equals("b") || name.equals("em") || name.equal
         {
           checkBoldItalics(e);
         }
+        else if (name.equals("script"))
+        {
+          checkScript(e);
+        }
 
         resourceType = XRefChecker.Type.HYPERLINK;
 
@@ -456,6 +464,20 @@ protected URI checkURI(String uri)
       return null;
     }
   }
+
+  protected void checkScript(XMLElement e) {
+      String type = e.getAttribute("type");
+      if (type == null || OPFChecker.isScriptType(type)) {
+        processJavascript();
+      }
+  }
+
+  protected void processJavascript()
+  {
+    report.info(path, FeatureEnum.HAS_SCRIPTS, "");
+    context.featureReport.report(FeatureEnum.HAS_SCRIPTS, EPUBLocation.create(path, 
+        parser.getLineNumber(), parser.getColumnNumber()));
+  }
 
   public void endElement()
   {
@@ -485,12 +507,7 @@ public void endElement()
     if (EpubConstants.HtmlNamespaceUri.equals(ns))
     {
 
-      if ("script".equals(name))
-      {
-        String attr = e.getAttribute("type");
-        report.info(path, FeatureEnum.HAS_SCRIPTS, (attr == null) ? "" : attr);
-      }
-      else if ("style".equals(name))
+      if ("style".equals(name))
       {
         String style = textNode.toString();
         if (style.length() > 0)

diff --git a/src/main/java/com/adobe/epubcheck/ops/OPSHandler30.java b/src/main/java/com/adobe/epubcheck/ops/OPSHandler30.java
@@ -36,7 +36,6 @@
 import com.adobe.epubcheck.vocab.StructureVocab.EPUB_TYPES;
 import com.adobe.epubcheck.vocab.Vocab;
 import com.adobe.epubcheck.vocab.VocabUtil;
-import com.adobe.epubcheck.xml.Namespaces;
 import com.adobe.epubcheck.xml.XMLAttribute;
 import com.adobe.epubcheck.xml.XMLElement;
 import com.adobe.epubcheck.xml.XMLParser;
@@ -269,10 +268,6 @@ else if (!context.mimeType.equals("image/svg+xml") && name.equals("svg"))
       requiredProperties.add(ITEM_PROPERTIES.SVG);
       processStartSvg(e);
     }
-    else if (name.equals("script"))
-    {
-      requiredProperties.add(ITEM_PROPERTIES.SCRIPTED);
-    }
     else if (EpubConstants.EpubTypeNamespaceUri.equals(e.getNamespace()) && name.equals("switch"))
     {
       requiredProperties.add(ITEM_PROPERTIES.SWITCH);
@@ -335,11 +330,18 @@ protected void processInlineScripts(com.adobe.epubcheck.xml.XMLElement e)
       String name = attr.getName().toLowerCase(Locale.ROOT);
       if (scriptEvents.contains(name) || mouseEvents.contains(name))
       {
-        requiredProperties.add(ITEM_PROPERTIES.SCRIPTED);
+        processJavascript();
         return;
       }
     }
   }
+
+  @Override
+  protected void processJavascript()
+  {
+    super.processJavascript();
+    requiredProperties.add(ITEM_PROPERTIES.SCRIPTED);
+  }
 
   protected void processLink(XMLElement e)
   {
@@ -753,8 +755,13 @@ protected void checkProperties()
     if (uncheckedProperties.contains(ITEM_PROPERTIES.REMOTE_RESOURCES))
     {
       uncheckedProperties.remove(ITEM_PROPERTIES.REMOTE_RESOURCES);
-      report.message(MessageId.OPF_018,
-          EPUBLocation.create(path, parser.getLineNumber(), parser.getColumnNumber()));
+      if (!requiredProperties.contains(ITEM_PROPERTIES.SCRIPTED)) {
+        report.message(MessageId.OPF_018,
+            EPUBLocation.create(path, parser.getLineNumber(), parser.getColumnNumber()));
+      } else {
+        report.message(MessageId.OPF_018b,
+            EPUBLocation.create(path, parser.getLineNumber(), parser.getColumnNumber()));
+      }
     }
 
     if (!uncheckedProperties.isEmpty())

diff --git a/src/main/resources/com/adobe/epubcheck/messages/MessageBundle.properties b/src/main/resources/com/adobe/epubcheck/messages/MessageBundle.properties
@@ -177,7 +177,8 @@ OPF_014=The property '%1$s' should be declared in the OPF file.
 OPF_015=The property '%1$s' should not be declared in the OPF file.
 OPF_016=The element \"rootfile\" is missing its required attribute \"full-path\".
 OPF_017=The attribute \"full-path\" on element \"rootfile\" must not be empty.
-OPF_018=The 'remote-resources' property was declared in the OPF, but no reference to remote resources has been found. Make sure this property is legitimate.
+OPF_018=The 'remote-resources' property was declared in the Package Document, but no reference to remote resources has been found.
+OPF_018b=The 'remote-resources' property was declared in the Package Document, but no reference to remote resources has been found; please check scripted content to make sure the property is legit. 
 OPF_019=Spine tag was not found in the OPF file.
 OPF_020=Excessive number of spine items.
 OPF_021=Use of non-registered URI scheme type in href: '%1$s'.
@@ -300,7 +301,7 @@ RSC_003=No rootfile tag with media type 'application/oebps-package+xml' was foun
 RSC_004=File '%1$s' could not be decrypted.
 RSC_005=Error while parsing file: %1$s
 RSC_006=Remote resource reference not allowed; resource must be placed in the OCF.
-RSC_006_SUG=Only audio and video remote resources are permitted.
+RSC_006b=Resource '%1$s' is located outside the EPUB Container; please check the resource is retrieved in scripted content. 
 RSC_007=Referenced resource '%1$s' could not be found in the EPUB.
 RSC_007w=Referenced resource '%1$s' could not be found in the EPUB.
 RSC_008=Referenced resource '%1$s' is not declared in the OPF manifest.

diff --git a/src/test/java/com/adobe/epubcheck/api/Epub30CheckExpandedTest.java b/src/test/java/com/adobe/epubcheck/api/Epub30CheckExpandedTest.java
@@ -367,6 +367,22 @@ public void testRemoteImgUndeclaredInOPF()
     Collections.addAll(expectedErrors, MessageId.RSC_006);
     testValidateDocument("invalid/remote-img-undeclared/");
   }
+
+  @Test
+  public void testRemoteImgAlsoUsedInScript()
+  {
+    // tests that remote images are not allowed, even when also retrieved in scripts
+    Collections.addAll(expectedErrors, MessageId.RSC_006);
+    testValidateDocument("invalid/remote-img-also-in-script/");
+  }
+
+  @Test
+  public void testRemoteImgAlsoReferencedInLink()
+  {
+    // tests that remote images are not allowed, even when also referenced as linked resources
+    Collections.addAll(expectedErrors, MessageId.RSC_006);
+    testValidateDocument("invalid/remote-img-also-in-link/");
+  }
 
   @Test
   public void testRemoteAudioWithMissingRemoteResourcesProperty()
@@ -464,6 +480,31 @@ public void testRemoteSVGContentDocInvalid() {
     testValidateDocument("invalid/remote-svg-contentdoc");
   }
 
+  @Test
+  public void testRemoteInScriptForeign() {
+    // test that a (foreign) resource used in a script MAY be a remote resource
+    // OPF_018b is expected to report that the 'remote-resources' property couldn't be verified
+    // RSC_006b is expected to report the remote item 
+    Collections.addAll(expectedUsages, MessageId.OPF_018b, MessageId.RSC_006b);
+    testValidateDocument("valid/remote-in-script-foreign", true, false);
+  }
+
+  @Test
+  public void testRemoteInScriptCMT() {
+    // test that SVG Content Documents MUST NOT be remote resources
+    // OPF_018b is expected to report that the 'remote-resources' property couldn't be verified
+    // RSC_006b is expected to report the remote item 
+    Collections.addAll(expectedUsages, MessageId.OPF_018b, MessageId.RSC_006b);
+    testValidateDocument("valid/remote-in-script-cmt", true, false);
+  }
+
+  @Test
+  public void testRemoteSpineItem() {
+    // test that top-level Content Documents MUST NOT be remote resources
+    expectedErrors.add(MessageId.RSC_006);
+    testValidateDocument("invalid/remote-spine-item");
+  }
+
   @Test
   public void testValidateEPUB30_circularFallback()
   {

diff --git a/src/test/resources/30/expanded/invalid/remote-img-also-in-link/META-INF/container.xml b/src/test/resources/30/expanded/invalid/remote-img-also-in-link/META-INF/container.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
+   <rootfiles>
+      <rootfile full-path="OPS/package.opf" media-type="application/oebps-package+xml"/>
+   </rootfiles>
+</container>