TIKA-4153 -- revert changes to robots.txt detection and add unit test…

… for robots file starting with comments (cherry picked from commit 7825b59)
apache · Oct 16, 2023 · 575298c · 575298c
1 parent a427bac
commit 575298c
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 10 deletions.
diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -2135,16 +2135,14 @@
     <!-- should have a higher priority than rfc822 - TIKA-3489 -->
     <magic priority="55">
       <match minShouldMatch="2">
-        <match minShouldMatch="1">
-          <match value="user-agent:" type="stringignorecase" offset="0"/>
-          <match value="sitemap:" type="stringignorecase" offset="0"/>
-        </match>
-        <match minShouldMatch="1">
-          <match value="\nuser-agent:" type="stringignorecase" offset="0:1000"/>
-          <match value="\nallow:" type="stringignorecase" offset="0:1000"/>
-          <match value="\ndisallow:" type="stringignorecase" offset="0:1000"/>
-          <match value="\nsitemap:" type="stringignorecase" offset="0:1000"/>
-        </match>
+        <match value="user-agent:" type="stringignorecase" offset="0"/>
+        <match value="allow:" type="stringignorecase" offset="0"/>
+        <match value="disallow:" type="stringignorecase" offset="0"/>
+        <match value="sitemap:" type="stringignorecase" offset="0"/>
+        <match value="\nuser-agent:" type="stringignorecase" offset="0:1000"/>
+        <match value="\nallow:" type="stringignorecase" offset="0:1000"/>
+        <match value="\ndisallow:" type="stringignorecase" offset="0:1000"/>
+        <match value="\nsitemap:" type="stringignorecase" offset="0:1000"/>
       </match>
     </magic>
     <sub-class-of type="text/plain"/>

diff --git a/...ndard/tika-parsers-standard-package/src/test/java/org/apache/tika/mime/TestMimeTypes.java b/...ndard/tika-parsers-standard-package/src/test/java/org/apache/tika/mime/TestMimeTypes.java
@@ -1023,6 +1023,7 @@ public void testEmail() throws IOException {
     @Test
     public void testRobots() throws Exception {
         assertTypeByData("text/x-robots", "testRobots.txt");
+        assertTypeByData("text/x-robots", "testRobots2.txt");
     }
 
     @Test

diff --git a/...-standard/tika-parsers-standard-package/src/test/resources/test-documents/testRobots2.txt b/...-standard/tika-parsers-standard-package/src/test/resources/test-documents/testRobots2.txt
@@ -0,0 +1,11 @@
+# elevate robots begin
+# robots.txt, added by the Elevate plugin for WordPress
+# file version: 1
+User-agent: *
+Disallow: /wp-admin/
+Disallow: /readme.html
+Disallow: /trackback/
+Allow: /wp-admin/admin-ajax.php
+Allow: /wp-content/uploads
+Sitemap: https://blahdeblah.com/sitemap.xml
+# elevate robots end