Merge pull request #37 from benbalter/prefetch-content

Prefetch content in parallel

lib/site-inspector/checks/content.rb

@@ -32,14 +32,36 @@ def doctype
         document.internal_subset.name
       end
 
+      def prefetch
+        options = SiteInspector.typhoeus_defaults.merge(followlocation: true)
+        ["robots.txt", "sitemap.xml", "humans.txt", random_path].each do |path|
+          request = Typhoeus::Request.new(URI.join(endpoint.uri, path), options)
+          SiteInspector.hydra.queue(request)
+        end
+        SiteInspector.hydra.run
+      end
+
+      def proper_404s?
+        @proper_404s ||= !path_exists?(random_path)
+      end
+
       def to_h
+        prefetch
         {
           doctype:     doctype,
           sitemap_xml: sitemap_xml?,
           robots_txt:  robots_txt?,
-          humans_txt:  humans_txt?
+          humans_txt:  humans_txt?,
+          proper_404s: proper_404s?
         }
       end
+
+      private
+
+      def random_path
+        require 'securerandom'
+        @random_path ||= SecureRandom.hex
+      end
     end
   end
 end

lib/site-inspector/checks/headers.rb

@@ -56,14 +56,6 @@ def secure_cookies?
         !!(cookie =~ /(; secure.*; httponly|; httponly.*; secure)/i)
       end
 
-
-      def proper_404s?
-        @proper_404s ||= begin
-          require 'securerandom'
-          endpoint.request(path: SecureRandom.hex, followlocation: true).response_code == 404
-        end
-      end
-
       # Returns an array of hashes of downcased key/value header pairs (or an empty hash)
       def all
         @all ||= (response && response.headers) ? Hash[response.headers.map{ |k,v| [k.downcase,v] }] : {}
@@ -83,8 +75,7 @@ def to_h
           :click_jacking_protection => click_jacking_protection || false,
           :server => server,
           :xss_protection => xss_protection || false,
-          :secure_cookies => secure_cookies?,
-          :proper_404s => proper_404s?
+          :secure_cookies => secure_cookies?
         }
       end
     end

spec/checks/site_inspector_endpoint_content_spec.rb

@@ -66,4 +66,24 @@
       to_return(:status => 200)
     expect(subject.humans_txt?).to eql(true)
   end
+
+  context "404s" do
+    it "knows when an endpoint returns a proper 404" do
+      stub_request(:get, /http\:\/\/example.com\/.*/).
+        to_return(:status => 404)
+      expect(subject.proper_404s?).to eql(true)
+    end
+
+    it "knows when an endpoint doesn't return a proper 404" do
+      stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).
+        to_return(:status => 200)
+      expect(subject.proper_404s?).to eql(false)
+    end
+
+    it "generates a random path" do
+      path = subject.send(:random_path)
+      expect(path).to match /[a-z0-9]{32}/i
+      expect(subject.send(:random_path)).to eql(path)
+    end
+  end
 end

spec/checks/site_inspector_endpoint_headers_spec.rb

@@ -71,16 +71,4 @@ def stub_header(header, value)
   it "knows if the cookies are secure" do
 
   end
-
-  it "knows when an endpoint returns a proper 404" do
-    stub_request(:get, /http\:\/\/example.com\/.*/).
-      to_return(:status => 404)
-    expect(subject.proper_404s?).to eql(true)
-  end
-
-  it "knows when an endpoint doesn't return a proper 404" do
-    stub_request(:get, /http\:\/\/example.com\/[a-z0-9]{32}/i).
-      to_return(:status => 200)
-    expect(subject.proper_404s?).to eql(false)
-  end
 end