Merge pull request #108 from Yelp/capturing-env-vars

Capturing env vars in files

.secrets.baseline

@@ -1,11 +1,17 @@
 {
-  "exclude_regex": "test_data/.*|tests/.*",
-  "generated_at": "2018-07-12T23:20:29Z",
+  "exclude_regex": "test_data/.*|tests/.*|^.secrets.baseline$",
+  "generated_at": "2018-12-21T22:29:02Z",
   "plugins_used": [
+    {
+      "name": "AWSKeyDetector"
+    },
     {
       "base64_limit": 4.5,
       "name": "Base64HighEntropyString"
     },
+    {
+      "name": "BasicAuthDetector"
+    },
     {
       "hex_limit": 3,
       "name": "HexHighEntropyString"
@@ -15,58 +21,70 @@
     }
   ],
   "results": {
+    "README.md": [
+      {
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "line_number": 153,
+        "type": "Basic Auth Credentials"
+      }
+    ],
     "detect_secrets/plugins/high_entropy_strings.py": [
       {
         "hashed_secret": "88a7b59d2e9172960b72b65f7839b9da2453f3e9",
         "is_secret": false,
-        "line_number": 215,
+        "line_number": 261,
         "type": "Hex High Entropy String"
       }
     ],
     "detect_secrets/plugins/private_key.py": [
       {
         "hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b",
         "is_secret": false,
-        "line_number": 34,
+        "line_number": 43,
         "type": "Private Key"
       },
       {
         "hashed_secret": "daefe0b4345a654580dcad25c7c11ff4c944a8c0",
         "is_secret": false,
-        "line_number": 35,
+        "line_number": 44,
         "type": "Private Key"
       },
       {
         "hashed_secret": "f0778f3e140a61d5bbbed5430773e52af2f5fba4",
         "is_secret": false,
-        "line_number": 36,
+        "line_number": 45,
         "type": "Private Key"
       },
       {
         "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd",
         "is_secret": false,
-        "line_number": 37,
+        "line_number": 46,
         "type": "Private Key"
       },
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_secret": false,
-        "line_number": 38,
+        "line_number": 47,
         "type": "Private Key"
       },
       {
         "hashed_secret": "11200d1bf5e1eb358b5d823c443347d97e982a85",
         "is_secret": false,
-        "line_number": 39,
+        "line_number": 48,
         "type": "Private Key"
       },
       {
         "hashed_secret": "9279619d0c9a9529b0b223e3b809f4df24b8ba8b",
         "is_secret": false,
-        "line_number": 40,
+        "line_number": 49,
+        "type": "Private Key"
+      },
+      {
+        "hashed_secret": "4ada9713ec27066b2ffe0b7bd9c9c8d635dc4ab2",
+        "line_number": 50,
         "type": "Private Key"
       }
     ]
   },
-  "version": "0.9.1"
+  "version": "0.11.0"
 }

README.md

@@ -163,7 +163,7 @@ committing secrets.
 ### Things that won't be prevented
 
 * Multi-line secrets
-* Default passwords that do not trigger the `KeywordDetector` (e.g. `paaassword = "paaassword"`)
+* Default passwords that do not trigger the `KeywordDetector` (e.g. `login = "hunter2"`)
 
 ### Plugin Configuration
 

detect_secrets/main.py

@@ -133,7 +133,7 @@ def _get_existing_baseline(import_filename):
             return json.loads(stdin)
 
 
-def _read_from_file(filename):
+def _read_from_file(filename):  # pragma: no cover
     """Used for mocking."""
     with open(filename) as f:
         return json.loads(f.read())

detect_secrets/plugins/aws.py

@@ -9,7 +9,7 @@
 
 
 class AWSKeyDetector(RegexBasedDetector):
-    secret_type = 'AWS key'
+    secret_type = 'AWS Access Key'
     blacklist = (
         re.compile(r'AKIA[0-9A-Z]{16}'),
     )

detect_secrets/plugins/core/ini_file_parser.py

@@ -6,10 +6,23 @@ class IniFileParser(object):
 
     _comment_regex = re.compile(r'\s*[;#]')
 
-    def __init__(self, file):
+    def __init__(self, file, add_header=False):
         self.parser = configparser.ConfigParser()
         self.parser.optionxform = str
-        self.parser.read_file(file)
+
+        if not add_header:
+            self.parser.read_file(file)
+        else:
+            # This supports environment variables, or other files that look
+            # like config files, without a section header.
+            content = '[global]\n' + file.read()
+
+            try:
+                # python2.7 compatible
+                self.parser.read_string(unicode(content))
+            except NameError:
+                # python3 compatible
+                self.parser.read_string(content)
 
         # Hacky way to keep track of line location
         file.seek(0)

detect_secrets/plugins/high_entropy_strings.py

@@ -51,17 +51,23 @@ def __init__(self, charset, limit, *args):
 
     def analyze(self, file, filename):
         file_type_analyzers = (
-            (self._analyze_ini_file, configparser.Error,),
+            (self._analyze_ini_file(), configparser.Error,),
             (self._analyze_yaml_file, yaml.YAMLError,),
+            (super(HighEntropyStringsPlugin, self).analyze, Exception,),
+            (self._analyze_ini_file(add_header=True), configparser.Error,),
         )
 
         for analyze_function, exception_class in file_type_analyzers:
             try:
-                return analyze_function(file, filename)
+                output = analyze_function(file, filename)
+                if output:
+                    return output
             except exception_class:
-                file.seek(0)
+                pass
 
-        return super(HighEntropyStringsPlugin, self).analyze(file, filename)
+            file.seek(0)
+
+        return {}
 
     def calculate_shannon_entropy(self, data):
         """Returns the entropy of a given string.
@@ -154,21 +160,24 @@ def non_quoted_string_regex(self, strict=True):
         finally:
             self.regex = old_regex
 
-    def _analyze_ini_file(self, file, filename):
+    def _analyze_ini_file(self, add_header=False):
         """
         :returns: same format as super().analyze()
         """
-        potential_secrets = {}
+        def wrapped(file, filename):
+            potential_secrets = {}
 
-        with self.non_quoted_string_regex():
-            for value, lineno in IniFileParser(file).iterator():
-                potential_secrets.update(self.analyze_string(
-                    value,
-                    lineno,
-                    filename,
-                ))
+            with self.non_quoted_string_regex():
+                for value, lineno in IniFileParser(file, add_header).iterator():
+                    potential_secrets.update(self.analyze_string(
+                        value,
+                        lineno,
+                        filename,
+                    ))
 
-        return potential_secrets
+            return potential_secrets
+
+        return wrapped
 
     def _analyze_yaml_file(self, file, filename):
         """

test_data/config.env

@@ -0,0 +1 @@
+mimi=gX69YO4CvBsVjzAwYxdGyDd30t5+9ez31gKATtj4

test_data/config.yaml

@@ -3,7 +3,7 @@ credentials:
     other_value_here: 1234567890a
     nested:
         value: AKIAabcdefghijklmnop
-        value: abcdefghijklmnop
+        other_value: abcdefghijklmnop
 list_of_keys:
     - 123
     - 456

testing/mocks.py

@@ -135,7 +135,7 @@ class PrinterShim(object):
         def __init__(self):
             self.clear()
 
-        def add(self, message):
+        def add(self, message, *args, **kwargs):
             self.message += str(message) + '\n'
 
         def clear(self):

tests/main_test.py

@@ -15,32 +15,6 @@
 from testing.mocks import mock_printer
 
 
-@pytest.fixture
-def mock_baseline_initialize():
-    def mock_initialize_function(plugins, exclude_regex, *args, **kwargs):
-        return secrets_collection_factory(
-            plugins=plugins,
-            exclude_regex=exclude_regex,
-        )
-
-    with mock.patch(
-        'detect_secrets.main.baseline.initialize',
-        side_effect=mock_initialize_function,
-    ) as mock_initialize:
-        yield mock_initialize
-
-
-@pytest.fixture
-def mock_merge_baseline():
-    with mock.patch(
-        'detect_secrets.main.baseline.merge_baseline',
-    ) as m:
-        # This return value needs to have the `results` key, so that it can
-        # formatted appropriately for output.
-        m.return_value = {'results': {}}
-        yield m
-
-
 class TestMain(object):
     """These are smoke tests for the console usage of detect_secrets.
     Most of the functional test cases should be within their own module tests.
@@ -269,6 +243,16 @@ def test_audit_short_file(self, filename, expected_output):
 
         BashColor.enable_color()
 
+    def test_audit_diff_not_enough_files(self):
+        assert main('audit --diff fileA'.split()) == 1
+
+    def test_audit_same_file(self):
+        with mock_printer(main_module) as printer_shim:
+            assert main('audit --diff .secrets.baseline .secrets.baseline'.split()) == 0
+            assert printer_shim.message.strip() == (
+                'No difference, because it\'s the same file!'
+            )
+
 
 @contextmanager
 def mock_stdin(response=None):
@@ -282,3 +266,29 @@ def mock_stdin(response=None):
             m.stdin.isatty.return_value = False
             m.stdin.read.return_value = response
             yield
+
+
+@pytest.fixture
+def mock_baseline_initialize():
+    def mock_initialize_function(plugins, exclude_regex, *args, **kwargs):
+        return secrets_collection_factory(
+            plugins=plugins,
+            exclude_regex=exclude_regex,
+        )
+
+    with mock.patch(
+        'detect_secrets.main.baseline.initialize',
+        side_effect=mock_initialize_function,
+    ) as mock_initialize:
+        yield mock_initialize
+
+
+@pytest.fixture
+def mock_merge_baseline():
+    with mock.patch(
+        'detect_secrets.main.baseline.merge_baseline',
+    ) as m:
+        # This return value needs to have the `results` key, so that it can
+        # formatted appropriately for output.
+        m.return_value = {'results': {}}
+        yield m

tests/plugins/base_test.py

@@ -6,7 +6,7 @@
 
 
 def test_fails_if_no_secret_type_defined():
-    class MockPlugin(BasePlugin):
+    class MockPlugin(BasePlugin):       # pragma: no cover
         def analyze_string(self, *args, **kwargs):
             pass
 

tests/plugins/high_entropy_strings_test.py

@@ -111,6 +111,25 @@ def test_ignored_lines(self, content_to_format):
 
         assert len(results) == 0
 
+    def test_entropy_lower_limit(self):
+        with pytest.raises(ValueError):
+            Base64HighEntropyString(-1)
+
+    def test_entropy_upper_limit(self):
+        with pytest.raises(ValueError):
+            Base64HighEntropyString(15)
+
+
+class TestBase64HighEntropyStrings(HighEntropyStringsTest):
+
+    def setup(self):
+        super(TestBase64HighEntropyStrings, self).setup(
+            # Testing default limit, as suggested by truffleHog.
+            Base64HighEntropyString(4.5),
+            'c3VwZXIgc2VjcmV0IHZhbHVl',     # too short for high entropy
+            'c3VwZXIgbG9uZyBzdHJpbmcgc2hvdWxkIGNhdXNlIGVub3VnaCBlbnRyb3B5',
+        )
+
     def test_ini_file(self):
         # We're testing two files here, because we want to make sure that
         # the HighEntropyStrings regex is reset back to normal after
@@ -151,31 +170,25 @@ def test_yaml_file(self):
         with open('test_data/config.yaml') as f:
             secrets = plugin.analyze(f, 'test_data/config.yaml')
 
-        assert len(secrets.values()) == 1
+        assert len(secrets.values()) == 2
         for secret in secrets.values():
             location = str(secret).splitlines()[1]
             assert location in (
                 'Location:    test_data/config.yaml:3',
+                'Location:    test_data/config.yaml:5',
             )
 
-    def test_entropy_lower_limit(self):
-        with pytest.raises(ValueError):
-            Base64HighEntropyString(-1)
-
-    def test_entropy_upper_limit(self):
-        with pytest.raises(ValueError):
-            Base64HighEntropyString(15)
+    def test_env_file(self):
+        plugin = Base64HighEntropyString(4.5)
+        with open('test_data/config.env') as f:
+            secrets = plugin.analyze(f, 'test_data/config.env')
 
-
-class TestBase64HighEntropyStrings(HighEntropyStringsTest):
-
-    def setup(self):
-        super(TestBase64HighEntropyStrings, self).setup(
-            # Testing default limit, as suggested by truffleHog.
-            Base64HighEntropyString(4.5),
-            'c3VwZXIgc2VjcmV0IHZhbHVl',     # too short for high entropy
-            'c3VwZXIgbG9uZyBzdHJpbmcgc2hvdWxkIGNhdXNlIGVub3VnaCBlbnRyb3B5',
-        )
+        assert len(secrets.values()) == 1
+        for secret in secrets.values():
+            location = str(secret).splitlines()[1]
+            assert location in (
+                'Location:    test_data/config.env:1',
+            )
 
 
 class TestHexHighEntropyStrings(HighEntropyStringsTest):