Add PoliCheck (Azure#20776)

* Add PoliCheck * Fix up some policheck issues
zihzhan-msft · Sep 30, 2021 · abc9a1c · abc9a1c
1 parent d1b2a62
commit abc9a1c
Show file tree

Hide file tree

Showing 18 changed files with 138 additions and 81 deletions.
diff --git a/eng/guardian-tools/policheck/PolicheckExclusions.xml b/eng/guardian-tools/policheck/PolicheckExclusions.xml
@@ -0,0 +1,12 @@
+<PoliCheckExclusions>
+<!-- All strings must be UPPER CASE -->
+<!-- Each of these exclusions is a folder name - if \[name]\ exists in the file path, it will be skipped -->
+<!--<Exclusion Type="FolderPathFull">ABC|XYZ</Exclusion>-->
+<!-- Each of these exclusions is a folder name - if any folder or file starts with "\[name]", it will be 
+skipped -->
+<!--<Exclusion Type="FolderPathStart">ABC|XYZ</Exclusion>-->
+<!-- Each of these file types will be completely skipped for the entire scan -->
+<!--<Exclusion Type="FileType">.ABC|.XYZ</Exclusion>-->
+<!-- The specified file names will be skipped during the scan regardless which folder they are in -->
+<!--<Exclusion Type="FileName">ABC.TXT|XYZ.CS</Exclusion>-->
+</PoliCheckExclusions>
diff --git a/eng/pipelines/aggregate-reports.yml b/eng/pipelines/aggregate-reports.yml
@@ -8,60 +8,105 @@ pr:
     include:
       - eng/pipelines/aggregate-reports.yml
 
-jobs:
-- job: 'ValidateDependencies'
-  variables:
+pool:
+  name: azsdk-pool-mms-win-2019-general
+  vmImage: MMS2019
+
+variables:
   - template: ./templates/variables/globals.yml
 
-  pool:
-    name: azsdk-pool-mms-win-2019-general
-    vmImage: MMS2019
-
-  steps:
-  - template: /eng/pipelines/templates/steps/analyze_dependency.yml
-
-  - task: AzureFileCopy@2
-    displayName: 'Upload dependency report'
-    condition: and(succeededOrFailed(), eq(variables['System.TeamProject'], 'internal'))
-    inputs:
-      sourcePath: '$(Build.ArtifactStagingDirectory)/reports'
-      azureSubscription: 'Azure SDK Artifacts'
-      destination: AzureBlob
-      storage: azuresdkartifacts
-      containerName: 'azure-sdk-for-python'
-      blobPrefix: dependencies
-
-  - task: PowerShell@2
-    displayName: "Verify Repository Resource Refs"
-    inputs:
-      pwsh: true
-      workingDirectory: $(Build.SourcesDirectory)
-      filePath: eng/common/scripts/Verify-Resource-Ref.ps1
-
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
-    displayName: 'Run CredScan'
-    condition: succeededOrFailed()
-    inputs:
-      suppressionsFile: 'eng\CredScanSuppression.json'
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
-    displayName: 'Post Analysis'
-    condition: succeededOrFailed()
-    inputs:
-      GdnBreakAllTools: false
-      GdnBreakGdnToolCredScan: true
-      GdnBreakGdnToolCredScanSeverity: Error
-      GdnBreakBaselineFiles: $(Build.SourcesDirectory)\eng\python.gdnbaselines
-      GdnBreakBaselines: baseline
-      # Used for generating baseline file.
-      # GdnBreakOutputBaselineFile: python
-      # GdnBreakOutputBaseline: baseline
-    continueOnError: true
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
-    displayName: 'Publish Security Analysis Logs'
-    continueOnError: true
-    condition: succeededOrFailed()
-  - template: ../common/pipelines/templates/steps/verify-links.yml
-    parameters:
-      Directory: ""
-      CheckLinkGuidance: $true
-      Condition: succeededOrFailed()
+stages:
+  - stage: ValidateDependencies
+    displayName: Validate Dependencies
+
+    jobs:
+      - job: ValidateDependencies
+        timeoutInMinutes: 120
+        steps:
+
+          - template: /eng/pipelines/templates/steps/analyze_dependency.yml
+
+          - task: AzureFileCopy@2
+            displayName: 'Upload dependency report'
+            condition: and(succeededOrFailed(), eq(variables['System.TeamProject'], 'internal'))
+            inputs:
+              sourcePath: '$(Build.ArtifactStagingDirectory)/reports'
+              azureSubscription: 'Azure SDK Artifacts'
+              destination: AzureBlob
+              storage: azuresdkartifacts
+              containerName: 'azure-sdk-for-python'
+              blobPrefix: dependencies
+
+          - task: PowerShell@2
+            displayName: "Verify Repository Resource Refs"
+            inputs:
+              pwsh: true
+              workingDirectory: $(Build.SourcesDirectory)
+              filePath: eng/common/scripts/Verify-Resource-Ref.ps1
+
+          - template: ../common/pipelines/templates/steps/verify-links.yml
+            parameters:
+              Directory: ""
+              CheckLinkGuidance: $true
+              Condition: succeededOrFailed()
+
+  - stage: ComplianceTools
+    displayName: Compliance Tools
+    dependsOn: []
+
+    jobs:
+      - job: ComplianceTools
+        timeoutInMinutes: 120
+        steps:
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
+            displayName: 'Run CredScan'
+            condition: succeededOrFailed()
+            inputs:
+              suppressionsFile: 'eng\CredScanSuppression.json'
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
+            displayName: 'Post Analysis'
+            condition: succeededOrFailed()
+            inputs:
+              GdnBreakAllTools: false
+              GdnBreakGdnToolCredScan: true
+              GdnBreakGdnToolCredScanSeverity: Error
+              GdnBreakBaselineFiles: $(Build.SourcesDirectory)\eng\python.gdnbaselines
+              GdnBreakBaselines: baseline
+              # Used for generating baseline file.
+              # GdnBreakOutputBaselineFile: python
+              # GdnBreakOutputBaseline: baseline
+            continueOnError: true
+
+          - pwsh: |
+              azcopy copy "https://azuresdkartifacts.blob.core.windows.net/policheck/PythonPoliCheckExclusion.mdb?$(azuresdk-policheck-blob-SAS)" `
+              "$(Build.BinariesDirectory)"
+            displayName: 'Download PoliCheck Exclusion Database'
+            condition: succeededOrFailed()
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2
+            displayName: 'Run PoliCheck'
+            inputs:
+              targetType: F
+              targetArgument: '$(Build.SourcesDirectory)'
+              result: PoliCheck.sarif
+              optionsFC: 0
+              optionsXS: 1
+              optionsPE: 1|2|3|4
+              optionsRulesDBPath: "$(Build.BinariesDirectory)/PythonPoliCheckExclusion.mdb"
+              optionsUEPATH: "$(Build.SourcesDirectory)/eng/guardian-tools/policheck/PolicheckExclusions.xml"
+            condition: succeededOrFailed()
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
+            displayName: 'Post Analysis (PoliCheck)'
+            inputs:
+              GdnBreakAllTools: false
+              GdnBreakGdnToolPoliCheck: true
+              GdnBreakGdnToolPoliCheckSeverity: Warning
+            condition: succeededOrFailed()
+            continueOnError: true
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
+            displayName: 'Publish Security Analysis Logs'
+            continueOnError: true
+            condition: succeededOrFailed()
diff --git a/.../azure/cognitiveservices/vision/contentmoderator/operations/text_moderation_operations.py b/.../azure/cognitiveservices/vision/contentmoderator/operations/text_moderation_operations.py
@@ -35,10 +35,10 @@ def __init__(self, client, config, serializer, deserializer):
 
     def screen_text(
             self, text_content_type, text_content, language=None, autocorrect=False, pii=False, list_id=None, classify=False, custom_headers=None, raw=False, callback=None, **operation_config):
-        """Detect profanity and match against custom and shared blacklists.
+        """Detect profanity and match against custom and shared blocklists.
 
         Detects profanity in more than 100 languages and match against custom
-        and shared blacklists.
+        and shared blocklists.
 
         :param text_content_type: The content type. Possible values include:
          'text/plain', 'text/html', 'text/xml', 'text/markdown'

diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_retry.py b/sdk/core/azure-core/azure/core/pipeline/policies/_retry.py
@@ -152,14 +152,14 @@ def _is_read_error(self, err):
 
     def _is_method_retryable(self, settings, request, response=None):
         """Checks if a given HTTP method should be retried upon, depending if
-        it is included on the method whitelist.
+        it is included on the method allowlist.
 
         :param dict settings: The retry settings.
         :param request: The PipelineRequest object.
         :type request: ~azure.core.pipeline.PipelineRequest
         :param response: The PipelineResponse object.
         :type response: ~azure.core.pipeline.PipelineResponse
-        :return: True if method should be retried upon. False if not in method whitelist.
+        :return: True if method should be retried upon. False if not in method allowlist.
         :rtype: bool
         """
         if response and request.method.upper() in ['POST', 'PATCH'] and \
@@ -173,7 +173,7 @@ def _is_method_retryable(self, settings, request, response=None):
     def is_retry(self, settings, response):
         """Checks if method/status code is retryable.
 
-        Based on whitelists and control variables such as the number of
+        Based on allowlists and control variables such as the number of
         total retries to allow, whether to respect the Retry-After header,
         whether this header is present, and whether the returned status
         code is on the list of status codes to be retried upon on the
@@ -244,7 +244,7 @@ def increment(self, settings, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 if hasattr(response, 'http_request') and hasattr(response, 'http_response'):

diff --git a/...azure/eventhub/extensions/checkpointstoreblobaio/_vendor/storage/blob/_shared/policies.py b/...azure/eventhub/extensions/checkpointstoreblobaio/_vendor/storage/blob/_shared/policies.py
@@ -75,7 +75,7 @@ def retry_hook(settings, **kwargs):
 
 
 def is_retry(response, mode):
-    """Is this method/status code retryable? (Based on whitelists and control
+    """Is this method/status code retryable? (Based on allowlists and control
     variables such as the number of total retries to allow, whether to
     respect the Retry-After header, whether this header is present, and
     whether the returned status code is on the list of status codes to
@@ -456,7 +456,7 @@ def increment(self, settings, request, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 settings['history'].append(RequestHistory(request, http_response=response))

diff --git a/...ob/azure/eventhub/extensions/checkpointstoreblob/_vendor/storage/blob/_shared/policies.py b/...ob/azure/eventhub/extensions/checkpointstoreblob/_vendor/storage/blob/_shared/policies.py
@@ -75,7 +75,7 @@ def retry_hook(settings, **kwargs):
 
 
 def is_retry(response, mode):
-    """Is this method/status code retryable? (Based on whitelists and control
+    """Is this method/status code retryable? (Based on allowlists and control
     variables such as the number of total retries to allow, whether to
     respect the Retry-After header, whether this header is present, and
     whether the returned status code is on the list of status codes to
@@ -456,7 +456,7 @@ def increment(self, settings, request, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 settings['history'].append(RequestHistory(request, http_response=response))

diff --git a/...toretable/azure/eventhub/extensions/checkpointstoretable/_vendor/data/tables/_policies.py b/...toretable/azure/eventhub/extensions/checkpointstoretable/_vendor/data/tables/_policies.py
@@ -133,7 +133,7 @@ def __init__(self, **kwargs):
         self.retry_to_secondary = kwargs.get('retry_to_secondary', False)
 
     def is_retry(self, settings, response):
-        """Is this method/status code retryable? (Based on whitelists and control
+        """Is this method/status code retryable? (Based on allowlists and control
         variables such as the number of total retries to allow, whether to
         respect the Retry-After header, whether this header is present, and
         whether the returned status code is on the list of status codes to

diff --git a/...azure/eventhub/extensions/checkpointstoretable/_vendor/data/tables/aio/_policies_async.py b/...azure/eventhub/extensions/checkpointstoretable/_vendor/data/tables/aio/_policies_async.py
@@ -56,7 +56,7 @@ def __init__(self, **kwargs):
         self.retry_to_secondary = kwargs.get('retry_to_secondary', False)
 
     def is_retry(self, settings, response):
-        """Is this method/status code retryable? (Based on whitelists and control
+        """Is this method/status code retryable? (Based on allowlists and control
         variables such as the number of total retries to allow, whether to
         respect the Retry-After header, whether this header is present, and
         whether the returned status code is on the list of status codes to

diff --git a/sdk/eventhub/test-resources.json b/sdk/eventhub/test-resources.json
@@ -59,7 +59,7 @@
       "type": "string",
       "defaultValue": "10",
       "metadata": {
-        "description": "The maximum duration, in minutes, that a single test is permitted to run before it is considered at-risk for being hung."
+        "description": "The maximum duration, in minutes, that a single test is permitted to run before it is considered at-risk of not responding"
       }
     }
   },

diff --git a/sdk/search/azure-search-documents/azure/search/documents/_search_client.py b/sdk/search/azure-search-documents/azure/search/documents/_search_client.py
@@ -132,7 +132,7 @@ def get_document(self, key, selected_fields=None, **kwargs):
 
         :param key: The primary key value for the document to retrieve
         :type key: str
-        :param selected_fields: a whitelist of fields to include in the results
+        :param selected_fields: a allowlist of fields to include in the results
         :type selected_fields: List[str]
         :rtype:  dict
 

diff --git a/sdk/search/azure-search-documents/azure/search/documents/aio/_search_client_async.py b/sdk/search/azure-search-documents/azure/search/documents/aio/_search_client_async.py
@@ -109,7 +109,7 @@ async def get_document(self, key, selected_fields=None, **kwargs):
 
         :param key: The primary key value for the document to retrieve
         :type key: str
-        :param selected_fields: a whitelist of fields to include in the results
+        :param selected_fields: a allowlist of fields to include in the results
         :type selected_fields: List[str]
         :rtype:  dict
 

diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies.py
@@ -75,7 +75,7 @@ def retry_hook(settings, **kwargs):
 
 
 def is_retry(response, mode):
-    """Is this method/status code retryable? (Based on whitelists and control
+    """Is this method/status code retryable? (Based on allowlists and control
     variables such as the number of total retries to allow, whether to
     respect the Retry-After header, whether this header is present, and
     whether the returned status code is on the list of status codes to
@@ -456,7 +456,7 @@ def increment(self, settings, request, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 settings['history'].append(RequestHistory(request, http_response=response))

diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies.py
@@ -75,7 +75,7 @@ def retry_hook(settings, **kwargs):
 
 
 def is_retry(response, mode):
-    """Is this method/status code retryable? (Based on whitelists and control
+    """Is this method/status code retryable? (Based on allowlists and control
     variables such as the number of total retries to allow, whether to
     respect the Retry-After header, whether this header is present, and
     whether the returned status code is on the list of status codes to
@@ -456,7 +456,7 @@ def increment(self, settings, request, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 settings['history'].append(RequestHistory(request, http_response=response))

diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies.py
@@ -75,7 +75,7 @@ def retry_hook(settings, **kwargs):
 
 
 def is_retry(response, mode):
-    """Is this method/status code retryable? (Based on whitelists and control
+    """Is this method/status code retryable? (Based on allowlists and control
     variables such as the number of total retries to allow, whether to
     respect the Retry-After header, whether this header is present, and
     whether the returned status code is on the list of status codes to
@@ -456,7 +456,7 @@ def increment(self, settings, request, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 settings['history'].append(RequestHistory(request, http_response=response))

diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies.py
@@ -75,7 +75,7 @@ def retry_hook(settings, **kwargs):
 
 
 def is_retry(response, mode):
-    """Is this method/status code retryable? (Based on whitelists and control
+    """Is this method/status code retryable? (Based on allowlists and control
     variables such as the number of total retries to allow, whether to
     respect the Retry-After header, whether this header is present, and
     whether the returned status code is on the list of status codes to
@@ -456,7 +456,7 @@ def increment(self, settings, request, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 settings['history'].append(RequestHistory(request, http_response=response))

diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_policies.py b/sdk/tables/azure-data-tables/azure/data/tables/_policies.py
@@ -133,7 +133,7 @@ def __init__(self, **kwargs):
         self.retry_to_secondary = kwargs.get('retry_to_secondary', False)
 
     def is_retry(self, settings, response):
-        """Is this method/status code retryable? (Based on whitelists and control
+        """Is this method/status code retryable? (Based on allowlists and control
         variables such as the number of total retries to allow, whether to
         respect the Retry-After header, whether this header is present, and
         whether the returned status code is on the list of status codes to

diff --git a/sdk/tables/azure-data-tables/azure/data/tables/aio/_policies_async.py b/sdk/tables/azure-data-tables/azure/data/tables/aio/_policies_async.py
@@ -56,7 +56,7 @@ def __init__(self, **kwargs):
         self.retry_to_secondary = kwargs.get('retry_to_secondary', False)
 
     def is_retry(self, settings, response):
-        """Is this method/status code retryable? (Based on whitelists and control
+        """Is this method/status code retryable? (Based on allowlists and control
         variables such as the number of total retries to allow, whether to
         respect the Retry-After header, whether this header is present, and
         whether the returned status code is on the list of status codes to