Memory tuning

src/ReleaseHistory.md

@@ -1,5 +1,8 @@
 # SARIF Package Release History (SDK, Driver, Converters, and Multitool)
 
+## **v4.1.0** UNRELEASED
+* NEW: Add `PeakWorkingSet` to `--trace` command to report maximum working set value during analysis. [#2619](https://github.com/microsoft/sarif-sdk/pull/2619)
+
 ## **v4.0.0** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/4.0.0) | [Driver](https://www.nuget.org/packages/Sarif.Driver/4.0.0) | [Converters](https://www.nuget.org/packages/Sarif.Converters/4.0.0) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/4.0.0) | [Multitool Library](https://www.nuget.org/packages/Sarif.Multitool.Library/4.0.0)
 * BRK: `SarifLogger` no longer allows providing a `Tool` instance. Use the `run` parameter instead (and populate it with any custom `Tool` object). [#2614](https://github.com/microsoft/sarif-sdk/pull/2614)
 * BRK: `SarifLogger` updates version details differently. [#2611](https://github.com/microsoft/sarif-sdk/pull/2611)

src/Samples/Sarif.Sdk.Sample/Program.cs

@@ -329,8 +329,8 @@ private static void SerializeSarifResult(string filePath, int numResult, bool us
                         OptionallyEmittedData.Hashes |
                         OptionallyEmittedData.RegionSnippets,
                     run: run,
-                    kinds: new ResultKind[] { ResultKind.Fail, ResultKind.Pass },
-                    levels: new FailureLevel[] { FailureLevel.Error, FailureLevel.Warning, FailureLevel.Note },
+                    kinds: BaseLogger.Fail,
+                    levels: BaseLogger.ErrorWarning,
                     analysisTargets: null,
                     invocationTokensToRedact: null,
                     invocationPropertiesToLog: null,

src/Samples/SarifTrim/SarifTrim.csproj

@@ -37,6 +37,10 @@
       <HintPath>..\..\..\packages\Newtonsoft.Json.6.0.8\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
     <Reference Include="System" />
+    <Reference Include="System.Collections.Immutable, Version=1.2.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\..\..\packages\System.Collections.Immutable.1.2.3\lib\netstandard2.0\System.Collections.Immutable.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="System.Core" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />

src/Samples/SarifTrim/packages.config

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net472" />
+  <package id="System.Collections.Immutable" version="1.5.0" targetFramework="net472" />
 </packages>

src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs

@@ -30,7 +30,6 @@ public abstract class AnalyzeCommandBase<TContext, TOptions> : PluginDriverComma
         private Run _run;
         private Tool _tool;
         internal TContext _rootContext;
-        private CacheByFileHashLogger _cacheByFileHashLogger;
         private IDictionary<string, HashData> _pathToHashDataMap;
 
         public Exception ExecutionException { get; set; }
@@ -225,7 +224,7 @@ protected virtual bool ShouldEnqueue(string file, TContext context)
 
             if (!shouldEnqueue)
             {
-                Warnings.LogFileSkippedDueToSize(context, file, fileSizeInKb);
+                Notes.LogFileSkippedDueToSize(context, file, fileSizeInKb);
             }
 
             return shouldEnqueue;
@@ -239,16 +238,10 @@ internal AggregatingLogger InitializeLogger(AnalyzeOptionsBase analyzeOptions)
 
             if (!analyzeOptions.Quiet)
             {
-                _consoleLogger = new ConsoleLogger(analyzeOptions.Quiet, _tool.Driver.Name, analyzeOptions.Level, analyzeOptions.Kind) { CaptureOutput = _captureConsoleOutput };
+                _consoleLogger = new ConsoleLogger(analyzeOptions.Quiet, _tool.Driver.Name, analyzeOptions.FailureLevels, analyzeOptions.ResultKinds) { CaptureOutput = _captureConsoleOutput };
                 logger.Loggers.Add(_consoleLogger);
             }
 
-            if ((analyzeOptions.DataToInsert.ToFlags() & OptionallyEmittedData.Hashes) != 0)
-            {
-                _cacheByFileHashLogger = new CacheByFileHashLogger(analyzeOptions.Level, analyzeOptions.Kind);
-                logger.Loggers.Add(_cacheByFileHashLogger);
-            }
-
             return logger;
         }
 
@@ -305,10 +298,7 @@ protected virtual TContext CreateContext(
                 Policy = policy ?? new PropertiesDictionary()
             };
 
-            context.Traces =
-                options.Traces.Any() ?
-                    (DefaultTraces)Enum.Parse(typeof(DefaultTraces), string.Join(",", options.Traces)) :
-                    DefaultTraces.None;
+            context.Traces = options.Traces;
 
             context.MaxFileSizeInKilobytes =
                 options.MaxFileSizeInKilobytes >= 0
@@ -427,8 +417,8 @@ private void InitializeOutputFile(TOptions analyzeOptions, TContext context)
                                     quiet: analyzeOptions.Quiet,
                                     invocationTokensToRedact: GenerateSensitiveTokensList(),
                                     invocationPropertiesToLog: analyzeOptions.InvocationPropertiesToLog,
-                                    levels: analyzeOptions.Level,
-                                    kinds: analyzeOptions.Kind);
+                                    levels: analyzeOptions.FailureLevels,
+                                    kinds: analyzeOptions.ResultKinds);
                         }
                         else
                         {
@@ -441,8 +431,8 @@ private void InitializeOutputFile(TOptions analyzeOptions, TContext context)
                                     analysisTargets: null,
                                     invocationTokensToRedact: GenerateSensitiveTokensList(),
                                     invocationPropertiesToLog: analyzeOptions.InvocationPropertiesToLog,
-                                    levels: analyzeOptions.Level,
-                                    kinds: analyzeOptions.Kind);
+                                    levels: analyzeOptions.FailureLevels,
+                                    kinds: analyzeOptions.ResultKinds);
                         }
                         _pathToHashDataMap = sarifLogger.AnalysisTargetToHashDataMap;
                         sarifLogger.AnalysisStarted();
@@ -628,42 +618,6 @@ protected virtual TContext DetermineApplicabilityAndAnalyze(
                 rootContext.Policy,
                 target);
 
-            if ((options.DataToInsert.ToFlags() & OptionallyEmittedData.Hashes) != 0)
-            {
-                _cacheByFileHashLogger.HashToResultsMap.TryGetValue(context.Hashes.Sha256, out List<Tuple<ReportingDescriptor, Result, int?>> cachedResultTuples);
-                _cacheByFileHashLogger.HashToNotificationsMap.TryGetValue(context.Hashes.Sha256, out List<Notification> cachedNotifications);
-
-                bool replayCachedData = (cachedResultTuples != null || cachedNotifications != null);
-
-                if (replayCachedData)
-                {
-                    context.Logger.AnalyzingTarget(context);
-
-                    if (cachedResultTuples != null)
-                    {
-                        foreach (Tuple<ReportingDescriptor, Result, int?> cachedResultTuple in cachedResultTuples)
-                        {
-                            Result clonedResult = cachedResultTuple.Item2.DeepClone();
-                            ReportingDescriptor cachedReportingDescriptor = cachedResultTuple.Item1;
-
-                            UpdateLocationsAndMessageWithCurrentUri(clonedResult.Locations, clonedResult.Message, context.TargetUri);
-                            context.Logger.Log(cachedReportingDescriptor, clonedResult, cachedResultTuple.Item3);
-                        }
-                    }
-
-                    if (cachedNotifications != null)
-                    {
-                        foreach (Notification cachedNotification in cachedNotifications)
-                        {
-                            Notification clonedNotification = cachedNotification.DeepClone();
-                            UpdateLocationsAndMessageWithCurrentUri(clonedNotification.Locations, cachedNotification.Message, context.TargetUri);
-                            context.Logger.LogConfigurationNotification(clonedNotification);
-                        }
-                    }
-                    return context;
-                }
-            }
-
             if (context.TargetLoadException != null)
             {
                 Errors.LogExceptionLoadingTarget(context);

src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs

@@ -3,9 +3,13 @@
 
 using System;
 using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
 
 using CommandLine;
 
+using Microsoft.CodeAnalysis.Sarif.Writers;
+
 namespace Microsoft.CodeAnalysis.Sarif.Driver
 {
     [Verb("analyze", HelpText = "Analyze one or more binary files for security and correctness issues.")]
@@ -15,9 +19,9 @@ public AnalyzeOptionsBase()
         {
             // TODO: these defaults need to be converted to the configuration
             // property pattern as followed by MaxFileSizeInKilobytes.
-            Traces = new string[] { };
-            Kind = new List<ResultKind> { ResultKind.Fail };
-            Level = new List<FailureLevel> { FailureLevel.Warning, FailureLevel.Error };
+            Trace = new string[] { };
+            Kind = BaseLogger.Fail;
+            Level = BaseLogger.ErrorWarning;
 
             MaxFileSizeInKilobytes = AnalyzeContextBase.MaxFileSizeInKilobytesDefaultValue;
         }
@@ -100,7 +104,21 @@ public AnalyzeOptionsBase()
             HelpText = "Execution traces, expressed as a semicolon-delimited list, that " +
                        "should be emitted to the console and log file (if appropriate). " +
                        "Valid values: ScanTime.")]
-        public virtual IEnumerable<string> Traces { get; set; }
+        public virtual IEnumerable<string> Trace { get; set; }
+
+        private DefaultTraces? defaultTraces;
+        public DefaultTraces Traces
+        {
+            get
+            {
+                defaultTraces ??=
+                    this.Trace.Any()
+                        ? (DefaultTraces)Enum.Parse(typeof(DefaultTraces), string.Join(",", this.Trace))
+                        : DefaultTraces.None;
+
+                return this.defaultTraces.Value;
+            }
+        }
 
         [Option(
             "level",
@@ -109,13 +127,33 @@ public AnalyzeOptionsBase()
             HelpText = "A semicolon delimited list to filter output of scan results to one or more failure levels. Valid values: Error, Warning and Note.")]
         public IEnumerable<FailureLevel> Level { get; set; }
 
+        private IImmutableSet<FailureLevel> failureLevels;
+        public IImmutableSet<FailureLevel> FailureLevels
+        {
+            get
+            {
+                this.failureLevels ??= new List<FailureLevel>(Level).ToImmutableHashSet();
+                return this.failureLevels;
+            }
+        }
+
         [Option(
             "kind",
             Separator = ';',
             Default = new ResultKind[] { ResultKind.Fail },
             HelpText = "A semicolon delimited list to filter output to one or more result kinds. Valid values: Fail (for literal scan results), Pass, Review, Open, NotApplicable and Informational.")]
         public IEnumerable<ResultKind> Kind { get; set; }
 
+        private IImmutableSet<ResultKind> resultKinds;
+        public IImmutableSet<ResultKind> ResultKinds
+        {
+            get
+            {
+                this.resultKinds ??= new List<ResultKind>(Kind).ToImmutableHashSet();
+                return this.resultKinds;
+            }
+        }
+
         [Option(
             "baseline",
             HelpText = "A SARIF file to be used as baseline.")]
@@ -131,5 +169,12 @@ public AnalyzeOptionsBase()
             HelpText = "The maximum file size (in kilobytes) that will be analyzed.",
             Default = -1)]
         public long MaxFileSizeInKilobytes { get; set; }
+
+        [Option(
+            "max-targets",
+            HelpText = "The maximum number of targets that will be prepared for analysis. Use this to limit memory usage.",
+            Default = 25000)]
+        public int MaxTargets { get; set; }
+
     }
 }