Download inputs from previous jobs + improve resilience

MihuBot · May 23, 2024 · 91e22a7 · 91e22a7
1 parent b791ad2
commit 91e22a7
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 13 deletions.
diff --git a/Runner/FuzzLibrariesJob.cs b/Runner/FuzzLibrariesJob.cs
@@ -1,4 +1,5 @@
-using System.Text.RegularExpressions;
+using System.IO.Compression;
+using System.Text.RegularExpressions;
 
 namespace Runner;
 
@@ -101,6 +102,7 @@ private async Task RunFuzzersAsync(string fuzzerNamePattern)
             int remainingFuzzers = matchingFuzzers.Length - i;
             TimeSpan remainingTime = MaxJobDuration - JobStopwatch.Elapsed - TimeSpan.FromMinutes(15);
             int durationSeconds = (int)(remainingTime / remainingFuzzers).TotalSeconds;
+            durationSeconds = Math.Min(3600, durationSeconds);
 
             ArgumentOutOfRangeException.ThrowIfLessThan(durationSeconds, 60);
 
@@ -116,17 +118,28 @@ private async Task<bool> RunFuzzerAsync(string fuzzerName, int durationSeconds)
 
         string fuzzerDirectory = $"{DeploymentPath}/{fuzzerName}";
         string inputsDirectory = $"{fuzzerName}-inputs";
+        string artifactPathPrefix = $"{fuzzerName}-artifact-";
 
         Directory.CreateDirectory(inputsDirectory);
 
-        string artifactPathPrefix = $"{fuzzerName}-artifact-";
+        try
+        {
+            var remoteInputsZipBlob = PersistentStateClient.GetBlobClient($"{inputsDirectory}.zip");
+            if (await remoteInputsZipBlob.ExistsAsync(JobTimeout))
+            {
+                var content = (await remoteInputsZipBlob.DownloadContentAsync(JobTimeout)).Value;
+                ZipFile.ExtractToDirectory(content.Content.ToStream(), inputsDirectory);
+            }
+        }
+        catch (Exception ex)
+        {
+            await LogAsync($"Failed to download previous inputs archive: {ex}");
+        }
 
         int parallelism = Environment.ProcessorCount;
-
         await LogAsync($"Starting {parallelism} parallel fuzzer instances");
 
         using CancellationTokenSource failureCts = new();
-
         int failureStackUploaded = 0;
 
         await Parallel.ForEachAsync(Enumerable.Range(1, parallelism), async (i, _) =>
@@ -173,7 +186,7 @@ await RunProcessAsync(
 
         if (Directory.EnumerateFiles(inputsDirectory).Any())
         {
-            await ZipAndUploadArtifactAsync($"{fuzzerName}-inputs", inputsDirectory);
+            await ZipAndUploadArtifactAsync(inputsDirectory, inputsDirectory);
         }
 
         return failureStackUploaded == 0;

diff --git a/Runner/JobBase.cs b/Runner/JobBase.cs
@@ -5,6 +5,7 @@
 using System.Runtime.InteropServices;
 using System.Threading.Channels;
 using System.IO.Compression;
+using Azure.Storage.Blobs;
 
 namespace Runner;
 
@@ -56,6 +57,8 @@ private bool TryGetArgument(string argument, [NotNullWhen(true)] out string? val
         return value.Length > 0;
     }
 
+    protected BlobContainerClient PersistentStateClient => new(new Uri(Metadata["PersistentStateSasUri"]));
+
     public JobBase(HttpClient client, Dictionary<string, string> metadata)
     {
         _client = client;
@@ -197,6 +200,12 @@ private async Task ReadChannelAsync()
             catch { }
         });
 
+        try
+        {
+            Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.AboveNormal;
+        }
+        catch { }
+
         try
         {
             ChannelReader<string> reader = _channel.Reader;
@@ -251,6 +260,12 @@ protected async Task RunProcessAsync(
 
         process.Start();
 
+        try
+        {
+            process.PriorityClass = ProcessPriorityClass.BelowNormal;
+        }
+        catch { }
+
         await Task.WhenAll(
             Task.Run(() => ReadOutputStreamAsync(process.StandardOutput)),
             Task.Run(() => ReadOutputStreamAsync(process.StandardError)),
@@ -319,16 +334,27 @@ protected async Task UploadArtifactAsync(string path, string? fileName = null)
             JobTimeout);
     }
 
-    private async Task PostAsJsonAsync(string path, object? value)
+    private async Task PostAsJsonAsync(string path, object? value, int attemptsRemaining = 2)
     {
-        try
-        {
-            using var response = await _client.PostAsJsonAsync($"{path}/{_jobId}", value, JobTimeout);
-        }
-        catch (Exception ex)
+        while (true)
         {
-            await LogAsync($"Failed to post resource '{path}': {ex}");
-            throw;
+            try
+            {
+                using var response = await _client.PostAsJsonAsync($"{path}/{_jobId}", value, JobTimeout);
+                response.EnsureSuccessStatusCode();
+                return;
+            }
+            catch (Exception ex) when (!JobTimeout.IsCancellationRequested)
+            {
+                await LogAsync($"Failed to post resource '{path}': {ex}");
+
+                if (--attemptsRemaining == 0)
+                {
+                    throw;
+                }
+
+                await Task.Delay(1_000, JobTimeout);
+            }
         }
     }
 

diff --git a/Runner/Runner.csproj b/Runner/Runner.csproj
@@ -8,6 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.20.0" />
     <PackageReference Include="Hardware.Info" Version="100.1.0" />
   </ItemGroup>