From 749a3e29da90c0cf6c6fb1d91266a722729ea968 Mon Sep 17 00:00:00 2001 From: Ethan Celletti Date: Wed, 17 Apr 2024 08:46:57 -0700 Subject: [PATCH] Adding AcceptEncodings override (#127) * Adding AcceptEncodings override * Fixing docs --- samples/Sample.CLI/Program.cs | 37 ++++++++++++- src/Agent/API.xml | 3 +- .../AssetCanister/AssetCanisterApiClient.cs | 52 ++++--------------- 3 files changed, 48 insertions(+), 44 deletions(-) diff --git a/samples/Sample.CLI/Program.cs b/samples/Sample.CLI/Program.cs index fc53be7..0eb3864 100644 --- a/samples/Sample.CLI/Program.cs +++ b/samples/Sample.CLI/Program.cs @@ -19,6 +19,7 @@ using EdjCase.ICP.Candid.Utilities; using EdjCase.ICP.BLS.Models; using System.Diagnostics; +using System.IO.Compression; public class Program { @@ -162,7 +163,41 @@ string outputFilePath AssetCanisterApiClient client = new(this.agent, canisterId); Console.WriteLine($"Downloading asset '{key}'..."); - byte[] assetBytes = await client.DownloadAssetAsync(key); + (byte[] assetBytes, string contentEncoding) = await client.DownloadAssetAsync(key); + switch (contentEncoding) + { + case "identity": + break; + case "gzip": + using (var memoryStream = new MemoryStream(assetBytes)) + using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) + using (var decompressedStream = new MemoryStream()) + { + gzipStream.CopyTo(decompressedStream); + assetBytes = decompressedStream.ToArray(); + } + break; + case "deflate": + using (var memoryStream = new MemoryStream(assetBytes)) + using (var deflateStream = new DeflateStream(memoryStream, CompressionMode.Decompress)) + using (var decompressedStream = new MemoryStream()) + { + deflateStream.CopyTo(decompressedStream); + assetBytes = decompressedStream.ToArray(); + } + break; + case "br": + using (var memoryStream = new MemoryStream(assetBytes)) + using (var brotliStream = new BrotliStream(memoryStream, CompressionMode.Decompress)) + using (var decompressedStream = new MemoryStream()) + { + brotliStream.CopyTo(decompressedStream); + assetBytes = decompressedStream.ToArray(); + } + break; + default: + throw new NotImplementedException($"Content encoding {contentEncoding} is not implemented"); + } File.WriteAllBytes(outputFilePath, assetBytes); Console.WriteLine($"Downloaded asset '{key}' to {outputFilePath}"); } diff --git a/src/Agent/API.xml b/src/Agent/API.xml index 653a75b..8de8b54 100644 --- a/src/Agent/API.xml +++ b/src/Agent/API.xml @@ -1249,12 +1249,13 @@ Additional headers to be included in the request. The maximum age of the asset in seconds. - + A helper method to download an asset from the asset canister in chunks. The key of the asset to download. The maximum number of concurrent chunk downloads. + The encodings to accept for the asset in order. Defaults to ["gzip", "deflate", "br", "identity"] The downloaded asset content as a byte array. diff --git a/src/Agent/Standards/AssetCanister/AssetCanisterApiClient.cs b/src/Agent/Standards/AssetCanister/AssetCanisterApiClient.cs index 5c7d6ac..6b8879e 100644 --- a/src/Agent/Standards/AssetCanister/AssetCanisterApiClient.cs +++ b/src/Agent/Standards/AssetCanister/AssetCanisterApiClient.cs @@ -187,10 +187,15 @@ public async Task UploadAssetChunkedAsync( /// /// The key of the asset to download. /// The maximum number of concurrent chunk downloads. + /// The encodings to accept for the asset in order. Defaults to ["gzip", "deflate", "br", "identity"] /// The downloaded asset content as a byte array. - public async Task DownloadAssetAsync(string key, int maxConcurrency = 10) + public async Task<(byte[] Asset, string ContentEncoding)> DownloadAssetAsync( + string key, + int maxConcurrency = 10, + List? acceptEncodings = null + ) { - List acceptEncodings = new() { "identity", "gzip", "deflate", "br" }; + acceptEncodings ??= new() { "gzip", "deflate", "br", "identity" }; GetResult result = await this.GetAsync(key, acceptEncodings); if (!result.TotalLength.TryToUInt64(out ulong totalLength)) @@ -199,12 +204,12 @@ public async Task DownloadAssetAsync(string key, int maxConcurrency = 10 } if (totalLength == (ulong)result.Content.Length) { - return result.Content; + return (result.Content, result.ContentEncoding); } int chunkCount = (int)Math.Ceiling((double)totalLength / result.Content.Length); // Create a list to store the chunk tasks - List> chunkTasks = new List>(); + List> chunkTasks = new(); // Create a semaphore to limit the number of concurrent tasks SemaphoreSlim semaphore = new(maxConcurrency); @@ -236,44 +241,7 @@ public async Task DownloadAssetAsync(string key, int maxConcurrency = 10 // Combine all the bytes into one byte[] byte[] combinedBytes = result.Content.Concat(chunkTasks.SelectMany(t => t.Result)).ToArray(); - switch (result.ContentEncoding) - { - case "identity": - case null: - case "": - break; - case "gzip": - using (var memoryStream = new MemoryStream(combinedBytes)) - using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) - using (var decompressedStream = new MemoryStream()) - { - gzipStream.CopyTo(decompressedStream); - combinedBytes = decompressedStream.ToArray(); - } - break; - case "deflate": - using (var memoryStream = new MemoryStream(combinedBytes)) - using (var deflateStream = new DeflateStream(memoryStream, CompressionMode.Decompress)) - using (var decompressedStream = new MemoryStream()) - { - deflateStream.CopyTo(decompressedStream); - combinedBytes = decompressedStream.ToArray(); - } - break; - case "br": - using (var memoryStream = new MemoryStream(combinedBytes)) - using (var brotliStream = new BrotliStream(memoryStream, CompressionMode.Decompress)) - using (var decompressedStream = new MemoryStream()) - { - brotliStream.CopyTo(decompressedStream); - combinedBytes = decompressedStream.ToArray(); - } - break; - default: - throw new NotImplementedException($"Content encoding {result.ContentEncoding} is not supported"); - } - - return combinedBytes; + return (combinedBytes, result.ContentEncoding); } ///