Skip to content

Commit

Permalink
[DataMovement] Adding Tests for Recorded File Upload (Add fix Creatin…
Browse files Browse the repository at this point in the history
…g File at Initial Start) (#39037)

* Added Single Share File Upload Tests; Create before putrange

* WIP

* Removed testbase for startTransfer upload test base; other pr comments
  • Loading branch information
amnguye committed Oct 3, 2023
1 parent 441ae5f commit f7270c8
Show file tree
Hide file tree
Showing 10 changed files with 1,046 additions and 44 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
Azure.Storage.Blobs.BlobClientOptions.ServiceVersion;
#elif BlobDataMovementSDK
Azure.Storage.Blobs.BlobClientOptions.ServiceVersion;
#elif ShareDataMovementSDK
Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion;
#else
// If you see this error, you've included this shared source file from a
// client library that it doesn't know how to help you with. Either add
Expand All @@ -39,7 +41,7 @@ internal static class StorageVersionExtensions
/// Gets the latest version of the service supported by this SDK.
/// </summary>
public const ServiceVersion LatestVersion =
#if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK|| BlobDataMovementSDK
#if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK|| BlobDataMovementSDK || ShareDataMovementSDK
ServiceVersion.V2023_11_03;
#else
ERROR_STORAGE_SERVICE_NOT_DEFINED;
Expand All @@ -49,7 +51,7 @@ internal static class StorageVersionExtensions
/// Gets the latest version of the service supported by this SDK.
/// </summary>
internal const ServiceVersion MaxVersion =
#if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK|| BlobDataMovementSDK
#if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK|| BlobDataMovementSDK || ShareDataMovementSDK
ServiceVersion.V2023_11_03;
#else
ERROR_STORAGE_SERVICE_NOT_DEFINED;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "net",
"TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
"Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_b5e6d0b779"
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Azure.Core;
Expand Down Expand Up @@ -55,6 +56,32 @@ internal ShareFileStorageResource(
_etagDownloadLock = etagLock;
}

internal async Task CreateAsync(
bool overwrite,
long maxSize,
CancellationToken cancellationToken)
{
if (!overwrite)
{
// If overwrite is not enabled, we should check if the
// file exists first before creating because Create call will
// automatically overwrite the file if it already exists.
Response<bool> exists = await ShareFileClient.ExistsAsync(cancellationToken).ConfigureAwait(false);
if (exists.Value)
{
throw Errors.ShareFileAlreadyExists(ShareFileClient.Path);
}
}
await ShareFileClient.CreateAsync(
maxSize: maxSize,
httpHeaders: _options?.HttpHeaders,
metadata: _options?.FileMetadata,
smbProperties: _options?.SmbProperties,
filePermission: _options?.FilePermissions,
conditions: _options?.DestinationConditions,
cancellationToken: cancellationToken).ConfigureAwait(false);
}

protected override Task CompleteTransferAsync(
bool overwrite,
CancellationToken cancellationToken = default)
Expand All @@ -72,11 +99,21 @@ protected override async Task CopyBlockFromUriAsync(
CancellationToken cancellationToken = default)
{
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);

if (range.Offset == 0)
{
await CreateAsync(overwrite, completeLength, cancellationToken).ConfigureAwait(false);
if (range.Length == 0)
{
return;
}
}

await ShareFileClient.UploadRangeFromUriAsync(
sourceUri: sourceResource.Uri,
range: range,
sourceRange: range,
options: _options.ToShareFileUploadRangeFromUriOptions(),
options: _options?.ToShareFileUploadRangeFromUriOptions(),
cancellationToken: cancellationToken).ConfigureAwait(false);
}

Expand All @@ -91,21 +128,22 @@ protected override async Task CopyFromStreamAsync(
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);

long position = options?.Position != default ? options.Position.Value : 0;
if ((streamLength == completeLength) && position == 0)

// Create the File beforehand if it hasn't been created
if (position == 0)
{
// Default to Upload
await ShareFileClient.UploadAsync(
stream,
_options.ToShareFileUploadOptions(),
cancellationToken: cancellationToken).ConfigureAwait(false);
return;
await CreateAsync(overwrite, completeLength, cancellationToken).ConfigureAwait(false);
if (completeLength == 0)
{
return;
}
}

// Otherwise upload the Range
await ShareFileClient.UploadRangeAsync(
new HttpRange(position, streamLength),
stream,
_options.ToShareFileUploadRangeOptions(),
_options?.ToShareFileUploadRangeOptions(),
cancellationToken).ConfigureAwait(false);
}

Expand All @@ -121,7 +159,7 @@ await ShareFileClient.UploadRangeFromUriAsync(
sourceUri: sourceResource.Uri,
range: new HttpRange(0, completeLength),
sourceRange: new HttpRange(0, completeLength),
options: _options.ToShareFileUploadRangeFromUriOptions(),
options: _options?.ToShareFileUploadRangeFromUriOptions(),
cancellationToken: cancellationToken).ConfigureAwait(false);
}

Expand Down Expand Up @@ -156,7 +194,7 @@ protected override async Task<StorageResourceReadStreamResult> ReadStreamAsync(
{
CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
Response<ShareFileDownloadInfo> response = await ShareFileClient.DownloadAsync(
_options.ToShareFileDownloadOptions(new HttpRange(position, length)),
_options?.ToShareFileDownloadOptions(new HttpRange(position, length)),
cancellationToken).ConfigureAwait(false);
return response.Value.ToStorageResourceReadStreamResult();
}
Expand All @@ -171,4 +209,12 @@ protected override StorageResourceCheckpointData GetDestinationCheckpointData()
throw new NotImplementedException();
}
}

#pragma warning disable SA1402 // File may only contain a single type
internal partial class Errors
#pragma warning restore SA1402 // File may only contain a single type
{
public static InvalidOperationException ShareFileAlreadyExists(string pathName)
=> new InvalidOperationException($"Share File `{pathName}` already exists. Cannot overwrite file.");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,16 @@
<!-- Required Shared Source for testing -->
<Compile Include="$(AzureStorageSharedSources)Constants.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)Errors.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)Errors.Clients.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StorageConnectionString.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StorageVersionExtensions.cs" LinkBase="Shared\Storage" />
<Compile Include="$(AzureStorageSharedSources)SharedAccessSignatureCredentials.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)UriQueryParamsCollection.cs" LinkBase="Shared" />
</ItemGroup>
<ItemGroup>
<Compile Include="$(AzureStorageSharedTestSources)\**\*.cs" LinkBase="Shared\Storage" />
<Compile Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Files.Shares\tests\DisposingShare.cs" LinkBase="Shared\Storage" />
<Compile Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Files.Shares\tests\ShareClientTestFixtureAttribute.cs" LinkBase="Shared\Storage" />
<Compile Remove="$(AzureStorageSharedTestSources)\AzuriteFixture.cs"/>
<Compile Remove="$(AzureStorageSharedTestSources)\AzuriteNUnitFixture.cs"/>
<Compile Remove="$(AzureStorageSharedTestSources)\ClientSideEncryptionTestExtensions.cs" />
Expand All @@ -33,6 +37,14 @@
</None>
<None Include="$(AzureStorageSharedTestSources)\azurite_cert.pem" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
<Compile Include="$(AzureStorageDataMovementSharedSources)DataTransferStatusInternal.cs" LinkBase="Shared\DataMovement" />
</ItemGroup>
<ItemGroup>
<Compile Include="$(AzureStorageDataMovementTestSharedSources)TestEventsRaised.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)DisposingLocalDirectory.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferUploadTestBase.cs" LinkBase="Shared\DataMovement" />
</ItemGroup>
<ItemGroup>
<Content Include="Resources\**">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Azure.Storage.Test.Shared;
using Azure.Storage.Files.Shares;
using Azure.Storage.Files.Shares.Tests;
using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder<
Azure.Storage.Files.Shares.ShareServiceClient,
Azure.Storage.Files.Shares.ShareClientOptions>;

namespace Azure.Storage.DataMovement.Files.Shares.Tests
{
internal static class ClientBuilderExtensions
{
public static string GetNewShareName(this SharesClientBuilder clientBuilder)
=> $"test-share-{clientBuilder.Recording.Random.NewGuid()}";
public static string GetNewDirectoryName(this SharesClientBuilder clientBuilder)
=> $"test-directory-{clientBuilder.Recording.Random.NewGuid()}";
public static string GetNewNonAsciiDirectoryName(this SharesClientBuilder clientBuilder)
=> $"test-dire¢t Ø®ϒ%3A-{clientBuilder.Recording.Random.NewGuid()}";
public static string GetNewFileName(this SharesClientBuilder clientBuilder)
=> $"test-file-{clientBuilder.Recording.Random.NewGuid()}";
public static string GetNewNonAsciiFileName(this SharesClientBuilder clientBuilder)
=> $"test-ƒ¡£€‽%3A-{clientBuilder.Recording.Random.NewGuid()}";

/// <summary>
/// Creates a new <see cref="ClientBuilder{TServiceClient, TServiceClientOptions}"/>
/// setup to generate <see cref="BlobServiceClient"/>s.
/// </summary>
/// <param name="tenants"><see cref="TenantConfigurationBuilder"/> powering this client builder.</param>
/// <param name="serviceVersion">Service version for clients to target.</param>
public static SharesClientBuilder GetNewShareClientBuilder(TenantConfigurationBuilder tenants, ShareClientOptions.ServiceVersion serviceVersion)
=> new SharesClientBuilder(
ServiceEndpoint.File,
tenants,
(uri, clientOptions) => new ShareServiceClient(uri, clientOptions),
(uri, sharedKeyCredential, clientOptions) => new ShareServiceClient(uri, sharedKeyCredential, clientOptions),
(uri, tokenCredential, clientOptions) => new ShareServiceClient(uri, tokenCredential, clientOptions),
(uri, azureSasCredential, clientOptions) => new ShareServiceClient(uri, azureSasCredential, clientOptions),
() => new ShareClientOptions(serviceVersion));

public static async Task<DisposingShare> GetTestShareAsync(
this SharesClientBuilder clientBuilder,
ShareServiceClient service = default,
string shareName = default,
IDictionary<string, string> metadata = default,
ShareClientOptions options = default)
{
service ??= clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigDefault, options);
metadata ??= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
shareName ??= clientBuilder.GetNewShareName();
ShareClient share = clientBuilder.AzureCoreRecordedTestBase.InstrumentClient(service.GetShareClient(shareName));
return await DisposingShare.CreateAsync(share, metadata);
}
}
}
Loading

0 comments on commit f7270c8

Please sign in to comment.