azblob: Upgrade to STG 85-86 (#21381)

* Regenerating AzBlob to STG 85/86 (#20724)

* Regenerating azblob to stg 85/86

* Updating CopyFromURL

* minor change

* minor fixes

* undo some minor fixes

* Updating Go code generator

* Fixing calls to pipeline

* Adding custom UnmarshalXML for BlobItem and BlobPrefix

* Updating constructor method for AppendBlobClient

* Updating Client constructors

* Undoing minor fixes to blob examples

* Fixing authpolicy

* Updating azcore version

* Fixing client strings

* Const for service client

* Minor fix

* fixing go mod files

* Shared constants client name

* Addressing comments

* [Feature STG 85/86] Cold Tier

* [Checksum Work] BlockBlob Client: Transactional/Source Content Validation (#21033)

* Adding test for transactional validation in block blob

* StageBlockFromURL tests

* Deprecating options in CommitBlockList

* CopyFromURL test

* Updating checksum behavior

* Record tests

* Updated recording

* Updated recording again

* Added error for user gen checksums, tests for UploadStream, UploadBuffer, and UploadFile

* Added recorded test

* Updated CommitBlockList, added tests for CommitBlockList, and added CRC64 test for Upload

* Updating UploadStream test

* Recorded test

* Recorded test

* Fixing CommitBlockList errors

* Fixing linting issues

* Addressing comment + handling CI issues

* Removing TransactionalValidation from CommitBlockList + cleaning up tests

* Rerecorded tests

* Update azcore (#21188)

* Updating azcore

* Updating changelog.md

* Updating go sum file

* Update perf go file

* Copy Blob from URL/Put Blob from URL copy source tags  (#21128)

* Regenerating AzBlob to STG 85/86 (#20724)

* Regenerating azblob to stg 85/86

* Updating CopyFromURL

* minor change

* minor fixes

* undo some minor fixes

* Updating Go code generator

* Fixing calls to pipeline

* Adding custom UnmarshalXML for BlobItem and BlobPrefix

* Updating constructor method for AppendBlobClient

* Updating Client constructors

* Undoing minor fixes to blob examples

* Fixing authpolicy

* Updating azcore version

* Fixing client strings

* Const for service client

* Minor fix

* fixing go mod files

* Shared constants client name

* Addressing comments

* [Feature STG 85/86] Cold Tier  (#21032)

* Adding Cold Tier + tests

* Recorded tests

* Updated CHANGELOG.md

* Fixing linting issues

* Updating Cold tier test and recording

* Addressing commits

* Adding CopySourceBlobTags

* Fixing enum variable name, adding default test, changelog update

* Cleaning up constants

* Moving const to constants.go

* Small change

* Adding PossibleBlobCopySourceTagsValues() Method for CopySourceTags

* List System Containers (#21243)

* Listing system containers

* Updating CHANGELOG.md

* Record test

* Filter Tags API on Container (#21197)

* Adding FilterBlobs to container client

* Updating and adding tests

* Updating URL

Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

* Updating comment

Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

* Updating CHANGELOG.md

* Updating test

---------

Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

* Encryption Scope SAS (#21294)

* Adding encryption sas to blob, account, and identity sas

* Fixing issues with Blob SAS

* Undo some changes

* Undo some changes pt 2

* Undo some changes pt 3

* Adding doc comment

* Updating variable names in the tests and updated account sas test

* Updating tests

* Adding back comment

* Updating CHANGELOG.md

* Update sdk/storage/azblob/sas/query_params.go

Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

* Update sdk/storage/azblob/CHANGELOG.md

Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

* Removing SI from Account SAS

---------

Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

* Test Encryption Scope support for Sync Blob Copy (SyncCopyFromUrl) (#21332)

* Test List Blobs Fix for Invalid XML Characters

* minor fixes to changelog

---------

Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com>

sdk/storage/azblob/CHANGELOG.md

@@ -3,15 +3,22 @@
 ## 1.1.1 (Unreleased)
 
 ### Features Added
+* Added support for [Cold Tier](https://learn.microsoft.com/azure/storage/blobs/access-tiers-overview?tabs=azure-portal).
+* Added `CopySourceTag` option for `UploadBlobFromURLOptions`
+* Added [FilterBlobs by Tags](https://learn.microsoft.com/rest/api/storageservices/find-blobs-by-tags-container) API for container client.
+* Added `System` option to `ListContainersInclude` to allow listing of system containers (i.e, $web).
+* Updated the SAS Version to `2021-12-02` and added `Encryption Scope` to Account SAS, Service SAS, and User Delegation SAS
 
 ### Breaking Changes
 
 ### Bugs Fixed
-* Fixed issue where some requests fail with mismatch in string to sign. 
+* Fixed issue where some requests fail with mismatch in string to sign.
+* Fixed service SAS creation where expiry time or permissions can be omitted when stored access policy is used. Fixes [#21229](https://github.com/Azure/azure-sdk-for-go/issues/21229).
 
 * Fixed service SAS creation where expiry time or permissions can be omitted when stored access policy is used. Fixes [#21229](https://github.com/Azure/azure-sdk-for-go/issues/21229).
 
 ### Other Changes
+* Updating version of azcore to 1.6.0.
 
 ## 1.1.0 (2023-07-13)
 

sdk/storage/azblob/appendblob/client.go

@@ -9,6 +9,7 @@ package appendblob
 import (
 	"context"
 	"errors"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"io"
 	"os"
 	"time"
@@ -35,12 +36,14 @@ type Client base.CompositeClient[generated.BlobClient, generated.AppendBlobClien
 func NewClient(blobURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
 	authPolicy := shared.NewStorageChallengePolicy(cred)
 	conOptions := shared.GetClientOptions(options)
-	conOptions.PerRetryPolicies = append(conOptions.PerRetryPolicies, authPolicy)
-	pl := runtime.NewPipeline(exported.ModuleName,
-		exported.ModuleVersion, runtime.PipelineOptions{},
-		&conOptions.ClientOptions)
+	plOpts := runtime.PipelineOptions{PerRetry: []policy.Policy{authPolicy}}
 
-	return (*Client)(base.NewAppendBlobClient(blobURL, pl, nil)), nil
+	azClient, err := azcore.NewClient(shared.AppendBlobClient, exported.ModuleVersion, plOpts, &conOptions.ClientOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	return (*Client)(base.NewAppendBlobClient(blobURL, azClient, nil)), nil
 }
 
 // NewClientWithNoCredential creates an instance of Client with the specified values.
@@ -49,12 +52,13 @@ func NewClient(blobURL string, cred azcore.TokenCredential, options *ClientOptio
 //   - options - client options; pass nil to accept the default values
 func NewClientWithNoCredential(blobURL string, options *ClientOptions) (*Client, error) {
 	conOptions := shared.GetClientOptions(options)
-	pl := runtime.NewPipeline(exported.ModuleName,
-		exported.ModuleVersion,
-		runtime.PipelineOptions{},
-		&conOptions.ClientOptions)
 
-	return (*Client)(base.NewAppendBlobClient(blobURL, pl, nil)), nil
+	azClient, err := azcore.NewClient(shared.AppendBlobClient, exported.ModuleVersion, runtime.PipelineOptions{}, &conOptions.ClientOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	return (*Client)(base.NewAppendBlobClient(blobURL, azClient, nil)), nil
 }
 
 // NewClientWithSharedKeyCredential creates an instance of Client with the specified values.
@@ -64,13 +68,14 @@ func NewClientWithNoCredential(blobURL string, options *ClientOptions) (*Client,
 func NewClientWithSharedKeyCredential(blobURL string, cred *blob.SharedKeyCredential, options *ClientOptions) (*Client, error) {
 	authPolicy := exported.NewSharedKeyCredPolicy(cred)
 	conOptions := shared.GetClientOptions(options)
-	conOptions.PerRetryPolicies = append(conOptions.PerRetryPolicies, authPolicy)
-	pl := runtime.NewPipeline(exported.ModuleName,
-		exported.ModuleVersion,
-		runtime.PipelineOptions{},
-		&conOptions.ClientOptions)
+	plOpts := runtime.PipelineOptions{PerRetry: []policy.Policy{authPolicy}}
+
+	azClient, err := azcore.NewClient(shared.AppendBlobClient, exported.ModuleVersion, plOpts, &conOptions.ClientOptions)
+	if err != nil {
+		return nil, err
+	}
 
-	return (*Client)(base.NewAppendBlobClient(blobURL, pl, cred)), nil
+	return (*Client)(base.NewAppendBlobClient(blobURL, azClient, cred)), nil
 }
 
 // NewClientFromConnectionString creates an instance of Client with the specified values.
@@ -130,7 +135,7 @@ func (ab *Client) WithSnapshot(snapshot string) (*Client, error) {
 	}
 	p.Snapshot = snapshot
 
-	return (*Client)(base.NewAppendBlobClient(p.String(), ab.generated().Pipeline(), ab.sharedKey())), nil
+	return (*Client)(base.NewAppendBlobClient(p.String(), ab.generated().InternalClient(), ab.sharedKey())), nil
 }
 
 // WithVersionID creates a new AppendBlobURL object identical to the source but with the specified version id.
@@ -142,7 +147,7 @@ func (ab *Client) WithVersionID(versionID string) (*Client, error) {
 	}
 	p.VersionID = versionID
 
-	return (*Client)(base.NewAppendBlobClient(p.String(), ab.generated().Pipeline(), ab.sharedKey())), nil
+	return (*Client)(base.NewAppendBlobClient(p.String(), ab.generated().InternalClient(), ab.sharedKey())), nil
 }
 
 // Create creates a 0-size append blob. Call AppendBlock to append data to an append blob.

sdk/storage/azblob/appendblob/client_test.go

@@ -12,6 +12,7 @@ import (
 	"crypto/md5"
 	"encoding/binary"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"hash/crc64"
 	"io"
 	"math/rand"
@@ -371,6 +372,149 @@ func (s *AppendBlobUnrecordedTestsSuite) TestAppendBlockFromURL() {
 	_require.Equal(destBuffer, sourceData)
 }
 
+func (s *AppendBlobUnrecordedTestsSuite) TestBlobEncryptionScopeSAS() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
+	_require.NoError(err)
+
+	containerName := testcommon.GenerateContainerName(testName)
+	containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
+	defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
+
+	blobClient := containerClient.NewAppendBlobClient(testcommon.GenerateBlobName("appendsrc"))
+
+	// Get source abClient URL with SAS for AppendBlockFromURL.
+	blobParts, _ := blob.ParseURL(blobClient.URL())
+
+	encryptionScope, err := testcommon.GetRequiredEnv(testcommon.EncryptionScopeEnvVar)
+	_require.Nil(err)
+	credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
+	_require.Nil(err)
+	perms := sas.BlobPermissions{Read: true, Create: true, Write: true, Delete: true}
+
+	blobParts.SAS, err = sas.BlobSignatureValues{
+		Protocol:        sas.ProtocolHTTPS,                    // Users MUST use HTTPS (not HTTP)
+		ExpiryTime:      time.Now().UTC().Add(48 * time.Hour), // 48-hours before expiration
+		ContainerName:   blobParts.ContainerName,
+		BlobName:        blobParts.BlobName,
+		Permissions:     perms.String(),
+		EncryptionScope: encryptionScope,
+	}.SignWithSharedKey(credential)
+	_require.NoError(err)
+
+	blobURLWithSAS := blobParts.String()
+
+	// create new client with sas url
+	blobClient, err = appendblob.NewClientWithNoCredential(blobURLWithSAS, nil)
+	_require.Nil(err)
+
+	createResponse, err := blobClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	_require.Equal(*createResponse.EncryptionScope, encryptionScope)
+}
+
+func (s *AppendBlobUnrecordedTestsSuite) TestAccountEncryptionScopeSAS() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
+	_require.NoError(err)
+
+	containerName := testcommon.GenerateContainerName(testName)
+	containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
+	defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
+
+	blobName := testcommon.GenerateBlobName("appendsrc")
+	blobClient := containerClient.NewAppendBlobClient(blobName)
+
+	// Get blob URL with SAS for AppendBlockFromURL.
+	blobParts, _ := blob.ParseURL(blobClient.URL())
+
+	encryptionScope, err := testcommon.GetRequiredEnv(testcommon.EncryptionScopeEnvVar)
+	_require.Nil(err)
+
+	credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
+	_require.Nil(err)
+
+	blobParts.SAS, err = sas.AccountSignatureValues{
+		Protocol:        sas.ProtocolHTTPS,                    // Users MUST use HTTPS (not HTTP)
+		ExpiryTime:      time.Now().UTC().Add(48 * time.Hour), // 48-hours before expiration
+		Permissions:     to.Ptr(sas.AccountPermissions{Read: true, Create: true, Write: true, Delete: true}).String(),
+		ResourceTypes:   to.Ptr(sas.AccountResourceTypes{Service: true, Container: true, Object: true}).String(),
+		EncryptionScope: encryptionScope,
+	}.SignWithSharedKey(credential)
+	_require.NoError(err)
+
+	blobURLWithSAS := blobParts.String()
+	blobClient, err = appendblob.NewClientWithNoCredential(blobURLWithSAS, nil)
+	_require.NoError(err)
+
+	createResp, err := blobClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	_require.NotNil(createResp)
+	_require.Equal(*createResp.EncryptionScope, encryptionScope)
+}
+
+func (s *AppendBlobUnrecordedTestsSuite) TestGetUserDelegationEncryptionScopeSAS() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+	accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDefault)
+	_require.Greater(len(accountName), 0)
+
+	cred, err := testcommon.GetGenericTokenCredential()
+	_require.NoError(err)
+
+	svcClient, err := service.NewClient("https://"+accountName+".blob.core.windows.net/", cred, nil)
+	_require.NoError(err)
+
+	containerName := testcommon.GenerateContainerName(testName)
+	cntClientTokenCred := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
+	defer testcommon.DeleteContainer(context.Background(), _require, cntClientTokenCred)
+
+	blobName := testcommon.GenerateBlobName("appendsrc")
+	blobClient := cntClientTokenCred.NewAppendBlobClient(blobName)
+
+	// Set current and past time and create key
+	now := time.Now().UTC().Add(-10 * time.Second)
+	expiry := now.Add(2 * time.Hour)
+	info := service.KeyInfo{
+		Start:  to.Ptr(now.UTC().Format(sas.TimeFormat)),
+		Expiry: to.Ptr(expiry.UTC().Format(sas.TimeFormat)),
+	}
+
+	udc, err := svcClient.GetUserDelegationCredential(context.Background(), info, nil)
+	_require.NoError(err)
+
+	// get permissions and details for sas
+	encryptionScope, err := testcommon.GetRequiredEnv(testcommon.EncryptionScopeEnvVar)
+	_require.Nil(err)
+
+	permissions := sas.BlobPermissions{Read: true, Create: true, Write: true, List: true, Add: true, Delete: true}
+
+	blobParts, _ := blob.ParseURL(blobClient.URL())
+
+	// Create Blob Signature Values with desired permissions and sign with user delegation credential
+	blobParts.SAS, err = sas.BlobSignatureValues{
+		Protocol:        sas.ProtocolHTTPS,
+		StartTime:       time.Now().UTC().Add(time.Second * -10),
+		ExpiryTime:      time.Now().UTC().Add(15 * time.Minute),
+		Permissions:     permissions.String(),
+		ContainerName:   containerName,
+		EncryptionScope: encryptionScope,
+	}.SignWithUserDelegation(udc)
+	_require.NoError(err)
+
+	blobURLWithSAS := blobParts.String()
+	blobClient, err = appendblob.NewClientWithNoCredential(blobURLWithSAS, nil)
+	_require.NoError(err)
+
+	createResp, err := blobClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	_require.NotNil(createResp)
+	_require.Equal(*createResp.EncryptionScope, encryptionScope)
+
+}
+
 func (s *AppendBlobUnrecordedTestsSuite) TestAppendBlockFromURLWithMD5() {
 	_require := require.New(s.T())
 	testName := s.T().Name()

sdk/storage/azblob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "go",
   "TagPrefix": "go/storage/azblob",
-  "Tag": "go/storage/azblob_0776f1b95b"
+  "Tag": "go/storage/azblob_7a25fb98e8"
 }

sdk/storage/azblob/blob/client.go

@@ -8,6 +8,7 @@ package blob
 
 import (
 	"context"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"io"
 	"os"
@@ -37,10 +38,13 @@ type Client base.Client[generated.BlobClient]
 func NewClient(blobURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
 	authPolicy := shared.NewStorageChallengePolicy(cred)
 	conOptions := shared.GetClientOptions(options)
-	conOptions.PerRetryPolicies = append(conOptions.PerRetryPolicies, authPolicy)
-	pl := runtime.NewPipeline(exported.ModuleName, exported.ModuleVersion, runtime.PipelineOptions{}, &conOptions.ClientOptions)
+	plOpts := runtime.PipelineOptions{PerRetry: []policy.Policy{authPolicy}}
 
-	return (*Client)(base.NewBlobClient(blobURL, pl, &cred)), nil
+	azClient, err := azcore.NewClient(shared.BlobClient, exported.ModuleVersion, plOpts, &conOptions.ClientOptions)
+	if err != nil {
+		return nil, err
+	}
+	return (*Client)(base.NewBlobClient(blobURL, azClient, &cred)), nil
 }
 
 // NewClientWithNoCredential creates an instance of Client with the specified values.
@@ -49,9 +53,12 @@ func NewClient(blobURL string, cred azcore.TokenCredential, options *ClientOptio
 //   - options - client options; pass nil to accept the default values
 func NewClientWithNoCredential(blobURL string, options *ClientOptions) (*Client, error) {
 	conOptions := shared.GetClientOptions(options)
-	pl := runtime.NewPipeline(exported.ModuleName, exported.ModuleVersion, runtime.PipelineOptions{}, &conOptions.ClientOptions)
 
-	return (*Client)(base.NewBlobClient(blobURL, pl, nil)), nil
+	azClient, err := azcore.NewClient(shared.BlobClient, exported.ModuleVersion, runtime.PipelineOptions{}, &conOptions.ClientOptions)
+	if err != nil {
+		return nil, err
+	}
+	return (*Client)(base.NewBlobClient(blobURL, azClient, nil)), nil
 }
 
 // NewClientWithSharedKeyCredential creates an instance of Client with the specified values.
@@ -61,10 +68,13 @@ func NewClientWithNoCredential(blobURL string, options *ClientOptions) (*Client,
 func NewClientWithSharedKeyCredential(blobURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
 	authPolicy := exported.NewSharedKeyCredPolicy(cred)
 	conOptions := shared.GetClientOptions(options)
-	conOptions.PerRetryPolicies = append(conOptions.PerRetryPolicies, authPolicy)
-	pl := runtime.NewPipeline(exported.ModuleName, exported.ModuleVersion, runtime.PipelineOptions{}, &conOptions.ClientOptions)
+	plOpts := runtime.PipelineOptions{PerRetry: []policy.Policy{authPolicy}}
 
-	return (*Client)(base.NewBlobClient(blobURL, pl, cred)), nil
+	azClient, err := azcore.NewClient(shared.BlobClient, exported.ModuleVersion, plOpts, &conOptions.ClientOptions)
+	if err != nil {
+		return nil, err
+	}
+	return (*Client)(base.NewBlobClient(blobURL, azClient, cred)), nil
 }
 
 // NewClientFromConnectionString creates an instance of Client with the specified values.
@@ -116,7 +126,7 @@ func (b *Client) WithSnapshot(snapshot string) (*Client, error) {
 	}
 	p.Snapshot = snapshot
 
-	return (*Client)(base.NewBlobClient(p.String(), b.generated().Pipeline(), b.credential())), nil
+	return (*Client)(base.NewBlobClient(p.String(), b.generated().InternalClient(), b.credential())), nil
 }
 
 // WithVersionID creates a new AppendBlobURL object identical to the source but with the specified version id.
@@ -128,7 +138,7 @@ func (b *Client) WithVersionID(versionID string) (*Client, error) {
 	}
 	p.VersionID = versionID
 
-	return (*Client)(base.NewBlobClient(p.String(), b.generated().Pipeline(), b.credential())), nil
+	return (*Client)(base.NewBlobClient(p.String(), b.generated().InternalClient(), b.credential())), nil
 }
 
 // Delete marks the specified blob or snapshot for deletion. The blob is later deleted during garbage collection.
@@ -261,8 +271,8 @@ func (b *Client) SetLegalHold(ctx context.Context, legalHold bool, options *SetL
 // CopyFromURL synchronously copies the data at the source URL to a block blob, with sizes up to 256 MB.
 // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/copy-blob-from-url.
 func (b *Client) CopyFromURL(ctx context.Context, copySource string, options *CopyFromURLOptions) (CopyFromURLResponse, error) {
-	copyOptions, smac, mac, lac := options.format()
-	resp, err := b.generated().CopyFromURL(ctx, copySource, copyOptions, smac, mac, lac)
+	copyOptions, smac, mac, lac, cpkScopeInfo := options.format()
+	resp, err := b.generated().CopyFromURL(ctx, copySource, copyOptions, smac, mac, lac, cpkScopeInfo)
 	return resp, err
 }