[AzDatalake] Feedback + Service Client

sdk/storage/azdatalake/directory/client.go

@@ -16,7 +16,6 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/exported"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/shared"
-	"strings"
 )
 
 // ClientOptions contains the optional parameters when creating a Client.
@@ -30,10 +29,9 @@ type Client base.CompositeClient[generated.PathClient, generated.PathClient, blo
 //   - cred - an Azure AD credential, typically obtained via the azidentity module
 //   - options - client options; pass nil to accept the default values
 func NewClient(directoryURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
-	blobURL := strings.Replace(directoryURL, ".dfs.", ".blob.", 1)
-	directoryURL = strings.Replace(directoryURL, ".blob.", ".dfs.", 1)
+	blobURL, directoryURL := shared.GetURLs(directoryURL)
 
-	authPolicy := shared.NewStorageChallengePolicy(cred)
+	authPolicy := runtime.NewBearerTokenPolicy(cred, []string{shared.TokenScope}, nil)
 	conOptions := shared.GetClientOptions(options)
 	plOpts := runtime.PipelineOptions{
 		PerRetry: []policy.Policy{authPolicy},
@@ -62,8 +60,7 @@ func NewClient(directoryURL string, cred azcore.TokenCredential, options *Client
 //   - directoryURL - the URL of the storage account e.g. https://<account>.dfs.core.windows.net/fs/dir?<sas token>
 //   - options - client options; pass nil to accept the default values
 func NewClientWithNoCredential(directoryURL string, options *ClientOptions) (*Client, error) {
-	blobURL := strings.Replace(directoryURL, ".dfs.", ".blob.", 1)
-	directoryURL = strings.Replace(directoryURL, ".blob.", ".dfs.", 1)
+	blobURL, directoryURL := shared.GetURLs(directoryURL)
 
 	conOptions := shared.GetClientOptions(options)
 	plOpts := runtime.PipelineOptions{}
@@ -91,8 +88,7 @@ func NewClientWithNoCredential(directoryURL string, options *ClientOptions) (*Cl
 //   - cred - a SharedKeyCredential created with the matching storage account and access key
 //   - options - client options; pass nil to accept the default values
 func NewClientWithSharedKeyCredential(directoryURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
-	blobURL := strings.Replace(directoryURL, ".dfs.", ".blob.", 1)
-	directoryURL = strings.Replace(directoryURL, ".blob.", ".dfs.", 1)
+	blobURL, directoryURL := shared.GetURLs(directoryURL)
 
 	authPolicy := exported.NewSharedKeyCredPolicy(cred)
 	conOptions := shared.GetClientOptions(options)
@@ -112,8 +108,11 @@ func NewClientWithSharedKeyCredential(directoryURL string, cred *SharedKeyCreden
 	blobClientOpts := blob.ClientOptions{
 		ClientOptions: options.ClientOptions,
 	}
-	blobSharedKeyCredential, _ := blob.NewSharedKeyCredential(cred.AccountName(), cred.AccountKey())
-	blobClient, _ := blob.NewClientWithSharedKeyCredential(blobURL, blobSharedKeyCredential, &blobClientOpts)
+	blobSharedKey, err := cred.ConvertToBlobSharedKey()
+	if err != nil {
+		return nil, err
+	}
+	blobClient, _ := blob.NewClientWithSharedKeyCredential(blobURL, blobSharedKey, &blobClientOpts)
 	dirClient := base.NewPathClient(directoryURL, blobURL, blobClient, azClient, nil, (*base.ClientOptions)(conOptions))
 
 	return (*Client)(dirClient), nil

sdk/storage/azdatalake/file/client.go

@@ -16,7 +16,6 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/exported"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/shared"
-	"strings"
 )
 
 // ClientOptions contains the optional parameters when creating a Client.
@@ -30,10 +29,8 @@ type Client base.CompositeClient[generated.PathClient, generated.PathClient, blo
 //   - cred - an Azure AD credential, typically obtained via the azidentity module
 //   - options - client options; pass nil to accept the default values
 func NewClient(fileURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
-	blobURL := strings.Replace(fileURL, ".dfs.", ".blob.", 1)
-	fileURL = strings.Replace(fileURL, ".blob.", ".dfs.", 1)
-
-	authPolicy := shared.NewStorageChallengePolicy(cred)
+	blobURL, fileURL := shared.GetURLs(fileURL)
+	authPolicy := runtime.NewBearerTokenPolicy(cred, []string{shared.TokenScope}, nil)
 	conOptions := shared.GetClientOptions(options)
 	plOpts := runtime.PipelineOptions{
 		PerRetry: []policy.Policy{authPolicy},
@@ -62,8 +59,7 @@ func NewClient(fileURL string, cred azcore.TokenCredential, options *ClientOptio
 //   - fileURL - the URL of the storage account e.g. https://<account>.dfs.core.windows.net/fs/file.txt?<sas token>
 //   - options - client options; pass nil to accept the default values
 func NewClientWithNoCredential(fileURL string, options *ClientOptions) (*Client, error) {
-	blobURL := strings.Replace(fileURL, ".dfs.", ".blob.", 1)
-	fileURL = strings.Replace(fileURL, ".blob.", ".dfs.", 1)
+	blobURL, fileURL := shared.GetURLs(fileURL)
 
 	conOptions := shared.GetClientOptions(options)
 	plOpts := runtime.PipelineOptions{}
@@ -91,8 +87,7 @@ func NewClientWithNoCredential(fileURL string, options *ClientOptions) (*Client,
 //   - cred - a SharedKeyCredential created with the matching storage account and access key
 //   - options - client options; pass nil to accept the default values
 func NewClientWithSharedKeyCredential(fileURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
-	blobURL := strings.Replace(fileURL, ".dfs.", ".blob.", 1)
-	fileURL = strings.Replace(fileURL, ".blob.", ".dfs.", 1)
+	blobURL, fileURL := shared.GetURLs(fileURL)
 
 	authPolicy := exported.NewSharedKeyCredPolicy(cred)
 	conOptions := shared.GetClientOptions(options)
@@ -112,8 +107,11 @@ func NewClientWithSharedKeyCredential(fileURL string, cred *SharedKeyCredential,
 	blobClientOpts := blob.ClientOptions{
 		ClientOptions: options.ClientOptions,
 	}
-	blobSharedKeyCredential, _ := blob.NewSharedKeyCredential(cred.AccountName(), cred.AccountKey())
-	blobClient, _ := blob.NewClientWithSharedKeyCredential(blobURL, blobSharedKeyCredential, &blobClientOpts)
+	blobSharedKey, err := cred.ConvertToBlobSharedKey()
+	if err != nil {
+		return nil, err
+	}
+	blobClient, _ := blob.NewClientWithSharedKeyCredential(blobURL, blobSharedKey, &blobClientOpts)
 	fileClient := base.NewPathClient(fileURL, blobURL, blobClient, azClient, nil, (*base.ClientOptions)(conOptions))
 
 	return (*Client)(fileClient), nil

sdk/storage/azdatalake/filesystem/client.go

@@ -11,7 +11,6 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/datalakeerror"
@@ -21,7 +20,6 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/shared"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/sas"
 	"net/http"
-	"strings"
 	"time"
 )
 
@@ -36,10 +34,8 @@ type Client base.CompositeClient[generated.FileSystemClient, generated.FileSyste
 //   - cred - an Azure AD credential, typically obtained via the azidentity module
 //   - options - client options; pass nil to accept the default values
 func NewClient(filesystemURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
-	containerURL := strings.Replace(filesystemURL, ".dfs.", ".blob.", 1)
-	filesystemURL = strings.Replace(filesystemURL, ".blob.", ".dfs.", 1)
-
-	authPolicy := shared.NewStorageChallengePolicy(cred)
+	containerURL, filesystemURL := shared.GetURLs(filesystemURL)
+	authPolicy := runtime.NewBearerTokenPolicy(cred, []string{shared.TokenScope}, nil)
 	conOptions := shared.GetClientOptions(options)
 	plOpts := runtime.PipelineOptions{
 		PerRetry: []policy.Policy{authPolicy},
@@ -68,9 +64,7 @@ func NewClient(filesystemURL string, cred azcore.TokenCredential, options *Clien
 //   - filesystemURL - the URL of the storage account e.g. https://<account>.dfs.core.windows.net/fs?<sas token>
 //   - options - client options; pass nil to accept the default values
 func NewClientWithNoCredential(filesystemURL string, options *ClientOptions) (*Client, error) {
-	containerURL := strings.Replace(filesystemURL, ".dfs.", ".blob.", 1)
-	filesystemURL = strings.Replace(filesystemURL, ".blob.", ".dfs.", 1)
-
+	containerURL, filesystemURL := shared.GetURLs(filesystemURL)
 	conOptions := shared.GetClientOptions(options)
 	plOpts := runtime.PipelineOptions{}
 	base.SetPipelineOptions((*base.ClientOptions)(conOptions), &plOpts)
@@ -97,9 +91,7 @@ func NewClientWithNoCredential(filesystemURL string, options *ClientOptions) (*C
 //   - cred - a SharedKeyCredential created with the matching storage account and access key
 //   - options - client options; pass nil to accept the default values
 func NewClientWithSharedKeyCredential(filesystemURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
-	containerURL := strings.Replace(filesystemURL, ".dfs.", ".blob.", 1)
-	filesystemURL = strings.Replace(filesystemURL, ".blob.", ".dfs.", 1)
-
+	containerURL, filesystemURL := shared.GetURLs(filesystemURL)
 	authPolicy := exported.NewSharedKeyCredPolicy(cred)
 	conOptions := shared.GetClientOptions(options)
 	plOpts := runtime.PipelineOptions{
@@ -118,8 +110,11 @@ func NewClientWithSharedKeyCredential(filesystemURL string, cred *SharedKeyCrede
 	containerClientOpts := container.ClientOptions{
 		ClientOptions: options.ClientOptions,
 	}
-	blobSharedKeyCredential, _ := blob.NewSharedKeyCredential(cred.AccountName(), cred.AccountKey())
-	blobContainerClient, _ := container.NewClientWithSharedKeyCredential(containerURL, blobSharedKeyCredential, &containerClientOpts)
+	blobSharedKey, err := cred.ConvertToBlobSharedKey()
+	if err != nil {
+		return nil, err
+	}
+	blobContainerClient, _ := container.NewClientWithSharedKeyCredential(containerURL, blobSharedKey, &containerClientOpts)
 	fsClient := base.NewFilesystemClient(filesystemURL, containerURL, blobContainerClient, azClient, cred, (*base.ClientOptions)(conOptions))
 
 	return (*Client)(fsClient), nil

sdk/storage/azdatalake/filesystem/constants.go

@@ -17,6 +17,8 @@ const (
 	Filesystem PublicAccessType = azblob.PublicAccessTypeContainer
 )
 
+// TODO: figure out a way to import this from datalake rather than blob again
+
 // StatusType defines values for StatusType
 type StatusType = lease.StatusType
 

sdk/storage/azdatalake/filesystem/models.go

@@ -210,3 +210,9 @@ type LeaseAccessConditions = exported.LeaseAccessConditions
 
 // ModifiedAccessConditions contains a group of parameters for specifying access conditions.
 type ModifiedAccessConditions = exported.ModifiedAccessConditions
+
+// PathList contains the path list
+type PathList = generated.PathList
+
+// Path contains the path properties
+type Path = generated.Path

sdk/storage/azdatalake/filesystem/responses.go

@@ -28,28 +28,28 @@ type SetAccessPolicyResponse = container.SetAccessPolicyResponse
 // GetAccessPolicyResponse contains the response from method FilesystemClient.GetAccessPolicy.
 type GetAccessPolicyResponse struct {
 	// PublicAccess contains the information returned from the x-ms-blob-public-access header response.
-	PublicAccess *PublicAccessType `xml:"BlobPublicAccess"`
+	PublicAccess *PublicAccessType
 
 	// ClientRequestID contains the information returned from the x-ms-client-request-id header response.
-	ClientRequestID *string `xml:"ClientRequestID"`
+	ClientRequestID *string
 
 	// Date contains the information returned from the Date header response.
-	Date *time.Time `xml:"Date"`
+	Date *time.Time
 
 	// ETag contains the information returned from the ETag header response.
-	ETag *azcore.ETag `xml:"ETag"`
+	ETag *azcore.ETag
 
 	// LastModified contains the information returned from the Last-Modified header response.
-	LastModified *time.Time `xml:"LastModified"`
+	LastModified *time.Time
 
 	// RequestID contains the information returned from the x-ms-request-id header response.
-	RequestID *string `xml:"RequestID"`
+	RequestID *string
 
 	// a collection of signed identifiers
-	SignedIdentifiers []*SignedIdentifier `xml:"SignedIdentifier"`
+	SignedIdentifiers []*SignedIdentifier
 
 	// Version contains the information returned from the x-ms-version header response.
-	Version *string `xml:"Version"`
+	Version *string
 }
 
 // since we want to remove the blob prefix in access type
@@ -140,3 +140,6 @@ type ListPathsSegmentResponse = generated.FileSystemClientListPathsResponse
 
 // ListDeletedPathsSegmentResponse contains the response from method FilesystemClient.ListPathsSegment.
 type ListDeletedPathsSegmentResponse = generated.FileSystemClientListBlobHierarchySegmentResponse
+
+// ListBlobsHierarchySegmentResponse contains the response from method FilesystemClient.ListBlobsHierarchySegment.
+type ListBlobsHierarchySegmentResponse = generated.ListBlobsHierarchySegmentResponse

sdk/storage/azdatalake/go.mod

@@ -3,7 +3,7 @@ module github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake
 go 1.18
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
 	github.com/stretchr/testify v1.7.1

sdk/storage/azdatalake/go.sum

@@ -1,5 +1,5 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0 h1:xGLAFFd9D3iLGxYiUGPdITSzsFmU1K8VtfuUHWAoN7M=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=

sdk/storage/azdatalake/internal/base/clients.go

@@ -22,24 +22,6 @@ type ClientOptions struct {
 	pipelineOptions *runtime.PipelineOptions
 }
 
-type Client[T any] struct {
-	inner     *T
-	sharedKey *exported.SharedKeyCredential
-	options   *ClientOptions
-}
-
-func InnerClient[T any](client *Client[T]) *T {
-	return client.inner
-}
-
-func SharedKey[T any](client *Client[T]) *exported.SharedKeyCredential {
-	return client.sharedKey
-}
-
-func GetClientOptions[T any](client *Client[T]) *ClientOptions {
-	return client.options
-}
-
 func GetPipelineOptions(clOpts *ClientOptions) *runtime.PipelineOptions {
 	return clOpts.pipelineOptions
 }
@@ -96,3 +78,7 @@ func NewPathClient(dirURL string, dirURLWithBlobEndpoint string, client *blob.Cl
 		options:   options,
 	}
 }
+
+func GetCompositeClientOptions[T, K, U any](client *CompositeClient[T, K, U]) *ClientOptions {
+	return client.options
+}

sdk/storage/azdatalake/internal/exported/log_events.go

@@ -0,0 +1,20 @@
+//go:build go1.18
+// +build go1.18
+
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+package exported
+
+import (
+	"github.com/Azure/azure-sdk-for-go/sdk/internal/log"
+)
+
+// NOTE: these are publicly exported via type-aliasing in azdatalake/log.go
+const (
+	// EventUpload is used when we compute number of chunks to upload and size of each chunk.
+	EventUpload log.Event = "azdatalake.Upload"
+
+	// EventError is used for logging errors.
+	EventError log.Event = "azdatalake.Error"
+)

sdk/storage/azdatalake/internal/exported/shared_key_credential.go

@@ -12,6 +12,7 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	"net/http"
 	"net/url"
 	"sort"
@@ -48,9 +49,12 @@ func (c *SharedKeyCredential) AccountName() string {
 	return c.accountName
 }
 
-// AccountKey returns the Storage account's name.
-func (c *SharedKeyCredential) AccountKey() string {
-	return c.accountKeyString
+func (c *SharedKeyCredential) ConvertToBlobSharedKey() (*azblob.SharedKeyCredential, error) {
+	cred, err := azblob.NewSharedKeyCredential(c.accountName, c.accountKeyString)
+	if err != nil {
+		return nil, err
+	}
+	return cred, nil
 }
 
 // SetAccountKey replaces the existing account key with the specified account key.