Merge pull request #788 from Azure/dev

10.3.3 Release

ChangeLog.md

@@ -1,6 +1,30 @@
 
 # Change Log
 
+## Version 10.3.3
+
+### New features
+
+1. `azcopy list` is now supported on Azure Files and ADLS Gen 2, in addition to Blob Storage.
+1. The `--exclude-path` flag is now supported in the `sync` command.
+1. Added new environment variable `AZCOPY_USER_AGENT_PREFIX` to allow a prefix to be appended to the user agent strings.
+
+### Bug fixes
+
+1. Content properties (such as Content-Encoding and Cache-Control) are now included when syncing Blob -> Blob and Azure
+   Files -> Azure Files
+1. Custom metadata is now included when syncing Blob -> Blob and Azure Files -> Azure Files
+1. The `azcopy list` command no longer repeats parts of its output. (Previously it would sometimes repeat itself and show the same blob multiple times in the output.)
+1. The `--aad-endpoint` parameter is now visible, instead of hidden. It allows use of Azure Active Directory
+   authentication in national clouds (e.g. Azure China).
+1. On Windows, AzCopy now caches information about which proxy server should be used, instead of looking it up every
+   time. This significantly reduces CPU
+   usage when transferring many small files. It also solves a rare bug when transfers got permanently "stuck" with
+   one uncompleted file.
+1. When uploading to a write-only destination, there is now a clearer error message when the built-in file length check
+   fails. The message says how to fix the problem using `--check-length=false`.
+1. Size checks on managed disk imports are now clearer, and all run at the start of the import process instead of the end.
+
 ## Version 10.3.2
 
 ### Bug fixes

cmd/copyEnumeratorInit.go

@@ -230,20 +230,11 @@ func (cca *cookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
 		srcRelPath := cca.makeEscapedRelativePath(true, isDestDir, object)
 		dstRelPath := cca.makeEscapedRelativePath(false, isDestDir, object)
 
-		transfer := common.NewCopyTransfer(
+		transfer := object.ToNewCopyTransfer(
 			cca.autoDecompress && cca.fromTo.IsDownload(),
 			srcRelPath, dstRelPath,
-			object.lastModifiedTime,
-			object.size,
-			object.contentType, object.contentEncoding, object.contentDisposition, object.contentLanguage, object.cacheControl,
-			object.md5,
-			object.Metadata,
-			object.blobType,
-			azblob.AccessTierNone) // access tier is assigned conditionally
-
-		if cca.s2sPreserveAccessTier {
-			transfer.BlobTier = object.blobAccessTier
-		}
+			cca.s2sPreserveAccessTier,
+		)
 
 		return addTransfer(&jobPartOrder, transfer, cca)
 	}

cmd/credentialUtil.go

@@ -367,7 +367,7 @@ func createBlobPipeline(ctx context.Context, credInfo common.CredentialInfo) (pi
 		credential,
 		azblob.PipelineOptions{
 			Telemetry: azblob.TelemetryOptions{
-				Value: common.UserAgent,
+				Value: glcm.AddUserAgentPrefix(common.UserAgent),
 			},
 		},
 		ste.XferRetryOptions{
@@ -402,7 +402,7 @@ func createBlobFSPipeline(ctx context.Context, credInfo common.CredentialInfo) (
 				MaxRetryDelay: ste.UploadMaxRetryDelay,
 			},
 			Telemetry: azbfs.TelemetryOptions{
-				Value: common.UserAgent,
+				Value: glcm.AddUserAgentPrefix(common.UserAgent),
 			},
 		}), nil
 }
@@ -420,7 +420,7 @@ func createFilePipeline(ctx context.Context, credInfo common.CredentialInfo) (pi
 				MaxRetryDelay: ste.UploadMaxRetryDelay,
 			},
 			Telemetry: azfile.TelemetryOptions{
-				Value: common.UserAgent,
+				Value: glcm.AddUserAgentPrefix(common.UserAgent),
 			},
 		}), nil
 }

cmd/helpMessages.go

@@ -198,7 +198,7 @@ const cleanJobsCmdExample = "  azcopy jobs clean --with-status=completed"
 // ===================================== LIST COMMAND ===================================== //
 const listCmdShortDescription = "List the entities in a given resource"
 
-const listCmdLongDescription = `List the entities in a given resource. In the current release, only Blob containers are supported.`
+const listCmdLongDescription = `List the entities in a given resource. Blob, Files, and ADLS Gen 2 containers, folders, and accounts are supported.`
 
 const listCmdExample = "azcopy list [containerURL]"
 

cmd/list.go

@@ -24,14 +24,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net/url"
 	"strconv"
-	"strings"
+
+	"github.com/spf13/cobra"
 
 	"github.com/Azure/azure-storage-azcopy/common"
 	"github.com/Azure/azure-storage-azcopy/ste"
-	"github.com/Azure/azure-storage-blob-go/azblob"
-	"github.com/spf13/cobra"
 )
 
 func init() {
@@ -59,11 +57,12 @@ func init() {
 			// the expected argument in input is the container sas / or path of virtual directory in the container.
 			// verifying the location type
 			location := inferArgumentLocation(sourcePath)
-			if location != location.Blob() {
+			// Only support listing for Azure locations
+			if location != location.Blob() && location != location.File() && location != location.BlobFS() {
 				glcm.Error("invalid path passed for listing. given source is of type " + location.String() + " while expect is container / container path ")
 			}
 
-			err := HandleListContainerCommand(sourcePath)
+			err := HandleListContainerCommand(sourcePath, location)
 			if err == nil {
 				glcm.Exit(nil, common.EExitCode.Success())
 			} else {
@@ -89,17 +88,29 @@ type ListParameters struct {
 var parameters = ListParameters{}
 
 // HandleListContainerCommand handles the list container command
-func HandleListContainerCommand(source string) (err error) {
+func HandleListContainerCommand(source string, location common.Location) (err error) {
 	// TODO: Temporarily use context.TODO(), this should be replaced with a root context from main.
 	ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion)
 
 	credentialInfo := common.CredentialInfo{}
-	// Use source as resource URL, and it can be public access resource URL.
-	if credentialInfo.CredentialType, _, err = getBlobCredentialType(ctx, source, true, false); err != nil {
+
+	base, token, err := SplitAuthTokenFromResource(source, location)
+	if err != nil {
+		return err
+	}
+
+	level, err := determineLocationLevel(source, location, true)
+
+	if err != nil {
 		return err
+	}
+
+	// Treat our check as a destination because the isSource flag was designed for S2S transfers.
+	if credentialInfo, _, err = getCredentialInfoForLocation(ctx, location, base, token, false); err != nil {
+		return fmt.Errorf("failed to obtain credential info: %s", err.Error())
+	} else if location == location.File() && token == "" {
+		return errors.New("azure files requires a SAS token for authentication")
 	} else if credentialInfo.CredentialType == common.ECredentialType.OAuthToken() {
-		// Message user that they are using Oauth token for authentication,
-		// in case of silently using cached token without consciousness。
 		glcm.Info("List is using OAuth token for authentication.")
 
 		uotm := GetUserOAuthTokenManagerInstance()
@@ -110,85 +121,56 @@ func HandleListContainerCommand(source string) (err error) {
 		}
 	}
 
-	// Create Pipeline which will be used further in the blob operations.
-	p, err := createBlobPipeline(ctx, credentialInfo)
-	if err != nil {
-		return err
-	}
+	traverser, err := initResourceTraverser(source, location, &ctx, &credentialInfo, nil, nil, true, false, func() {})
 
-	// attempt to parse the source url
-	sourceURL, err := url.Parse(source)
 	if err != nil {
-		return errors.New("cannot parse source URL")
+		return fmt.Errorf("failed to initialize traverser: %s", err.Error())
 	}
 
-	util := copyHandlerUtil{} // TODO: util could be further refactored
-	// get the container url to be used for listing
-	literalContainerURL := util.getContainerURLFromString(*sourceURL)
-	containerURL := azblob.NewContainerURL(literalContainerURL, p)
+	var fileCount int64 = 0
+	var sizeCount int64 = 0
 
-	// get the search prefix to query the service
-	searchPrefix := ""
-	// if the source is container url, then searchPrefix is empty
-	if !util.urlIsContainerOrVirtualDirectory(sourceURL) {
-		searchPrefix = util.getBlobNameFromURL(sourceURL.Path)
-	}
-	if len(searchPrefix) > 0 {
-		// if the user did not specify / at the end of the virtual directory, add it before doing the prefix search
-		if strings.LastIndex(searchPrefix, "/") != len(searchPrefix)-1 {
-			searchPrefix += "/"
-		}
-	}
+	processor := func(object storedObject) error {
+		objectSummary := object.relativePath + "; Content Length: "
 
-	summary := common.ListContainerResponse{}
+		if level == level.Service() {
+			objectSummary = object.containerName + "/" + objectSummary
+		}
 
-	fileCount := 0
-	sizeCount := 0
+		if parameters.MachineReadable {
+			objectSummary += strconv.Itoa(int(object.size))
+		} else {
+			objectSummary += byteSizeToString(object.size)
+		}
 
-	// perform a list blob
-	for marker := (azblob.Marker{}); marker.NotDone(); {
-		// look for all blobs that start with the prefix
-		listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker,
-			azblob.ListBlobsSegmentOptions{Prefix: searchPrefix})
-		if err != nil {
-			return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error())
+		if parameters.RunningTally {
+			fileCount++
+			sizeCount += object.size
 		}
 
-		// Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute)
-		for _, blobInfo := range listBlob.Segment.BlobItems {
-			blobName := blobInfo.Name + "; Content Size: "
+		glcm.Info(objectSummary)
 
-			if parameters.MachineReadable {
-				blobName += strconv.Itoa(int(*blobInfo.Properties.ContentLength))
-			} else {
-				blobName += byteSizeToString(*blobInfo.Properties.ContentLength)
-			}
+		// No need to strip away from the name as the traverser has already done so.
+		return nil
+	}
 
-			if parameters.RunningTally {
-				fileCount++
-				sizeCount += int(*blobInfo.Properties.ContentLength)
-			}
+	err = traverser.traverse(nil, processor, nil)
 
-			if len(searchPrefix) > 0 {
-				// strip away search prefix from the blob name.
-				blobName = strings.Replace(blobName, searchPrefix, "", 1)
-			}
-			summary.Blobs = append(summary.Blobs, blobName)
-		}
-		marker = listBlob.NextMarker
-		printListContainerResponse(&summary)
+	if err != nil {
+		return fmt.Errorf("failed to traverse container: %s", err.Error())
+	}
 
-		if parameters.RunningTally {
-			glcm.Info("")
-			glcm.Info("File count: " + strconv.Itoa(fileCount))
+	if parameters.RunningTally {
+		glcm.Info("")
+		glcm.Info("File count: " + strconv.Itoa(int(fileCount)))
 
-			if parameters.MachineReadable {
-				glcm.Info("Total file size: " + strconv.Itoa(sizeCount))
-			} else {
-				glcm.Info("Total file size: " + byteSizeToString(int64(sizeCount)))
-			}
+		if parameters.MachineReadable {
+			glcm.Info("Total file size: " + strconv.Itoa(int(sizeCount)))
+		} else {
+			glcm.Info("Total file size: " + byteSizeToString(sizeCount))
 		}
 	}
+
 	return nil
 }
 
@@ -222,7 +204,7 @@ func byteSizeToString(size int64) string {
 		"GiB",
 		"TiB",
 		"PiB",
-		"EiB", //Let's face it, a file probably won't be more than 1000 exabytes in YEARS. (and int64 literally isn't large enough to handle too many exbibytes. 128 bit processors when)
+		"EiB", // Let's face it, a file, account, or container probably won't be more than 1000 exabytes in YEARS. (and int64 literally isn't large enough to handle too many exbibytes. 128 bit processors when)
 	}
 	unit := 0
 	floatSize := float64(size)

cmd/login.go

@@ -63,7 +63,7 @@ func init() {
 	rootCmd.AddCommand(lgCmd)
 
 	lgCmd.PersistentFlags().StringVar(&loginCmdArgs.tenantID, "tenant-id", "", "The Azure Active Directory tenant ID to use for OAuth device interactive login.")
-	lgCmd.PersistentFlags().StringVar(&loginCmdArgs.aadEndpoint, "aad-endpoint", "", "The Azure Active Directory endpoint to use for OAuth user interactive login.")
+	lgCmd.PersistentFlags().StringVar(&loginCmdArgs.aadEndpoint, "aad-endpoint", "", "The Azure Active Directory endpoint to use. The default ("+common.DefaultActiveDirectoryEndpoint+") is correct for the public Azure cloud. Set this parameter when authenticating in a national cloud. Not needed for Managed Service Identity")
 	// Use identity which aligns to Azure powershell and CLI.
 	lgCmd.PersistentFlags().BoolVar(&loginCmdArgs.identity, "identity", false, "Log in using virtual machine's identity, also known as managed service identity (MSI).")
 	// Use SPN certificate to log in.
@@ -78,10 +78,6 @@ func init() {
 	//login with SPN
 	lgCmd.PersistentFlags().StringVar(&loginCmdArgs.applicationID, "application-id", "", "Application ID of user-assigned identity. Required for service principal auth.")
 	lgCmd.PersistentFlags().StringVar(&loginCmdArgs.certPath, "certificate-path", "", "Path to certificate for SPN authentication. Required for certificate-based service principal auth.")
-
-	// hide flags
-	// temporaily hide aad-endpoint and support Production environment only.
-	lgCmd.PersistentFlags().MarkHidden("aad-endpoint")
 }
 
 type loginCmdArgs struct {

cmd/pathUtils.go

@@ -59,47 +59,24 @@ func determineLocationLevel(location string, locationType common.Location, sourc
 
 	case common.ELocation.Blob(),
 		common.ELocation.File(),
-		common.ELocation.BlobFS():
+		common.ELocation.BlobFS(),
+		common.ELocation.S3():
 		URL, err := url.Parse(location)
 
 		if err != nil {
 			return ELocationLevel.Service(), err
 		}
 
-		// blobURLParts is the same format and doesn't care about endpoint
-		bURL := azblob.NewBlobURLParts(*URL)
+		// GenericURLParts determines the correct resource URL parts to make use of
+		bURL := common.NewGenericResourceURLParts(*URL, locationType)
 
-		if strings.Contains(bURL.ContainerName, "*") && bURL.BlobName != "" {
+		if strings.Contains(bURL.GetContainerName(), "*") && bURL.GetObjectName() != "" {
 			return ELocationLevel.Service(), errors.New("can't use a wildcarded container name and specific blob name in combination")
 		}
 
-		if bURL.BlobName != "" {
-			return ELocationLevel.Object(), nil
-		} else if bURL.ContainerName != "" && !strings.Contains(bURL.ContainerName, "*") {
-			return ELocationLevel.Container(), nil
-		} else {
-			return ELocationLevel.Service(), nil
-		}
-	case common.ELocation.S3():
-		URL, err := url.Parse(location)
-
-		if err != nil {
-			return ELocationLevel.Service(), nil
-		}
-
-		s3URL, err := common.NewS3URLParts(*URL)
-
-		if err != nil {
-			return ELocationLevel.Service(), nil
-		}
-
-		if strings.Contains(s3URL.BucketName, "*") && s3URL.ObjectKey != "" {
-			return ELocationLevel.Service(), errors.New("can't use a wildcarded container name and specific object name in combination")
-		}
-
-		if s3URL.ObjectKey != "" {
+		if bURL.GetObjectName() != "" {
 			return ELocationLevel.Object(), nil
-		} else if s3URL.BucketName != "" && !strings.Contains(s3URL.BucketName, "*") {
+		} else if bURL.GetContainerName() != "" && !strings.Contains(bURL.GetContainerName(), "*") {
 			return ELocationLevel.Container(), nil
 		} else {
 			return ELocationLevel.Service(), nil

cmd/removeProcessor.go

@@ -53,5 +53,5 @@ func newRemoveTransferProcessor(cca *cookedCopyCmdArgs, numOfTransfersPerPart in
 	// note that the source and destination, along with the template are given to the generic processor's constructor
 	// this means that given an object with a relative path, this processor already knows how to schedule the right kind of transfers
 	return newCopyTransferProcessor(copyJobTemplate, numOfTransfersPerPart, cca.source, cca.destination,
-		shouldEncodeSource, false, reportFirstPart, reportFinalPart)
+		shouldEncodeSource, false, reportFirstPart, reportFinalPart, false)
 }