Merge dev to master for release 10.4 (#955)

ChangeLog.md

@@ -1,17 +1,85 @@
 
 # Change Log
 
-## Version 10.3.61 (private drop)
+## Version 10.4
 
-### Performance optimizations
+### New features
+
+1. `azcopy copy` now supports the persistence of ACLs between supported resources (Windows and Azure Files) using the `--persist-smb-permissions` flag.
+1. `azcopy copy` now supports the persistence of SMB property info between supported resources (Windows and Azure Files) 
+using the `--persist-smb-info` flag. The information that can be preserved is Created Time, Last Write Time and Attributes (e.g. Read Only).
+1. AzCopy can now transfer empty folders, and also transfer the properties of folders. This applies when both the source 
+and destination support real folders (Blob Storage does not, because it only supports virtual folders).
+1. On Windows, AzCopy can now activate the special privileges `SeBackupPrivilege` and `SeRestorePrivilege`.  Most admin-level 
+accounts have these privileges in a deactivated state, as do all members of the "Backup Operators" security group.  
+If you run AzCopy as one of those users 
+and supply the new flag `--backup`, AzCopy will activate the privileges. (Use an elevated command prompt, if running as Admin).
+At upload time, this allows AzCopy to read files 
+which you wouldn't otherwise have permission to see. At download time, it works with the `--preserve-smb-permissions` flag
+to allow preservation of permissions where the Owner is not the user running AzCopy.  The `--backup` flag will report a failure 
+if the privileges cannot be activated.   
+1. Status output from AzCopy `copy`, `sync`, `jobs list`, and `jobs status` now contains information about folders.
+   This includes new properties in the JSON output of copy, sync, list and jobs status commands, when `--output-type
+   json` is used.
+1. Empty folders are deleted when using `azcopy rm` on Azure Files.
+1. Snapshots of Azure File Shares are supported, for read-only access, in `copy`,`sync` and `list`. To use, add a
+   `sharesnapshot` parameter at end of URL for your Azure Files source. Remember to separate it from the existing query
+   string parameters (i.e. the SAS token) with a `&`.  E.g.
+   `https://<youraccount>.file.core.windows.net/sharename?st=2020-03-03T20%3A53%3A48Z&se=2020-03-04T20%3A53%3A48Z&sp=rl&sv=2018-03-28&sr=s&sig=REDACTED&sharesnapshot=2020-03-03T20%3A24%3A13.0000000Z`
+1. Benchmark mode is now supported for Azure Files and ADLS Gen 2 (in addition to the existing benchmark support for
+   Blob Storage).
+1. A special performance optimization is introduced, but only for NON-recursive cases in this release.  An `--include-pattern` that contains only `*` wildcards will be performance optimized when 
+   querying blob storage without the recursive flag. The section before the first `*` will be used as a server-side prefix, to filter the search results more efficiently. E.g. `--include-pattern abc*` will be implemented 
+as a prefix search for "abc". In a more complex example, `--include-pattern abc*123`, will be implemented as a prefix search for `abc`, followed by normal filtering for all matches of `abc*123`.  To non-recursively process blobs
+contained directly in a container or virtual directory include `/*` at the end of the URL (before the query string).  E.g. `http://account.blob.core.windows.net/container/*?<SAS>`.
+1. The `--cap-mbps` parameter now parses floating-point numbers. This will allow you to limit your maximum throughput to a fraction of a megabit per second.
+
+### Special notes
+
+1. A more user-friendly error message is returned when an unknown source/destination combination is supplied
+1. AzCopy has upgraded to service revision `2019-02-02`. Users targeting local emulators, Azure Stack, or other private/special
+ instances of Azure Storage may need to intentionally downgrade their service revision using the environment variable 
+ `AZCOPY_DEFAULT_SERVICE_API_VERSION`. Prior to this release, the default service revision was `2018-03-28`.
+1. For Azure Files to Azure Files transfers, --persist-smb-permissions and --persist-smb-info are available on all OS's. 
+(But for for uploads and downloads, those flags are only available on Windows.)
+1. AzCopy now includes a list of trusted domain suffixes for Azure Active Directory (AAD) authentication. 
+   After `azcopy login`, the resulting token will only be sent to locations that appear in the list. The list is:
+   `*.core.windows.net;*.core.chinacloudapi.cn;*.core.cloudapi.de;*.core.usgovcloudapi.net`. 
+   If necessary, you can add to the the list with the command-line flag: `--trusted-microsoft-suffixes`. For security,
+   you should only add Microsoft Azure domains. 
+1. When transferring over a million files, AzCopy will reduces its progress reporting frequency from every 2 seconds to every 2 minutes.   
+
+### Breaking changes
 
-1. Any `--include-pattern` that contains only `*` wildcards will be performance optimized when querying blob storage.  The section before the 
-first `*` will be used as a server-side prefix, to filter the search results more efficiently. E.g. "--include-path abc*" will be implemented 
-as a prefix search for "abc". In a more complex example, "--include-path abc\*123", will be implemented as a prefix search for "abc", followed
-by client-side filtering to find exact matches to abc\*123.
-2. When processing over a million files, AzCopy will report on its progress once ever 2 minutes instead of once every 2 seconds.  This reduces the CPU 
-load associated with progress reporting.
+1. To accommodate interfacing with JavaScript programs (and other languages that have similar issue with number precision), 
+   all the numbers in the JSON output have been converted to strings (i.e. with quotes around them).
+1. The TransferStatus value `SkippedFileAlreadyExists` has been renamed `SkippedEntityExists` and may now be used both 
+   for when files are skipped and for when the setting of folder properties is skipped.  This affects the input and 
+   output of `azcopy jobs show` and the status values shown in the JSON output format from `copy` and `sync`.
+1. The format and content of authentication information messages, in the JSON output format, e.g.
+   "Using OAuth token for authentication" has been changed.
 
+### Bug fixes
+
+1. AzCopy can now overwrite even Read-Only and Hidden files when downloading to Windows. (The read-only case requires the use of 
+   the new `--force-if-read-only` flag.)
+1. Fixed a nil dereference when a prefetching error occurs in a upload
+1. Fixed a nil dereference when attempting to close a log file while log-level is none
+1. AzCopy's scanning of Azure Files sources, for download or Service to Service transfers, is now much faster.
+1. Sources and destinations that are identified by their IPv4 address can now be used. This enables usage with storage
+   emulators.  Note that the `from-to` flag is typically needed when using such sources or destinations. E.g. `--from-to
+   BlobLocal` if downloading from a blob storage emulator to local disk.
+1. Filenames containing the character `:` can now safely be downloaded on Windows and uploaded to Azure Files
+1. Objects with names containing `+` can now safely be used in imported S3 object names
+1. The `check-length` flag is now exposed in benchmark mode, so that length checking can be turned off for more speed,
+   when benchmarking with small file sizes. (When using large file sizes, the overhead of the length check is
+   insignificant.)
+1. The in-app documentation for Service Principal Authentication has been corrected, to now include the application-id
+   parameter.
+1. ALL filter types are now disallowed when running `azcopy rm` against ADLS Gen2 endpoints. Previously 
+include/exclude patterns were disallowed, but exclude-path was not. That was incorrect. All should have been
+disallowed because none (other than include-path) are respected. 
+
 ## Version 10.3.4
 
 ### New features

azbfs/url_directory.go

@@ -28,6 +28,10 @@ func NewDirectoryURL(url url.URL, p pipeline.Pipeline) DirectoryURL {
 	return DirectoryURL{directoryClient: directoryClient, filesystem: urlParts.FileSystemName, pathParameter: urlParts.DirectoryOrFilePath}
 }
 
+func (d DirectoryURL) IsFileSystemRoot() bool {
+	return d.pathParameter == ""
+}
+
 // URL returns the URL endpoint used by the DirectoryURL object.
 func (d DirectoryURL) URL() url.URL {
 	return d.directoryClient.URL()
@@ -64,11 +68,22 @@ func (d DirectoryURL) NewDirectoryURL(dirName string) DirectoryURL {
 }
 
 // Create creates a new directory within a File System
-func (d DirectoryURL) Create(ctx context.Context) (*DirectoryCreateResponse, error) {
+func (d DirectoryURL) Create(ctx context.Context, recreateIfExists bool) (*DirectoryCreateResponse, error) {
+	var ifNoneMatch *string
+	if recreateIfExists {
+		ifNoneMatch = nil // the default ADLS Gen2 behavior, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create
+	} else {
+		star := "*" // see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create
+		ifNoneMatch = &star
+	}
+	return d.doCreate(ctx, ifNoneMatch)
+}
+
+func (d DirectoryURL) doCreate(ctx context.Context, ifNoneMatch *string) (*DirectoryCreateResponse, error) {
 	resp, err := d.directoryClient.Create(ctx, d.filesystem, d.pathParameter, PathResourceDirectory, nil,
 		PathRenameModeNone, nil, nil, nil, nil, nil,
 		nil, nil, nil, nil, nil, nil,
-		nil, nil, nil, nil, nil, nil,
+		nil, nil, nil, nil, nil, ifNoneMatch,
 		nil, nil, nil, nil, nil, nil,
 		nil, nil, nil)
 	return (*DirectoryCreateResponse)(resp), err
@@ -124,17 +139,17 @@ func (d DirectoryURL) ListDirectorySegment(ctx context.Context, marker *string,
 // It returns false if the directoryUrl is not able to get resource properties
 // It returns false if the url represent a file in the filesystem
 // TODO reconsider for SDK release
-func (d DirectoryURL) IsDirectory(ctx context.Context) bool {
+func (d DirectoryURL) IsDirectory(ctx context.Context) (bool, error) {
 	grep, err := d.GetProperties(ctx)
 	// If the error occurs while getting resource properties return false
 	if err != nil {
-		return false
+		return false, err
 	}
 	// return false if the resource type is not
 	if !strings.EqualFold(grep.XMsResourceType(), directoryResourceName) {
-		return false
+		return false, nil
 	}
-	return true
+	return true, nil
 }
 
 // NewFileUrl converts the current directory Url into the NewFileUrl

azbfs/url_file.go

@@ -53,6 +53,14 @@ func (f FileURL) WithPipeline(p pipeline.Pipeline) FileURL {
 	return NewFileURL(f.fileClient.URL(), p)
 }
 
+func (f FileURL) GetParentDir() (DirectoryURL, error) {
+	d, err := removeLastSectionOfPath(f.URL())
+	if err != nil {
+		return DirectoryURL{}, err
+	}
+	return NewDirectoryURL(d, f.fileClient.p), nil
+}
+
 // Create creates a new file or replaces a file. Note that this method only initializes the file.
 // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/create-file.
 func (f FileURL) Create(ctx context.Context, headers BlobFSHTTPHeaders) (*PathCreateResponse, error) {

azbfs/url_service.go

@@ -2,7 +2,9 @@ package azbfs
 
 import (
 	"context"
+	"errors"
 	"net/url"
+	"path"
 
 	"github.com/Azure/azure-pipeline-go/pipeline"
 )
@@ -70,3 +72,17 @@ func appendToURLPath(u url.URL, name string) url.URL {
 	u.Path += name
 	return u
 }
+
+func removeLastSectionOfPath(u url.URL) (url.URL, error) {
+	if len(u.Path) == 0 {
+		return url.URL{}, errors.New("cannot remove from path because it is empty")
+	}
+
+	trimmedPath := path.Dir(u.Path)
+	if trimmedPath == "." {
+		trimmedPath = "" // should never happen, given what we pass in, but just in case, let's not return the file-system-ish dot
+	}
+
+	u.Path = trimmedPath
+	return u, nil
+}

azbfs/zc_service_codes_common.go

@@ -99,6 +99,9 @@ const (
 	// ServiceCodeOutOfRangeQueryParameterValue means a query parameter specified in the request URI is outside the permissible range (400).
 	ServiceCodeOutOfRangeQueryParameterValue ServiceCodeType = "OutOfRangeQueryParameterValue"
 
+	/// ServiceCodePathAlreadyExists means that the path (e.g. when trying to create a directory) already exists
+	ServiceCodePathAlreadyExists ServiceCodeType = "PathAlreadyExists"
+
 	// ServiceCodeRequestBodyTooLarge means the size of the request body exceeds the maximum size permitted (413).
 	ServiceCodeRequestBodyTooLarge ServiceCodeType = "RequestBodyTooLarge"
 

azbfs/zt_test.go

@@ -129,7 +129,7 @@ func createNewFileSystem(c *chk.C, fsu azbfs.ServiceURL) (fs azbfs.FileSystemURL
 func createNewDirectoryFromFileSystem(c *chk.C, fileSystem azbfs.FileSystemURL) (dir azbfs.DirectoryURL, name string) {
 	dir, name = getDirectoryURLFromFileSystem(c, fileSystem)
 
-	cResp, err := dir.Create(ctx)
+	cResp, err := dir.Create(ctx, true)
 	c.Assert(err, chk.IsNil)
 	c.Assert(cResp.StatusCode(), chk.Equals, 201)
 	return dir, name

azbfs/zt_url_directory_test.go

@@ -27,7 +27,7 @@ func (dus *DirectoryUrlSuite) TestCreateDeleteDirectory(c *chk.C) {
 
 	// Create a directory url from the fileSystem Url
 	dirUrl, _ := getDirectoryURLFromFileSystem(c, fsURL)
-	cResp, err := dirUrl.Create(context.Background())
+	cResp, err := dirUrl.Create(context.Background(), true)
 	defer deleteDirectory(c, dirUrl)
 
 	// Assert the directory create response header attributes
@@ -49,7 +49,7 @@ func (dus *DirectoryUrlSuite) TestCreateSubDir(c *chk.C) {
 
 	// Create the directory Url from fileSystem Url and create directory
 	dirUrl, _ := getDirectoryURLFromFileSystem(c, fsURL)
-	cResp, err := dirUrl.Create(context.Background())
+	cResp, err := dirUrl.Create(context.Background(), true)
 	defer deleteDirectory(c, dirUrl)
 
 	c.Assert(err, chk.IsNil)
@@ -62,7 +62,7 @@ func (dus *DirectoryUrlSuite) TestCreateSubDir(c *chk.C) {
 
 	// Create the sub-directory url from directory Url and create sub-directory
 	subDirUrl, _ := getDirectoryURLFromDirectory(c, dirUrl)
-	cResp, err = subDirUrl.Create(context.Background())
+	cResp, err = subDirUrl.Create(context.Background(), true)
 	defer deleteDirectory(c, subDirUrl)
 
 	c.Assert(err, chk.IsNil)
@@ -85,7 +85,7 @@ func (dus *DirectoryUrlSuite) TestDirectoryCreateAndGetProperties(c *chk.C) {
 
 	// Create directory url from fileSystemUrl and create directory
 	dirUrl, _ := getDirectoryURLFromFileSystem(c, fsURL)
-	cResp, err := dirUrl.Create(context.Background())
+	cResp, err := dirUrl.Create(context.Background(), true)
 	defer deleteDirectory(c, dirUrl)
 
 	c.Assert(err, chk.IsNil)
@@ -113,7 +113,7 @@ func (dus *DirectoryUrlSuite) TestCreateDirectoryAndFiles(c *chk.C) {
 	// Create the directoryUrl from fileSystemUrl
 	// and create directory
 	dirUrl, _ := getDirectoryURLFromFileSystem(c, fsURL)
-	cResp, err := dirUrl.Create(context.Background())
+	cResp, err := dirUrl.Create(context.Background(), true)
 	defer deleteDirectory(c, dirUrl)
 
 	c.Assert(err, chk.IsNil)
@@ -139,6 +139,35 @@ func (dus *DirectoryUrlSuite) TestCreateDirectoryAndFiles(c *chk.C) {
 
 }
 
+// TestReCreateDirectory tests the creation of directories that already exist
+func (dus *DirectoryUrlSuite) TestReCreateDirectory(c *chk.C) {
+	// Create the file system
+	fsu := getBfsServiceURL()
+	fsURL, _ := createNewFileSystem(c, fsu)
+	defer delFileSystem(c, fsURL)
+
+	// Create the directoryUrl from fileSystemUrl and create directory
+	dirUrl, _ := getDirectoryURLFromFileSystem(c, fsURL)
+	cResp, err := dirUrl.Create(context.Background(), true)
+	defer deleteDirectory(c, dirUrl)
+	c.Assert(err, chk.IsNil)
+	c.Assert(cResp.StatusCode(), chk.Equals, http.StatusCreated)
+
+	// Re-create it (allowing overwrite)
+	// TODO: put some files in it before this, and make assertions about what happens to them after the re-creation
+	cResp, err = dirUrl.Create(context.Background(), true)
+	c.Assert(err, chk.IsNil)
+	c.Assert(cResp.StatusCode(), chk.Equals, http.StatusCreated)
+
+	// Attempt to re-create it (but do NOT allow overwrite)
+	cResp, err = dirUrl.Create(context.Background(), false) // <- false for re-create
+	c.Assert(err, chk.NotNil)
+	stgErr, ok := err.(azbfs.StorageError)
+	c.Assert(ok, chk.Equals, true)
+	c.Assert(stgErr.Response().StatusCode, chk.Equals, http.StatusConflict)
+	c.Assert(stgErr.ServiceCode(), chk.Equals, azbfs.ServiceCodePathAlreadyExists)
+}
+
 // TestDirectoryStructure tests creating dir, sub-dir inside dir and files
 // inside dirs and sub-dirs. Then verify the count of files / sub-dirs inside directory
 func (dus *DirectoryUrlSuite) TestDirectoryStructure(c *chk.C) {
@@ -149,7 +178,7 @@ func (dus *DirectoryUrlSuite) TestDirectoryStructure(c *chk.C) {
 
 	// Create a directory inside filesystem
 	dirUrl, _ := getDirectoryURLFromFileSystem(c, fsURL)
-	cResp, err := dirUrl.Create(context.Background())
+	cResp, err := dirUrl.Create(context.Background(), true)
 	defer deleteDirectory(c, dirUrl)
 
 	c.Assert(err, chk.IsNil)
@@ -162,7 +191,7 @@ func (dus *DirectoryUrlSuite) TestDirectoryStructure(c *chk.C) {
 
 	// Create a sub-dir inside the above create directory
 	subDirUrl, _ := getDirectoryURLFromDirectory(c, dirUrl)
-	cResp, err = subDirUrl.Create(context.Background())
+	cResp, err = subDirUrl.Create(context.Background(), true)
 	defer deleteDirectory(c, subDirUrl)
 
 	c.Assert(err, chk.IsNil)
@@ -225,7 +254,7 @@ func (dus *DirectoryUrlSuite) TestListDirectoryWithSpaces(c *chk.C) {
 
 	// Create a directory inside filesystem
 	dirUrl := fsURL.NewDirectoryURL("New Folder Test 2")
-	_, err := dirUrl.Create(context.Background())
+	_, err := dirUrl.Create(context.Background(), true)
 	defer deleteDirectory(c, dirUrl)
 
 	// Create a file inside directory

azure-pipelines.yml

@@ -32,7 +32,7 @@ jobs:
 
   - job: MacOS_Build
     pool:
-      vmImage: 'xcode9-macos10.13'
+      vmImage: 'macOS-10.14'
     steps:
       - task: GoTool@0
         inputs: