Merge pull request #702 from Azure/dev

10.3.1 Release

ChangeLog.md

@@ -1,6 +1,20 @@
 
 # Change Log
 
+## Version 10.3.1
+
+### New features
+
+1. Added helpful deprecation notice for legacy include/exclude flags.
+1. Added back request ID at log level INFO.
+1. Added back cancel-from-stdin option for partner integration.
+1. Added flag to define delete snapshot options for the remove command.
+
+### Bug fix
+
+1. Fixed race condition in shutdown of decompressingWriter.
+1. Made progress reporting more accurate.
+
 ## Version 10.3.0
 
 ### Breaking changes

cmd/copy.go

@@ -32,6 +32,7 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/Azure/azure-pipeline-go/pipeline"
@@ -73,6 +74,8 @@ type rawCopyCmdArgs struct {
 	excludePath           string
 	includeFileAttributes string
 	excludeFileAttributes string
+	legacyInclude         string // used only for warnings
+	legacyExclude         string // used only for warnings
 
 	// filters from flags
 	listOfFilesToCopy string
@@ -96,6 +99,7 @@ type rawCopyCmdArgs struct {
 	putMd5                   bool
 	md5ValidationOption      string
 	CheckLength              bool
+	deleteSnapshotsOption    string
 	// defines the type of the blob at the destination in case of upload / account to account copy
 	blobType      string
 	blockBlobTier string
@@ -307,10 +311,17 @@ func (raw rawCopyCmdArgs) cookWithId(jobId common.JobID) (cookedCopyCmdArgs, err
 	// This saves us time because we know *exactly* what we're looking for right off the bat.
 	// Note that exclude-path is handled as a filter unlike include-path.
 
+	if raw.legacyInclude != "" || raw.legacyExclude != "" {
+		return cooked, fmt.Errorf("the include and exclude parameters have been replaced by include-pattern; include-path; exclude-pattern and exclude-path. For info, run: azcopy copy help")
+	}
+
 	if (len(raw.include) > 0 || len(raw.exclude) > 0) && cooked.fromTo == common.EFromTo.BlobFSTrash() {
 		return cooked, fmt.Errorf("include/exclude flags are not supported for this destination")
 	}
 
+	// warn on exclude unsupported wildcards here. Include have to be later, to cover list-of-files
+	raw.warnIfHasWildcard(excludeWarningOncer, "exclude-path", raw.excludePath)
+
 	// unbuffered so this reads as we need it to rather than all at once in bulk
 	listChan := make(chan string)
 	var f *os.File
@@ -329,6 +340,14 @@ func (raw rawCopyCmdArgs) cookWithId(jobId common.JobID) (cookedCopyCmdArgs, err
 	go func() {
 		defer close(listChan)
 
+		addToChannel := func(v string, paramName string) {
+			// empty strings should be ignored, otherwise the source root itself is selected
+			if len(v) > 0 {
+				raw.warnIfHasWildcard(includeWarningOncer, paramName, v)
+				listChan <- v
+			}
+		}
+
 		if f != nil {
 			scanner := bufio.NewScanner(f)
 			checkBOM := false
@@ -363,21 +382,15 @@ func (raw rawCopyCmdArgs) cookWithId(jobId common.JobID) (cookedCopyCmdArgs, err
 					headerLineNum++
 				}
 
-				// empty strings should be ignored, otherwise the source root itself is selected
-				if len(v) > 0 {
-					listChan <- v
-				}
+				addToChannel(v, "list-of-files")
 			}
 		}
 
 		// This occurs much earlier than the other include or exclude filters. It would be preferable to move them closer later on in the refactor.
 		includePathList := raw.parsePatterns(raw.includePath)
 
 		for _, v := range includePathList {
-			// empty strings should be ignored, otherwise the source root itself is selected
-			if len(v) > 0 {
-				listChan <- v
-			}
+			addToChannel(v, "include-path")
 		}
 	}()
 
@@ -400,6 +413,12 @@ func (raw rawCopyCmdArgs) cookWithId(jobId common.JobID) (cookedCopyCmdArgs, err
 	cooked.noGuessMimeType = raw.noGuessMimeType
 	cooked.preserveLastModifiedTime = raw.preserveLastModifiedTime
 
+	// Make sure the given input is the one of the enums given by the blob SDK
+	err = cooked.deleteSnapshotsOption.Parse(raw.deleteSnapshotsOption)
+	if err != nil {
+		return cooked, err
+	}
+
 	if cooked.contentType != "" {
 		cooked.noGuessMimeType = true // As specified in the help text, noGuessMimeType is inferred here.
 	}
@@ -589,6 +608,19 @@ func (raw rawCopyCmdArgs) cookWithId(jobId common.JobID) (cookedCopyCmdArgs, err
 	return cooked, nil
 }
 
+var excludeWarningOncer = &sync.Once{}
+var includeWarningOncer = &sync.Once{}
+
+func (raw *rawCopyCmdArgs) warnIfHasWildcard(oncer *sync.Once, paramName string, value string) {
+	if strings.Contains(value, "*") || strings.Contains(value, "?") {
+		oncer.Do(func() {
+			glcm.Info(fmt.Sprintf("*** Warning *** The %s parameter does not support wildcards. The wildcard "+
+				"character provided will be interpreted literally and will not have any wildcard effect. To use wildcards "+
+				"(in filenames only, not paths) use include-pattern or exclude-pattern", paramName))
+		})
+	}
+}
+
 // When other commands use the copy command arguments to cook cook, set the blobType to None and validation option
 // else parsing the arguments will fail.
 func (raw *rawCopyCmdArgs) setMandatoryDefaults() {
@@ -656,6 +688,7 @@ type cookedCopyCmdArgs struct {
 	cacheControl             string
 	noGuessMimeType          bool
 	preserveLastModifiedTime bool
+	deleteSnapshotsOption    common.DeleteSnapshotsOption
 	putMd5                   bool
 	md5ValidationOption      common.HashValidationOption
 	CheckLength              bool
@@ -873,6 +906,7 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) {
 			PreserveLastModifiedTime: cca.preserveLastModifiedTime,
 			PutMd5:                   cca.putMd5,
 			MD5ValidationOption:      cca.md5ValidationOption,
+			DeleteSnapshotsOption:    cca.deleteSnapshotsOption,
 		},
 		// source sas is stripped from the source given by the user and it will not be stored in the part plan file.
 		SourceSAS: cca.sourceSAS,
@@ -1295,6 +1329,12 @@ func init() {
 			} else if len(args) == 2 { // normal copy
 				raw.src = args[0]
 				raw.dst = args[1]
+
+				// under normal copy, we may ask the user questions such as whether to overwrite a file
+				glcm.EnableInputWatcher()
+				if cancelFromStdin {
+					glcm.EnableCancelFromStdIn()
+				}
 			} else {
 				return errors.New("wrong number of arguments, please refer to the help page on usage of this command")
 			}
@@ -1379,6 +1419,12 @@ func init() {
 	cpCmd.PersistentFlags().MarkHidden("list-of-files")
 	cpCmd.PersistentFlags().MarkHidden("s2s-get-properties-in-backend")
 
+	// temp, to assist users with change in param names, by providing a clearer message when these obsolete ones are accidentally used
+	cpCmd.PersistentFlags().StringVar(&raw.legacyInclude, "include", "", "Legacy include param. DO NOT USE")
+	cpCmd.PersistentFlags().StringVar(&raw.legacyExclude, "exclude", "", "Legacy exclude param. DO NOT USE")
+	cpCmd.PersistentFlags().MarkHidden("include")
+	cpCmd.PersistentFlags().MarkHidden("exclude")
+
 	// Hide the flush-threshold flag since it is implemented only for CI.
 	cpCmd.PersistentFlags().Uint32Var(&ste.ADLSFlushThreshold, "flush-threshold", 7500, "Adjust the number of blocks to flush at once on accounts that have a hierarchical namespace.")
 	cpCmd.PersistentFlags().MarkHidden("flush-threshold")

cmd/remove.go

@@ -62,6 +62,11 @@ func init() {
 			return nil
 		},
 		Run: func(cmd *cobra.Command, args []string) {
+			glcm.EnableInputWatcher()
+			if cancelFromStdin {
+				glcm.EnableCancelFromStdIn()
+			}
+
 			cooked, err := raw.cook()
 			if err != nil {
 				glcm.Error("failed to parse user input due to error: " + err.Error())
@@ -86,4 +91,5 @@ func init() {
 	deleteCmd.PersistentFlags().StringVar(&raw.excludePath, "exclude-path", "", "Exclude these paths when removing. "+
 		"This option does not support wildcard characters (*). Checks relative path prefix. For example: myFolder;myFolder/subDirName/file.pdf")
 	deleteCmd.PersistentFlags().StringVar(&raw.listOfFilesToCopy, "list-of-files", "", "Defines the location of a file which contains the list of files and directories to be deleted. The relative paths should be delimited by line breaks, and the paths should NOT be URL-encoded.")
+	deleteCmd.PersistentFlags().StringVar(&raw.deleteSnapshotsOption, "delete-snapshots", "", "By default, the delete operation fails if a blob has snapshots. Specify 'include' to remove the root blob and all its snapshots; alternatively specify 'only' to remove only the snapshots but keep the root blob.")
 }

cmd/removeProcessor.go

@@ -37,7 +37,8 @@ func newRemoveTransferProcessor(cca *cookedCopyCmdArgs, numOfTransfersPerPart in
 		SourceSAS:      cca.sourceSAS,
 
 		// flags
-		LogLevel: cca.logVerbosity,
+		LogLevel:       cca.logVerbosity,
+		BlobAttributes: common.BlobTransferAttributes{DeleteSnapshotsOption: cca.deleteSnapshotsOption},
 	}
 
 	reportFirstPart := func(jobStarted bool) {

cmd/root.go

@@ -39,6 +39,7 @@ var azcopyLogPathFolder string
 var azcopyJobPlanFolder string
 var azcopyMaxFileAndSocketHandles int
 var outputFormatRaw string
+var cancelFromStdin bool
 var azcopyOutputFormat common.OutputFormat
 var cmdLineCapMegaBitsPerSecond uint32
 
@@ -110,6 +111,12 @@ func init() {
 
 	rootCmd.PersistentFlags().Uint32Var(&cmdLineCapMegaBitsPerSecond, "cap-mbps", 0, "Caps the transfer rate, in megabits per second. Moment-by-moment throughput might vary slightly from the cap. If this option is set to zero, or it is omitted, the throughput isn't capped.")
 	rootCmd.PersistentFlags().StringVar(&outputFormatRaw, "output-type", "text", "Format of the command's output. The choices include: text, json. The default value is 'text'.")
+
+	// Note: this is due to Windows not supporting signals properly
+	rootCmd.PersistentFlags().BoolVar(&cancelFromStdin, "cancel-from-stdin", false, "Used by partner teams to send in `cancel` through stdin to stop a job.")
+
+	// reserved for partner teams
+	rootCmd.PersistentFlags().MarkHidden("cancel-from-stdin")
 }
 
 // always spins up a new goroutine, because sometimes the aka.ms URL can't be reached (e.g. a constrained environment where

cmd/sync.go

@@ -47,6 +47,8 @@ type rawSyncCmdArgs struct {
 	exclude               string
 	includeFileAttributes string
 	excludeFileAttributes string
+	legacyInclude         string // for warning messages only
+	legacyExclude         string // for warning messages only
 
 	followSymlinks      bool
 	putMd5              bool
@@ -152,6 +154,11 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
 		return cooked, err
 	}
 
+	// warn on legacy filters
+	if raw.legacyInclude != "" || raw.legacyExclude != "" {
+		return cooked, fmt.Errorf("the include and exclude parameters have been replaced by include-pattern and exclude-pattern. They work on filenames only (not paths)")
+	}
+
 	// parse the filter patterns
 	cooked.include = raw.parsePatterns(raw.include)
 	cooked.exclude = raw.parsePatterns(raw.exclude)
@@ -523,6 +530,11 @@ func init() {
 			return nil
 		},
 		Run: func(cmd *cobra.Command, args []string) {
+			glcm.EnableInputWatcher()
+			if cancelFromStdin {
+				glcm.EnableCancelFromStdIn()
+			}
+
 			cooked, err := raw.cook()
 			if err != nil {
 				glcm.Error("error parsing the input given by the user. Failed with error " + err.Error())
@@ -550,6 +562,12 @@ func init() {
 	syncCmd.PersistentFlags().BoolVar(&raw.putMd5, "put-md5", false, "Create an MD5 hash of each file, and save the hash as the Content-MD5 property of the destination blob or file. (By default the hash is NOT created.) Only available when uploading.")
 	syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "Specifies how strictly MD5 hashes should be validated when downloading. This option is only available when downloading. Available values include: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing. (default 'FailIfDifferent').")
 
+	// temp, to assist users with change in param names, by providing a clearer message when these obsolete ones are accidentally used
+	syncCmd.PersistentFlags().StringVar(&raw.legacyInclude, "include", "", "Legacy include param. DO NOT USE")
+	syncCmd.PersistentFlags().StringVar(&raw.legacyExclude, "exclude", "", "Legacy exclude param. DO NOT USE")
+	syncCmd.PersistentFlags().MarkHidden("include")
+	syncCmd.PersistentFlags().MarkHidden("exclude")
+
 	// TODO follow sym link is not implemented, clarify behavior first
 	//syncCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "follow symbolic links when performing sync from local file system.")
 

cmd/zt_interceptors_for_test.go

@@ -123,6 +123,8 @@ func (*mockedLifecycleManager) GetEnvironmentVariable(env common.EnvironmentVari
 	return value
 }
 func (*mockedLifecycleManager) SetOutputFormat(common.OutputFormat) {}
+func (*mockedLifecycleManager) EnableInputWatcher()                 {}
+func (*mockedLifecycleManager) EnableCancelFromStdIn()              {}
 
 type dummyProcessor struct {
 	record []storedObject

common/chunkStatusLogger.go

@@ -34,6 +34,7 @@ import (
 type ChunkID struct {
 	Name         string
 	offsetInFile int64
+	length       int64
 
 	// What is this chunk's progress currently waiting on?
 	// Must be a pointer, because the ChunkID itself is a struct.
@@ -55,12 +56,13 @@ type ChunkID struct {
 	//   And maybe at that point, we would also put Length into chunkID, and use that in jptm.ReportChunkDone
 }
 
-func NewChunkID(name string, offsetInFile int64) ChunkID {
+func NewChunkID(name string, offsetInFile int64, length int64) ChunkID {
 	dummyWaitReasonIndex := int32(0)
 	zeroNotificationState := int32(0)
 	return ChunkID{
 		Name:                     name,
 		offsetInFile:             offsetInFile,
+		length:                   length,
 		waitReasonIndex:          &dummyWaitReasonIndex, // must initialize, so don't get nil pointer on usage
 		completionNotifiedToJptm: &zeroNotificationState,
 	}
@@ -94,6 +96,10 @@ func (id ChunkID) IsPseudoChunk() bool {
 	return id.offsetInFile < 0
 }
 
+func (id ChunkID) Length() int64 {
+	return id.length
+}
+
 var EWaitReason = WaitReason{0, ""}
 
 // WaitReason identifies the one thing that a given chunk is waiting on, at a given moment.

common/decompressingWriter.go

@@ -65,17 +65,20 @@ func (d decompressingWriter) decompressorFactory(tp CompressionType, preader *io
 
 func (d decompressingWriter) worker(tp CompressionType, preader *io.PipeReader, destination io.WriteCloser, workerError chan error) {
 
+	var err error
+	var dec io.ReadCloser
+
 	defer func() {
 		_ = destination.Close() // always close the destination file before we exit, since its a WriteCloser
 		_ = preader.Close()
+		workerError <- err // send the error AFTER we have closed everything, to avoid race conditions where callers assume all closes are completed when we return
 	}()
 
 	// make the decompressor. Must be in the worker method because,
 	// like the rest of read, this reads from the pipe.
 	// (Factory reads from pipe to read the zip/gzip file header)
-	dec, err := d.decompressorFactory(tp, preader)
+	dec, err = d.decompressorFactory(tp, preader)
 	if err != nil {
-		workerError <- err
 		return
 	}
 
@@ -84,7 +87,7 @@ func (d decompressingWriter) worker(tp CompressionType, preader *io.PipeReader,
 	b := decompressingWriterBufferPool.RentSlice(decompressingWriterCopyBufferSize)
 	_, err = io.CopyBuffer(destination, dec, b) // returns err==nil if hits EOF, as per docs
 	decompressingWriterBufferPool.ReturnSlice(b)
-	workerError <- err
+
 	return
 }
 

common/fe-ste-models.go

@@ -111,6 +111,41 @@ type PartNumber uint32
 type Version uint32
 type Status uint32
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+var EDeleteSnapshotsOption = DeleteSnapshotsOption(0)
+
+type DeleteSnapshotsOption uint8
+
+func (DeleteSnapshotsOption) None() DeleteSnapshotsOption    { return DeleteSnapshotsOption(0) }
+func (DeleteSnapshotsOption) Include() DeleteSnapshotsOption { return DeleteSnapshotsOption(1) }
+func (DeleteSnapshotsOption) Only() DeleteSnapshotsOption    { return DeleteSnapshotsOption(2) }
+
+func (d DeleteSnapshotsOption) String() string {
+	return enum.StringInt(d, reflect.TypeOf(d))
+}
+
+func (d *DeleteSnapshotsOption) Parse(s string) error {
+	// allow empty to mean "None"
+	if s == "" {
+		*d = EDeleteSnapshotsOption.None()
+		return nil
+	}
+
+	val, err := enum.ParseInt(reflect.TypeOf(d), s, true, true)
+	if err == nil {
+		*d = val.(DeleteSnapshotsOption)
+	}
+	return err
+}
+
+func (d DeleteSnapshotsOption) ToDeleteSnapshotsOptionType() azblob.DeleteSnapshotsOptionType {
+	if d == EDeleteSnapshotsOption.None() {
+		return azblob.DeleteSnapshotsOptionNone
+	}
+
+	return azblob.DeleteSnapshotsOptionType(strings.ToLower(d.String()))
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 type DeleteDestination uint32