Add support for SSE-C encryption

Changes implement 2 new flags --sse-customer-key and
--sse-copy-source-customer-key that can be used by user to
provide a key for server side encryption.

Once these options are set extra headers are added to request
accordingly to SSE-C specification [1]

We also ensure that object_get respects provided extra_headers

This PR squashes and rebases on current master changes
implemented by @jheller

[1] https://docs.aws.amazon.com/AmazonS3/latest/userguide/specifying-s3-c-encryption.html

S3/Config.py

@@ -132,6 +132,8 @@ class Config(object):
     extra_headers = SortedDict(ignore_case = True)
     force = False
     server_side_encryption = False
+    sse_customer_key = ""
+    sse_copy_source_customer_key = ""
     enable = None
     get_continue = False
     put_continue = False
@@ -293,6 +295,10 @@ def __init__(self, configfile = None, access_key=None, secret_key=None, access_t
                 warning('Cannot have server_side_encryption (S3 SSE) and KMS_key set (S3 KMS). KMS encryption will be used. Please set server_side_encryption to False')
             if self.kms_key and self.signature_v2 == True:
                 raise Exception('KMS encryption requires signature v4. Please set signature_v2 to False')
+            if self.sse_customer_key and len(self.sse_customer_key) != 32:
+                raise Exception('sse-customer-key must be 32 characters')
+            if self.sse_copy_source_customer_key and len(self.sse_copy_source_customer_key) != 32:
+                raise Exception('sse_copy_source_customer_key must be 32 characters')
 
     def role_config(self):
         """

S3/FileDict.py

@@ -43,7 +43,7 @@ def get_md5(self, relative_file):
         if 'md5' in self[relative_file]:
             return self[relative_file]['md5']
         md5 = self.get_hardlink_md5(relative_file)
-        if md5 is None and 'md5' in cfg.sync_checks:
+        if md5 is None and 'md5' in cfg.preserve_attrs_list:
             logging.debug(u"doing file I/O to read md5 of %s" % relative_file)
             md5 = Utils.hash_file_md5(self[relative_file]['full_name'])
         self.record_md5(relative_file, md5)

S3/FileLists.py

@@ -552,6 +552,23 @@ def _compare(src_list, dst_lst, src_remote, dst_remote, file):
                 attribs_match = False
                 debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))
 
+        # Check mtime. This compares local mtime to the upload time of remote file
+        compare_mtime = 'mtime' in cfg.sync_checks
+        if attribs_match and compare_mtime:
+            try:
+                src_mtime = src_list[file]['mtime']
+                dst_mtime = dst_list[file]['timestamp']
+            except (IOError,OSError):
+                # mtime sum verification failed - ignore that file altogether
+                debug(u"IGNR: %s (disappeared)" % (file))
+                warning(u"%s: file disappeared, ignoring." % (file))
+                raise
+
+            if src_mtime > dst_mtime:
+                ## checksums are different.
+                attribs_match = False
+                debug(u"XFER: %s (mtime newer than last upload: src=%s dst=%s)" % (file, src_mtime, dst_mtime))
+
         return attribs_match
 
     # we don't support local->local sync, use 'rsync' or something like that instead ;-)

S3/S3.py

@@ -8,17 +8,20 @@
 
 from __future__ import absolute_import, division
 
-import sys
-import os
-import time
+import base64
 import errno
-import mimetypes
+import hashlib
 import io
+import mimetypes
+import os
 import pprint
-from xml.sax import saxutils
+import sys
+import time
+from logging import debug, error, info, warning
 from socket import timeout as SocketTimeoutException
-from logging import debug, info, warning, error
 from stat import ST_SIZE
+from xml.sax import saxutils
+
 try:
     # python 3 support
     from urlparse import urlparse
@@ -38,26 +41,27 @@
 except ImportError:
     from md5 import md5
 
-from .BaseUtils import (getListFromXml, getTextFromXml, getRootTagName,
-                        decode_from_s3, encode_to_s3, s3_quote)
-from .Utils import (convertHeaderTupleListToDict, hash_file_md5, unicodise,
-                    deunicodise, check_bucket_name,
-                    check_bucket_name_dns_support, getHostnameFromBucket,
-                    calculateChecksum)
-from .SortedDict import SortedDict
 from .AccessLog import AccessLog
 from .ACL import ACL, GranteeLogDelivery
+from .BaseUtils import (decode_from_s3, encode_to_s3, getListFromXml,
+                        getRootTagName, getTextFromXml, s3_quote)
 from .BidirMap import BidirMap
 from .Config import Config
+from .ConnMan import ConnMan
+from .Crypto import (checksum_sha256_buffer, checksum_sha256_file,
+                     format_param_str, sign_request_v2, sign_request_v4)
 from .Exceptions import *
 from .MultiPart import MultiPartUpload
 from .S3Uri import S3Uri
-from .ConnMan import ConnMan
-from .Crypto import (sign_request_v2, sign_request_v4, checksum_sha256_file,
-                     checksum_sha256_buffer, format_param_str)
+from .SortedDict import SortedDict
+from .Utils import (calculateChecksum, check_bucket_name,
+                    check_bucket_name_dns_support,
+                    convertHeaderTupleListToDict, deunicodise,
+                    getHostnameFromBucket, hash_file_md5, unicodise)
 
 try:
     from ctypes import ArgumentError
+
     import magic
     try:
         ## https://github.com/ahupp/python-magic
@@ -701,6 +705,16 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
             headers['x-amz-server-side-encryption'] = 'aws:kms'
             headers['x-amz-server-side-encryption-aws-kms-key-id'] = self.config.kms_key
 
+        if self.config.sse_customer_key:
+            md5 = hashlib.md5()
+            sse_customer_key = self.config.sse_customer_key.encode()
+            md5.update(sse_customer_key)
+            md5_encoded = base64.b64encode(md5.digest())
+            encoded = base64.b64encode(sse_customer_key)
+            headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
+            headers["x-amz-server-side-encryption-customer-key"] = encoded.decode()
+            headers["x-amz-server-side-encryption-customer-key-md5"] = md5_encoded.decode()
+
         ## MIME-type handling
         headers["content-type"] = self.content_type(filename=filename)
 
@@ -755,10 +769,32 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
         response = self.send_file(request, src_stream, labels)
         return response
 
-    def object_get(self, uri, stream, dest_name, start_position = 0, extra_label = ""):
+    def object_get(self, uri, stream, dest_name, extra_headers, start_position = 0, extra_label = ""):
         if uri.type != "s3":
             raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
-        request = self.create_request("OBJECT_GET", uri = uri)
+        headers = SortedDict(ignore_case=True)
+        if extra_headers:
+            headers.update(extra_headers)
+        ## Set server side encryption
+        if self.config.server_side_encryption:
+            headers["x-amz-server-side-encryption"] = "AES256"
+
+        ## Set kms headers
+        if self.config.kms_key:
+            headers['x-amz-server-side-encryption'] = 'aws:kms'
+            headers['x-amz-server-side-encryption-aws-kms-key-id'] = self.config.kms_key
+
+        if self.config.sse_customer_key:
+            md5 = hashlib.md5()
+            sse_customer_key = self.config.sse_customer_key.encode()
+            md5.update(sse_customer_key)
+            md5_encoded = base64.b64encode(md5.digest())
+            encoded = base64.b64encode(sse_customer_key)
+            headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
+            headers["x-amz-server-side-encryption-customer-key"] = encoded.decode()
+            headers["x-amz-server-side-encryption-customer-key-md5"] = md5_encoded.decode()
+
+        request = self.create_request("OBJECT_GET", uri = uri, headers=headers)
         labels = { 'source' : uri.uri(), 'destination' : dest_name, 'extra' : extra_label }
         response = self.recv_file(request, stream, labels, start_position)
         return response
@@ -954,6 +990,16 @@ def object_copy(self, src_uri, dst_uri, extra_headers=None,
             headers['x-amz-server-side-encryption-aws-kms-key-id'] = \
                 self.config.kms_key
 
+        if self.config.sse_copy_source_customer_key:
+            md5 = hashlib.md5()
+            sse_copy_source_customer_key = self.config.sse_copy_source_customer_key.encode()
+            md5.update(sse_copy_source_customer_key)
+            md5_encoded = base64.b64encode(md5.digest())
+            encoded = base64.b64encode(sse_copy_source_customer_key)
+            headers["x-amz-copy-source-server-side-encryption-customer-algorithm"] = "AES256"
+            headers["x-amz-copy-source-server-side-encryption-customer-key"] = encoded.decode()
+            headers["x-amz-copy-source-server-side-encryption-customer-key-md5"] = md5_encoded.decode()
+
         # Following meta data are not updated in simple COPY by aws.
         if extra_headers:
             headers.update(extra_headers)
@@ -1828,19 +1874,32 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0,
             ## Non-recoverable error
             raise S3Error(response)
 
-        debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\'')))
-        ## when using KMS encryption, MD5 etag value will not match
-        md5_from_s3 = response["headers"].get("etag", "").strip('"\'')
-        if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms':
-            warning("MD5 Sums don't match!")
-            if retries:
-                warning("Retrying upload of %s" % (filename))
-                return self.send_file(request, stream, labels, buffer, throttle,
-                                      retries - 1, offset, chunk_size, use_expect_continue)
+        if self.config.sse_customer_key:
+            if response["headers"]["x-amz-server-side-encryption-customer-key-md5"] != \
+                request.headers["x-amz-server-side-encryption-customer-key-md5"]:
+                warning("MD5 of customer key don't match!")
+                if retries:
+                    warning("Retrying upload of %s" % (filename))
+                    return self.send_file(request, stream, labels, buffer, throttle, retries - 1, offset, chunk_size)
+                else:
+                    warning("Too many failures. Giving up on '%s'" % (filename))
+                    raise S3UploadError
             else:
-                warning("Too many failures. Giving up on '%s'" % (filename))
-                raise S3UploadError("Too many failures. Giving up on '%s'"
-                                    % filename)
+                debug("Match of x-amz-server-side-encryption-customer-key-md5")
+        else:
+            debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\'')))
+            ## when using KMS encryption, MD5 etag value will not match
+            md5_from_s3 = response["headers"].get("etag", "").strip('"\'')
+            if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms':
+                warning("MD5 Sums don't match!")
+                if retries:
+                    warning("Retrying upload of %s" % (filename))
+                    return self.send_file(request, stream, labels, buffer, throttle,
+                                        retries - 1, offset, chunk_size, use_expect_continue)
+                else:
+                    warning("Too many failures. Giving up on '%s'" % (filename))
+                    raise S3UploadError("Too many failures. Giving up on '%s'"
+                                        % filename)
 
         return response
 

s3cmd

@@ -531,6 +531,8 @@ def cmd_object_get(args):
 
     remote_count = len(remote_list)
 
+    extra_headers = copy(cfg.extra_headers)
+
     info(u"Summary: %d remote files to download" % remote_count)
 
     if remote_count > 0:
@@ -618,7 +620,7 @@ def cmd_object_get(args):
 
         try:
             try:
-                response = s3.object_get(uri, dst_stream, destination, start_position = start_position, extra_label = seq_label)
+                response = s3.object_get(uri, dst_stream, destination, extra_headers, start_position = start_position, extra_label = seq_label)
             finally:
                 dst_stream.close()
         except S3DownloadError as e:
@@ -1341,6 +1343,7 @@ def cmd_sync_remote2local(args):
     def _download(remote_list, seq, total, total_size, dir_cache):
         original_umask = os.umask(0)
         os.umask(original_umask)
+        extra_headers = copy(cfg.extra_headers)
         file_list = remote_list.keys()
         file_list.sort()
         ret = EX_OK
@@ -1375,7 +1378,7 @@ def cmd_sync_remote2local(args):
                     with io.open(chkptfd, mode='wb') as dst_stream:
                         dst_stream.stream_name = unicodise(chkptfname_b)
                         debug(u"created chkptfname=%s" % dst_stream.stream_name)
-                        response = s3.object_get(uri, dst_stream, dst_file, extra_label = seq_label)
+                        response = s3.object_get(uri, dst_stream, dst_file, extra_headers, extra_label = seq_label)
 
                     # download completed, rename the file to destination
                     if os.name == "nt":
@@ -1920,6 +1923,15 @@ def cmd_sync_local2remote(args):
         error(u"or disable encryption with --no-encrypt parameter.")
         sys.exit(EX_USAGE)
 
+    # Disable md5 checks if using SSE-C. Add mtime check
+    if cfg.sse_customer_key or cfg.sse_copy_source_customer_key:
+        try:
+            cfg.sync_checks.remove("md5")
+        except Exception:
+            pass
+        if cfg.sync_checks.count("mtime") == 0:
+            cfg.sync_checks.append("mtime")
+
     for arg in args[:-1]:
         if not os.path.exists(deunicodise(arg)):
             raise ParameterError("Invalid source: '%s' is not an existing file or directory" % arg)
@@ -2365,6 +2377,8 @@ def run_configure(config_file, args):
         ("gpg_passphrase", "Encryption password", "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3"),
         ("gpg_command", "Path to GPG program"),
         ("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP, and can only be proxied with Python 2.7 or newer"),
+        ("sse_customer_key", "Encryption key for server-side-encryption with customer key.\nMust be 32 characters"),
+        ("sse_copy_source_customer_key", "Encryption key for server-side-encryption with customer key.\nMust be 32 characters"),
         ("proxy_host", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't connect to S3 directly"),
         ("proxy_port", "HTTP Proxy server port"),
         ]
@@ -2804,6 +2818,8 @@ def main():
 
     optparser.add_option(      "--server-side-encryption", dest="server_side_encryption", action="store_true", help="Specifies that server-side encryption will be used when putting objects. [put, sync, cp, modify]")
     optparser.add_option(      "--server-side-encryption-kms-id", dest="kms_key", action="store", help="Specifies the key id used for server-side encryption with AWS KMS-Managed Keys (SSE-KMS) when putting objects. [put, sync, cp, modify]")
+    optparser.add_option(      "--sse-customer-key", dest="sse_customer_key", action="store", metavar="12345678901234567890123456789012", help="Specifies a customer provided key for server-side encryption. Must be 32 character string.")
+    optparser.add_option(      "--sse-copy-source-customer-key", dest="sse_copy_source_customer_key", action="store", metavar="12345678901234567890123456789012", help="Specifies the encryption key for copying or moving objects with a customer provided key for server-side encryption.")
 
     optparser.add_option(      "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % autodetected_encoding)
     optparser.add_option(      "--add-encoding-exts", dest="add_encoding_exts", metavar="EXTENSIONs", help="Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )")

s3cmd.1

@@ -417,6 +417,14 @@ Specifies the key id used for server\-side encryption
 with AWS KMS\-Managed Keys (SSE\-KMS) when putting
 objects. [put, sync, cp, modify]
 .TP
+\fB\-\-sse\-customer\-key\fR=12345678901234567890123456789012
+Specifies a customer key for server-side encryption, to be used
+when putting objects. Must be 32 characters.
+.TP
+\fB\-\-sse\-copy\-source\-customer\-key\fR=12345678901234567890123456789012
+Specifies the key for copying objects with server-side
+encryption customer key. Must be 32 characters.
+.TP
 \fB\-\-encoding\fR=ENCODING
 Override autodetected terminal and filesystem encoding
 (character set). Autodetected: UTF\-8