signV4: Avoid encoding ~ when found in the query part of the url (#811)

Ceph radosgw seems to include ~ in the upload ID when the client
initiates a new multipart upload.

minio-py was wrongly encoding ~ in the query part of the url.

This PR ensures that to decode %7E to ~ before signing the request.

minio/api.py

@@ -331,7 +331,8 @@ def make_bucket(self, bucket_name, location='us-east-1'):
                           headers, self._access_key,
                           self._secret_key,
                           self._session_token,
-                          content_sha256_hex)
+                          content_sha256_hex,
+                          datetime.utcnow())
 
         response = self._http.urlopen(method, url,
                                       body=content,
@@ -370,7 +371,7 @@ def list_buckets(self):
                           headers, self._access_key,
                           self._secret_key,
                           self._session_token,
-                          None)
+                          None, datetime.utcnow())
 
         response = self._http.urlopen(method, url,
                                       body=None,
@@ -1864,7 +1865,7 @@ def _get_bucket_location(self, bucket_name):
                           headers, self._access_key,
                           self._secret_key,
                           self._session_token,
-                          None)
+                          None, datetime.utcnow())
 
         response = self._http.urlopen(method, url,
                                       body=None,
@@ -1914,7 +1915,7 @@ def _url_open(self, method, bucket_name=None, object_name=None,
         headers = sign_v4(method, url, region,
                           fold_case_headers, self._access_key,
                           self._secret_key, self._session_token,
-                          content_sha256)
+                          content_sha256, datetime.utcnow())
 
         response = self._http.urlopen(method, url,
                                       body=body,

minio/signer.py

@@ -173,7 +173,8 @@ def sign_v4(method, url, region, headers=None,
             access_key=None,
             secret_key=None,
             session_token=None,
-            content_sha256=None):
+            content_sha256=None,
+            request_datetime=None):
     """
     Signature version 4.
 
@@ -188,6 +189,7 @@ def sign_v4(method, url, region, headers=None,
     :param session_token: Optional session token, set
        only for temporary credentials.
     :param content_sha256: Optional body sha256.
+    :param request_datetime: Optional request date/time
     """
 
     # If no access key or secret key is provided return headers.
@@ -211,8 +213,10 @@ def sign_v4(method, url, region, headers=None,
     host = remove_default_port(parsed_url)
     headers['Host'] = host
 
-    date = datetime.utcnow()
-    headers['X-Amz-Date'] = date.strftime("%Y%m%dT%H%M%SZ")
+    if request_datetime is None:
+        request_datetime = datetime.utcnow()
+
+    headers['X-Amz-Date'] = request_datetime.strftime("%Y%m%dT%H%M%SZ")
     headers['X-Amz-Content-Sha256'] = content_sha256
     if session_token:
         headers['X-Amz-Security-Token'] = session_token
@@ -226,14 +230,14 @@ def sign_v4(method, url, region, headers=None,
                                                signed_headers,
                                                content_sha256)
 
-    string_to_sign = generate_string_to_sign(date, region,
+    string_to_sign = generate_string_to_sign(request_datetime, region,
                                              canonical_req)
-    signing_key = generate_signing_key(date, region, secret_key)
+    signing_key = generate_signing_key(request_datetime, region, secret_key)
     signature = hmac.new(signing_key, string_to_sign.encode('utf-8'),
                          hashlib.sha256).hexdigest()
 
     authorization_header = generate_authorization_header(access_key,
-                                                         date,
+                                                         request_datetime,
                                                          region,
                                                          signed_headers,
                                                          signature)
@@ -253,7 +257,8 @@ def generate_canonical_request(method, parsed_url, headers, signed_headers, cont
     """
     # Should not encode ~. Decode it back if present.
     parsed_url_path = parsed_url.path.replace("%7E", "~")
-    lines = [method, parsed_url_path, parsed_url.query]
+    parsed_url_query = parsed_url.query.replace("%7E", "~")
+    lines = [method, parsed_url_path, parsed_url_query]
 
     # Headers added to canonical request.
     header_lines = []

tests/unit/sign_test.py

@@ -22,10 +22,11 @@
 
 from minio.signer import (generate_canonical_request, generate_string_to_sign,
                           generate_signing_key, generate_authorization_header,
-                          presign_v4)
+                          presign_v4, sign_v4)
 from minio.error import InvalidArgumentError
 from minio.compat import urlsplit, urlencode, queryencode
 from minio.fold_case_dict import FoldCaseDict
+from minio.helpers import get_target_url
 
 empty_hash = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
 dt = datetime(2015, 6, 20, 1, 2, 3, 0, pytz.utc)
@@ -116,6 +117,14 @@ def test_presigned_no_access_key(self):
     def test_presigned_invalid_expires(self):
         presign_v4('GET', 'http://localhost:9000/hello', None, None, region=None, headers={}, expires=0)
 
+class SignV4Test(TestCase):
+    def test_signv4(self):
+        # Construct target url.
+        url = get_target_url('http://localhost:9000', bucket_name='testbucket',
+             object_name='~testobject', bucket_region='us-east-1', query={'partID': '1', 'uploadID': '~abcd'})
+        hdrs = sign_v4('PUT', url, 'us-east-1', access_key='minio', secret_key='minio123', request_datetime=dt)
+        eq_(hdrs['Authorization'], 'AWS4-HMAC-SHA256 Credential=minio/20150620/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a2f4546f647981732bd90dfa5a7599c44dca92f44bea48ecc7565df06032c25b')
+
 class UnicodeEncodeTest(TestCase):
     def test_unicode_urlencode(self):
         eq_(urlencode('/test/123/汉字'), '/test/123/%E6%B1%89%E5%AD%97')