Implement credentials object (similar to minio-go) to enable AWS IAM (#…

…817)

.github/workflows/pythonpackage-linux.yml

@@ -26,7 +26,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip setuptools
         pip install urllib3 certifi pytz pyflakes faker nose
     - name: Test with nosetests
       run: |

.github/workflows/pythonpackage-windows.yml

@@ -26,7 +26,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip setuptools
         pip install urllib3 certifi pytz pyflakes faker nose
     - name: Test with nosetests
       run: |

examples/aws_credentials.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# IamEc2MetaData will call AWS metadata service to retrieve credentials
+#
+# The default AWS metadata service can be found at:
+#   -> 169.254.169.254/latest/meta-data/iam/security-credentials
+#
+# If you wish to retieve credentials from a different place you can provide 
+# the 'endpoint' paramater to the IamEc2MetaData credentials object
+
+from minio.credentials import IamEc2MetaData
+
+# Initialize Minio with IamEc2MetaData default credentials object
+client = Minio('s3.amazonaws.com', 
+               credentials=IamEc2MetaData())
+
+# Initialize Minio with IamEc2MetaData custom 
+
+client = Minio('s3.amazonaws.com', 
+               credentials=IamEc2MetaData(endpoint='custom.endpoint'))

examples/chain_credentials.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A Chain credentials provider, provides a way of chaining multiple providers together
+# and will pick the first available using priority order of the 'providers' list
+
+from minio.credentials import Chain, EnvAWS, EnvMinio, IamEc2MetaData
+
+client = Minio('s3.amazonaws.com', 
+               credentials=Chain(
+                   providers=[
+                       IamEc2MetaData(),
+                       EnvAWS(),
+                       EnvMinio()
+                   ]
+               ))

minio/api.py

@@ -94,6 +94,12 @@
 from .fold_case_dict import FoldCaseDict
 from .thread_pool import ThreadPool
 from .select import SelectObjectReader
+from .credentials import (Chain,
+                          Credentials,
+                          EnvAWS,
+                          EnvMinio,
+                          IamEc2MetaData,
+                          Static)
 
 # Comment format.
 _COMMENTS = '({0}; {1})'
@@ -144,7 +150,8 @@ def __init__(self, endpoint, access_key=None,
                  session_token=None,
                  secure=True,
                  region=None,
-                 http_client=None):
+                 http_client=None,
+                 credentials=None):
 
         # Validate endpoint.
         is_valid_endpoint(endpoint)
@@ -176,6 +183,22 @@ def __init__(self, endpoint, access_key=None,
         self._user_agent = _DEFAULT_USER_AGENT
         self._trace_output_stream = None
 
+        # Set credentials if possible
+
+        if credentials is not None:
+            self._credentials = credentials
+        else:
+            self._credentials = Credentials(
+                provider= Chain(
+                    providers=[
+                        Static(access_key, secret_key, session_token),
+                        EnvAWS(),
+                        EnvMinio(),
+                        IamEc2MetaData(),
+                    ]
+                )
+            ) 
+
         # Load CA certificates from SSL_CERT_FILE file if set
         ca_certs = os.environ.get('SSL_CERT_FILE')
         if not ca_certs:
@@ -328,9 +351,8 @@ def make_bucket(self, bucket_name, location='us-east-1'):
 
         # Get signature headers if any.
         headers = sign_v4(method, url, region,
-                          headers, self._access_key,
-                          self._secret_key,
-                          self._session_token,
+                          headers,
+                          self._credentials,
                           content_sha256_hex,
                           datetime.utcnow())
 
@@ -368,9 +390,8 @@ def list_buckets(self):
 
         # Get signature headers if any.
         headers = sign_v4(method, url, region,
-                          headers, self._access_key,
-                          self._secret_key,
-                          self._session_token,
+                          headers,
+                          self._credentials,
                           None, datetime.utcnow())
 
         response = self._http.urlopen(method, url,
@@ -1361,9 +1382,7 @@ def presigned_url(self, method,
                              bucket_region=region)
 
         return presign_v4(method, url,
-                          self._access_key,
-                          self._secret_key,
-                          session_token=self._session_token,
+                          credentials = self._credentials,
                           region=region,
                           expires=int(expires.total_seconds()),
                           response_headers=response_headers,
@@ -1452,7 +1471,7 @@ def presigned_post_policy(self, post_policy):
         date = datetime.utcnow()
         iso8601_date = date.strftime("%Y%m%dT%H%M%SZ")
         region = self._get_bucket_region(post_policy.form_data['bucket'])
-        credential_string = generate_credential_string(self._access_key,
+        credential_string = generate_credential_string(self._credentials.get().access_key,
                                                        date, region)
 
         policy = [
@@ -1465,7 +1484,7 @@ def presigned_post_policy(self, post_policy):
 
         post_policy_base64 = post_policy.base64(extras=policy)
         signature = post_presign_signature(date, region,
-                                           self._secret_key,
+                                           self._credentials.get().secret_key,
                                            post_policy_base64)
         form_data = {
             'policy': post_policy_base64,
@@ -1854,14 +1873,13 @@ def _get_bucket_location(self, bucket_name):
             return self._region
 
         # For anonymous requests no need to get bucket location.
-        if self._access_key is None or self._secret_key is None:
+        if self._credentials.get().access_key is None or self._credentials.get().secret_key is None:
             return 'us-east-1'
 
         # Get signature headers if any.
         headers = sign_v4(method, url, region,
-                          headers, self._access_key,
-                          self._secret_key,
-                          self._session_token,
+                          headers,
+                          self._credentials,
                           None, datetime.utcnow())
 
         response = self._http.urlopen(method, url,
@@ -1910,8 +1928,8 @@ def _url_open(self, method, bucket_name=None, object_name=None,
 
         # Get signature headers if any.
         headers = sign_v4(method, url, region,
-                          fold_case_headers, self._access_key,
-                          self._secret_key, self._session_token,
+                          fold_case_headers,
+                          self._credentials,
                           content_sha256, datetime.utcnow())
 
         response = self._http.urlopen(method, url,

minio/credentials/__init__.py

@@ -0,0 +1,6 @@
+from .static import Static
+from .credentials import Credentials
+from .chain import Chain
+from .aws_iam import IamEc2MetaData
+from .env_aws import EnvAWS
+from .env_minio import EnvMinio

minio/credentials/aws_iam.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C)
+# 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, json, urllib3, datetime
+from .credentials import Provider, Value, Expiry
+from minio.error import ResponseError
+
+class IamEc2MetaData(Provider):
+
+    iam_security_creds_path = '/latest/meta-data/iam/security-credentials'
+
+    default_expiry_window = datetime.timedelta(minutes=5)
+
+    def __init__(self, endpoint=None):
+        super(Provider, self).__init__()
+        if endpoint == "" or endpoint is None:
+            endpoint = "http://169.254.169.254"
+        self._endpoint = endpoint
+        self._expiry = Expiry()
+        self._http_client = urllib3.PoolManager(
+                retries=urllib3.Retry(
+                    total=5,
+                    backoff_factor=0.2,
+                    status_forcelist=[500, 502, 503, 504]
+                    )
+                )
+
+    def request_cred_list(self):
+        url = self._endpoint + self.iam_security_creds_path
+        try:
+            res = self._http_client.urlopen('GET', url)
+            if res['status'] != 200:
+                return []
+        except:
+            return []
+        creds = res['data'].split('\n')
+        return creds
+
+    def request_cred(self, creds_name):
+        url = self._endpoint + self.iam_security_creds_path + "/" + creds_name
+        res = self._http_client.urlopen('GET', url)
+        if res['status'] != 200:
+            raise ResponseError(res, 'GET')
+
+        data = json.loads(res['data'])
+        if data['Code'] != 'Success':
+            raise ResponseError(res)
+
+        return data
+
+    def retrieve(self):
+        creds_list = self.request_cred_list()
+        if len(creds_list) == 0:
+            return Value()
+
+        creds_name = creds_list[0]
+        role_creds = self.request_cred(creds_name)
+        expiration = datetime.datetime.strptime(role_creds['Expiration'], '%Y-%m-%dT%H:%M:%SZ')
+        self._expiry.set_expiration(expiration, self.default_expiry_window)
+
+        return Value(
+            access_key=role_creds['AccessKeyId'],
+            secret_key=role_creds['SecretAccessKey'],
+            session_token=role_creds['Token']
+        )
+
+    def is_expired(self):
+        return self._expiry.is_expired()

minio/credentials/chain.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C)
+# 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .credentials import Provider, Value
+
+class Chain(Provider):
+    def __init__(self, providers):
+        super(Provider, self).__init__()
+        self._providers = providers
+        self._current = None
+
+    def retrieve(self):
+        for provider in self._providers:
+            creds = provider.retrieve()
+            if ((creds.access_key is None or creds.access_key == "") and
+                (creds.secret_key is None or creds.secret_key == "")):
+                continue
+            self._current = provider
+            return creds
+        self._current = None
+        return Value()
+
+    def is_expired(self):
+        if self._current == None:
+            return True
+        return self._current.is_expired()

minio/credentials/config.json.sample

@@ -0,0 +1,17 @@
+{
+	"version": "8",
+	"hosts": {
+		"play": {
+			"url": "https://play.minio.io:9000",
+			"accessKey": "Q3AM3UQ867SPQQA43P2F",
+			"secretKey": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
+			"api": "S3v2"
+		},
+		"s3": {
+			"url": "https://s3.amazonaws.com",
+			"accessKey": "accessKey",
+			"secretKey": "secret",
+			"api": "S3v4"
+		}
+	}
+}