From c5d66fc1d5c498e896baf9f0d1403319f964c8aa Mon Sep 17 00:00:00 2001 From: Felipe Ballesteros <54366973+febg@users.noreply.github.com> Date: Wed, 22 Jan 2020 16:22:47 -0800 Subject: [PATCH] Implement credentials object (similar to minio-go) to enable AWS IAM (#817) --- .github/workflows/pythonpackage-linux.yml | 2 +- .github/workflows/pythonpackage-windows.yml | 2 +- examples/aws_credentials.py | 33 ++++++ examples/chain_credentials.py | 28 +++++ minio/api.py | 54 ++++++--- minio/credentials/__init__.py | 6 + minio/credentials/aws_iam.py | 81 ++++++++++++++ minio/credentials/chain.py | 39 +++++++ minio/credentials/config.json.sample | 17 +++ minio/credentials/credentials.empty | 0 minio/credentials/credentials.py | 72 ++++++++++++ minio/credentials/credentials.sample | 12 ++ minio/credentials/env_aws.py | 45 ++++++++ minio/credentials/env_minio.py | 42 +++++++ minio/credentials/file_aws_credentials.py | 63 +++++++++++ minio/credentials/file_minio_client.py | 58 ++++++++++ minio/credentials/static.py | 34 ++++++ minio/signer.py | 39 +++---- setup.py | 2 + tests/unit/chain_provider_test.py | 80 +++++++++++++ tests/unit/credentials_test.py | 43 +++++++ tests/unit/env_aws_provider_test.py | 73 ++++++++++++ tests/unit/env_minio_provider_test.py | 45 ++++++++ tests/unit/file_aws_credentials_test.py | 118 ++++++++++++++++++++ tests/unit/file_minio_client_test.py | 87 +++++++++++++++ tests/unit/iam_aws_test.py | 50 +++++++++ tests/unit/minio_mocks.py | 12 +- tests/unit/sign_test.py | 19 +++- tests/unit/static_test.py | 54 +++++++++ 29 files changed, 1160 insertions(+), 50 deletions(-) create mode 100644 examples/aws_credentials.py create mode 100644 examples/chain_credentials.py create mode 100644 minio/credentials/__init__.py create mode 100644 minio/credentials/aws_iam.py create mode 100644 minio/credentials/chain.py create mode 100644 minio/credentials/config.json.sample create mode 100644 minio/credentials/credentials.empty create mode 100644 minio/credentials/credentials.py create mode 100644 minio/credentials/credentials.sample create mode 100644 minio/credentials/env_aws.py create mode 100644 minio/credentials/env_minio.py create mode 100644 minio/credentials/file_aws_credentials.py create mode 100644 minio/credentials/file_minio_client.py create mode 100644 minio/credentials/static.py create mode 100644 tests/unit/chain_provider_test.py create mode 100644 tests/unit/credentials_test.py create mode 100644 tests/unit/env_aws_provider_test.py create mode 100644 tests/unit/env_minio_provider_test.py create mode 100644 tests/unit/file_aws_credentials_test.py create mode 100644 tests/unit/file_minio_client_test.py create mode 100644 tests/unit/iam_aws_test.py create mode 100644 tests/unit/static_test.py diff --git a/.github/workflows/pythonpackage-linux.yml b/.github/workflows/pythonpackage-linux.yml index 0dfecd3f..dc07c900 100644 --- a/.github/workflows/pythonpackage-linux.yml +++ b/.github/workflows/pythonpackage-linux.yml @@ -26,7 +26,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip setuptools pip install urllib3 certifi pytz pyflakes faker nose - name: Test with nosetests run: | diff --git a/.github/workflows/pythonpackage-windows.yml b/.github/workflows/pythonpackage-windows.yml index 39812292..38221946 100644 --- a/.github/workflows/pythonpackage-windows.yml +++ b/.github/workflows/pythonpackage-windows.yml @@ -26,7 +26,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip setuptools pip install urllib3 certifi pytz pyflakes faker nose - name: Test with nosetests run: | diff --git a/examples/aws_credentials.py b/examples/aws_credentials.py new file mode 100644 index 00000000..a39755c8 --- /dev/null +++ b/examples/aws_credentials.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# IamEc2MetaData will call AWS metadata service to retrieve credentials +# +# The default AWS metadata service can be found at: +# -> 169.254.169.254/latest/meta-data/iam/security-credentials +# +# If you wish to retieve credentials from a different place you can provide +# the 'endpoint' paramater to the IamEc2MetaData credentials object + +from minio.credentials import IamEc2MetaData + +# Initialize Minio with IamEc2MetaData default credentials object +client = Minio('s3.amazonaws.com', + credentials=IamEc2MetaData()) + +# Initialize Minio with IamEc2MetaData custom + +client = Minio('s3.amazonaws.com', + credentials=IamEc2MetaData(endpoint='custom.endpoint')) \ No newline at end of file diff --git a/examples/chain_credentials.py b/examples/chain_credentials.py new file mode 100644 index 00000000..fe31ebc2 --- /dev/null +++ b/examples/chain_credentials.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A Chain credentials provider, provides a way of chaining multiple providers together +# and will pick the first available using priority order of the 'providers' list + +from minio.credentials import Chain, EnvAWS, EnvMinio, IamEc2MetaData + +client = Minio('s3.amazonaws.com', + credentials=Chain( + providers=[ + IamEc2MetaData(), + EnvAWS(), + EnvMinio() + ] + )) \ No newline at end of file diff --git a/minio/api.py b/minio/api.py index 6a7c844b..f14dd6c9 100644 --- a/minio/api.py +++ b/minio/api.py @@ -94,6 +94,12 @@ from .fold_case_dict import FoldCaseDict from .thread_pool import ThreadPool from .select import SelectObjectReader +from .credentials import (Chain, + Credentials, + EnvAWS, + EnvMinio, + IamEc2MetaData, + Static) # Comment format. _COMMENTS = '({0}; {1})' @@ -144,7 +150,8 @@ def __init__(self, endpoint, access_key=None, session_token=None, secure=True, region=None, - http_client=None): + http_client=None, + credentials=None): # Validate endpoint. is_valid_endpoint(endpoint) @@ -176,6 +183,22 @@ def __init__(self, endpoint, access_key=None, self._user_agent = _DEFAULT_USER_AGENT self._trace_output_stream = None + # Set credentials if possible + + if credentials is not None: + self._credentials = credentials + else: + self._credentials = Credentials( + provider= Chain( + providers=[ + Static(access_key, secret_key, session_token), + EnvAWS(), + EnvMinio(), + IamEc2MetaData(), + ] + ) + ) + # Load CA certificates from SSL_CERT_FILE file if set ca_certs = os.environ.get('SSL_CERT_FILE') if not ca_certs: @@ -328,9 +351,8 @@ def make_bucket(self, bucket_name, location='us-east-1'): # Get signature headers if any. headers = sign_v4(method, url, region, - headers, self._access_key, - self._secret_key, - self._session_token, + headers, + self._credentials, content_sha256_hex, datetime.utcnow()) @@ -368,9 +390,8 @@ def list_buckets(self): # Get signature headers if any. headers = sign_v4(method, url, region, - headers, self._access_key, - self._secret_key, - self._session_token, + headers, + self._credentials, None, datetime.utcnow()) response = self._http.urlopen(method, url, @@ -1361,9 +1382,7 @@ def presigned_url(self, method, bucket_region=region) return presign_v4(method, url, - self._access_key, - self._secret_key, - session_token=self._session_token, + credentials = self._credentials, region=region, expires=int(expires.total_seconds()), response_headers=response_headers, @@ -1452,7 +1471,7 @@ def presigned_post_policy(self, post_policy): date = datetime.utcnow() iso8601_date = date.strftime("%Y%m%dT%H%M%SZ") region = self._get_bucket_region(post_policy.form_data['bucket']) - credential_string = generate_credential_string(self._access_key, + credential_string = generate_credential_string(self._credentials.get().access_key, date, region) policy = [ @@ -1465,7 +1484,7 @@ def presigned_post_policy(self, post_policy): post_policy_base64 = post_policy.base64(extras=policy) signature = post_presign_signature(date, region, - self._secret_key, + self._credentials.get().secret_key, post_policy_base64) form_data = { 'policy': post_policy_base64, @@ -1854,14 +1873,13 @@ def _get_bucket_location(self, bucket_name): return self._region # For anonymous requests no need to get bucket location. - if self._access_key is None or self._secret_key is None: + if self._credentials.get().access_key is None or self._credentials.get().secret_key is None: return 'us-east-1' # Get signature headers if any. headers = sign_v4(method, url, region, - headers, self._access_key, - self._secret_key, - self._session_token, + headers, + self._credentials, None, datetime.utcnow()) response = self._http.urlopen(method, url, @@ -1910,8 +1928,8 @@ def _url_open(self, method, bucket_name=None, object_name=None, # Get signature headers if any. headers = sign_v4(method, url, region, - fold_case_headers, self._access_key, - self._secret_key, self._session_token, + fold_case_headers, + self._credentials, content_sha256, datetime.utcnow()) response = self._http.urlopen(method, url, diff --git a/minio/credentials/__init__.py b/minio/credentials/__init__.py new file mode 100644 index 00000000..33691886 --- /dev/null +++ b/minio/credentials/__init__.py @@ -0,0 +1,6 @@ +from .static import Static +from .credentials import Credentials +from .chain import Chain +from .aws_iam import IamEc2MetaData +from .env_aws import EnvAWS +from .env_minio import EnvMinio diff --git a/minio/credentials/aws_iam.py b/minio/credentials/aws_iam.py new file mode 100644 index 00000000..49882815 --- /dev/null +++ b/minio/credentials/aws_iam.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os, json, urllib3, datetime +from .credentials import Provider, Value, Expiry +from minio.error import ResponseError + +class IamEc2MetaData(Provider): + + iam_security_creds_path = '/latest/meta-data/iam/security-credentials' + + default_expiry_window = datetime.timedelta(minutes=5) + + def __init__(self, endpoint=None): + super(Provider, self).__init__() + if endpoint == "" or endpoint is None: + endpoint = "http://169.254.169.254" + self._endpoint = endpoint + self._expiry = Expiry() + self._http_client = urllib3.PoolManager( + retries=urllib3.Retry( + total=5, + backoff_factor=0.2, + status_forcelist=[500, 502, 503, 504] + ) + ) + + def request_cred_list(self): + url = self._endpoint + self.iam_security_creds_path + try: + res = self._http_client.urlopen('GET', url) + if res['status'] != 200: + return [] + except: + return [] + creds = res['data'].split('\n') + return creds + + def request_cred(self, creds_name): + url = self._endpoint + self.iam_security_creds_path + "/" + creds_name + res = self._http_client.urlopen('GET', url) + if res['status'] != 200: + raise ResponseError(res, 'GET') + + data = json.loads(res['data']) + if data['Code'] != 'Success': + raise ResponseError(res) + + return data + + def retrieve(self): + creds_list = self.request_cred_list() + if len(creds_list) == 0: + return Value() + + creds_name = creds_list[0] + role_creds = self.request_cred(creds_name) + expiration = datetime.datetime.strptime(role_creds['Expiration'], '%Y-%m-%dT%H:%M:%SZ') + self._expiry.set_expiration(expiration, self.default_expiry_window) + + return Value( + access_key=role_creds['AccessKeyId'], + secret_key=role_creds['SecretAccessKey'], + session_token=role_creds['Token'] + ) + + def is_expired(self): + return self._expiry.is_expired() diff --git a/minio/credentials/chain.py b/minio/credentials/chain.py new file mode 100644 index 00000000..c9f7e8ab --- /dev/null +++ b/minio/credentials/chain.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .credentials import Provider, Value + +class Chain(Provider): + def __init__(self, providers): + super(Provider, self).__init__() + self._providers = providers + self._current = None + + def retrieve(self): + for provider in self._providers: + creds = provider.retrieve() + if ((creds.access_key is None or creds.access_key == "") and + (creds.secret_key is None or creds.secret_key == "")): + continue + self._current = provider + return creds + self._current = None + return Value() + + def is_expired(self): + if self._current == None: + return True + return self._current.is_expired() diff --git a/minio/credentials/config.json.sample b/minio/credentials/config.json.sample new file mode 100644 index 00000000..2f654f95 --- /dev/null +++ b/minio/credentials/config.json.sample @@ -0,0 +1,17 @@ +{ + "version": "8", + "hosts": { + "play": { + "url": "https://play.minio.io:9000", + "accessKey": "Q3AM3UQ867SPQQA43P2F", + "secretKey": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", + "api": "S3v2" + }, + "s3": { + "url": "https://s3.amazonaws.com", + "accessKey": "accessKey", + "secretKey": "secret", + "api": "S3v4" + } + } +} diff --git a/minio/credentials/credentials.empty b/minio/credentials/credentials.empty new file mode 100644 index 00000000..e69de29b diff --git a/minio/credentials/credentials.py b/minio/credentials/credentials.py new file mode 100644 index 00000000..5814402a --- /dev/null +++ b/minio/credentials/credentials.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABCMeta, abstractmethod +from datetime import datetime, timedelta + +class Value(object): + def __init__(self, access_key=None, secret_key=None, session_token=None): + self.access_key = access_key + self.secret_key = secret_key + self.session_token = session_token + +class Provider(object): + + __metaclass__ = ABCMeta + + @abstractmethod + def retrieve(self): + pass + + @abstractmethod + def is_expired(self): + pass + +class Expiry(object): + def __init__(self): + self._expiration = None + + def set_expiration(self, expiration, time_delta=None): + self._expiration = expiration + if time_delta is not None: + self._expiration = self._expiration + time_delta + + def is_expired(self): + if self._expiration is None: + return True + return self._expiration < datetime.now() + +class Credentials(object): + def __init__(self, forceRefresh=True, provider=None): + self._creds = None + self._forceRefresh = forceRefresh + self._provider = provider + + def get(self): + if self.is_expired(): + try: + creds = self._provider.retrieve() + except: + raise + self._creds = creds + self._forceRefresh = False + return self._creds + + def expire(self): + self._forceRefresh = True + + def is_expired(self): + return self._forceRefresh or self._provider.is_expired() diff --git a/minio/credentials/credentials.sample b/minio/credentials/credentials.sample new file mode 100644 index 00000000..e72bd803 --- /dev/null +++ b/minio/credentials/credentials.sample @@ -0,0 +1,12 @@ +[default] +aws_access_key_id = accessKey +aws_secret_access_key = secret +aws_session_token = token + +[no_token] +aws_access_key_id = accessKey +aws_secret_access_key = secret + +[with_colon] +aws_access_key_id: accessKey +aws_secret_access_key: secret \ No newline at end of file diff --git a/minio/credentials/env_aws.py b/minio/credentials/env_aws.py new file mode 100644 index 00000000..56b58d09 --- /dev/null +++ b/minio/credentials/env_aws.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from .credentials import Provider, Value + +class EnvAWS(Provider): + def __init__(self, retrieved=False): + super(Provider, self).__init__() + self.retrieved = retrieved + + def retrieve(self): + self.retrieved = False + + access_key = os.environ.get('AWS_ACCESS_KEY_ID') + if access_key == "": + access_key = os.environ.get('AWS_ACCESS_KEY') + + secret = os.environ.get('AWS_SECRET_ACCESS_KEY') + if secret == "": + secret = os.environ.get('AWS_SECRET_KEY') + + self.retrieved = True + return Value( + access_key=access_key, + secret_key=secret, + session_token=os.environ.get('AWS_SESSION_TOKEN') + ) + + def is_expired(self): + return not self.retrieved diff --git a/minio/credentials/env_minio.py b/minio/credentials/env_minio.py new file mode 100644 index 00000000..03a5a6d9 --- /dev/null +++ b/minio/credentials/env_minio.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +from .credentials import Provider, Value + +class EnvMinio(Provider): + def __init__(self, retrieved=False): + super(Provider, self).__init__() + self._retrieved = retrieved + + def retrieve(self): + self._retrieved = False + + id = os.environ.get('MINIO_ACCESS_KEY') + + secret = os.environ.get('MINIO_SECRET_KEY') + + self._retrieved = True + + return Value( + access_key=id, + secret_key=secret + ) + + def is_expired(self): + return not self._retrieved diff --git a/minio/credentials/file_aws_credentials.py b/minio/credentials/file_aws_credentials.py new file mode 100644 index 00000000..f48c8c31 --- /dev/null +++ b/minio/credentials/file_aws_credentials.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import configparser + +from .credentials import Provider, Value + +class FileAWSCredentials(Provider): + def __init__(self, filename=None, profile=None, retrieved=False): + super(Provider, self).__init__() + self._filename = filename + self._profile = profile + self._retrieved = retrieved + + def retrieve(self): + + if self._filename == "" or self._filename is None: + self._filename = os.environ.get('AWS_SHARED_CREDENTIALS_FILE') + if self._filename == "" or self._filename is None: + home_dir = os.environ.get('HOME') + self._filename = os.path.join(home_dir, '.aws', 'credentials') + + if self._profile == "" or self._profile is None: + self._profile = os.environ.get('AWS_PROFILE') + if self._profile == "" or self._profile is None: + self._profile = 'default' + + self._retrieved = False + ini_profile = configparser.ConfigParser() + ini_profile.read(self._filename) + access_key = secret = session_token = '' + try: + access_key = ini_profile.get(self._profile, 'aws_access_key_id') + secret = ini_profile.get(self._profile, 'aws_secret_access_key') + session_token = ini_profile.get(self._profile, 'aws_session_token') + except: + pass + + if access_key == '' or secret == '': + return Value() + + self._retrieved = True + return Value( + access_key=access_key, + secret_key=secret, + session_token=session_token) + + def is_expired(self): + return not self._retrieved diff --git a/minio/credentials/file_minio_client.py b/minio/credentials/file_minio_client.py new file mode 100644 index 00000000..533ca555 --- /dev/null +++ b/minio/credentials/file_minio_client.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import configparser +import os, json +import sys + +from .credentials import Provider, Value + +class FileMinioClient(Provider): + def __init__(self, filename=None, alias=None, retrieved=False): + super(Provider, self).__init__() + self._filename = filename + self._alias = alias + self._retrieved = retrieved + + def retrieve(self): + if self._filename == "" or self._filename is None: + home_dir = os.environ.get('HOME') + self._filename = os.path.join(home_dir, '.mc', 'config.json') + if sys.platform == 'win32': + self._filename = os.path.join(home_dir, 'mc', 'config.json') + if self._alias == "" or self._alias is None: + self._alias = os.environ.get('MINIO_ALIAS') + if self._alias == "" or self._alias is None: + self._alias = "s3" + + self._retrieved = False + + config = open(self._filename, 'r') + doc = json.load(config) + creds = doc['hosts'][self._alias] + + access_key = creds['accessKey'] + secret_key = creds['secretKey'] + + self._retrieved = True + + return Value( + access_key=access_key, + secret_key=secret_key + ) + + def is_expired(self): + return not self._retrieved diff --git a/minio/credentials/static.py b/minio/credentials/static.py new file mode 100644 index 00000000..b37af3d7 --- /dev/null +++ b/minio/credentials/static.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .credentials import Provider, Value + +class Static(Provider): + def __init__(self, access_key=None, secret_key=None, token=None): + super(Static, self).__init__() + self._access_key = access_key + self._secret_key = secret_key + self._session_token = token + + def retrieve(self): + return Value( + access_key=self._access_key, + secret_key=self._secret_key, + session_token=self._session_token + ) + + def is_expired(self): + return False \ No newline at end of file diff --git a/minio/signer.py b/minio/signer.py index 510b4954..503b24c7 100644 --- a/minio/signer.py +++ b/minio/signer.py @@ -58,7 +58,7 @@ def post_presign_signature(date, region, secret_key, policy_str): return signature -def presign_v4(method, url, access_key, secret_key, session_token=None, +def presign_v4(method, url, credentials, region=None, headers=None, expires=None, response_headers=None, request_date=None): """ @@ -66,10 +66,7 @@ def presign_v4(method, url, access_key, secret_key, session_token=None, :param method: Method to be presigned examples 'PUT', 'GET'. :param url: URL to be presigned. - :param access_key: Access key id for your AWS s3 account. - :param secret_key: Secret access key for your AWS s3 account. - :param session_token: Session token key set only for temporary - access credentials. + :param credentials: Credentials object with your AWS s3 account info. :param region: region of the bucket, it is optional. :param headers: any additional HTTP request headers to be presigned, it is optional. @@ -79,7 +76,7 @@ def presign_v4(method, url, access_key, secret_key, session_token=None, """ # Validate input arguments. - if not access_key or not secret_key: + if not credentials.get().access_key or not credentials.get().secret_key: raise InvalidArgumentError('Invalid access_key and secret_key.') if region is None: @@ -104,13 +101,13 @@ def presign_v4(method, url, access_key, secret_key, session_token=None, # Construct queries. query = {} query['X-Amz-Algorithm'] = _SIGN_V4_ALGORITHM - query['X-Amz-Credential'] = generate_credential_string(access_key, + query['X-Amz-Credential'] = generate_credential_string(credentials.get().access_key, request_date, region) query['X-Amz-Date'] = iso8601Date query['X-Amz-Expires'] = str(expires) - if session_token: - query['X-Amz-Security-Token'] = session_token + if credentials.get().session_token is not None: + query['X-Amz-Security-Token'] = credentials.get().session_token signed_headers = get_signed_headers(headers_to_sign) query['X-Amz-SignedHeaders'] = ';'.join(signed_headers) @@ -149,7 +146,8 @@ def presign_v4(method, url, access_key, secret_key, session_token=None, content_hash_hex) string_to_sign = generate_string_to_sign(request_date, region, canonical_request) - signing_key = generate_signing_key(request_date, region, secret_key) + signing_key = generate_signing_key(request_date, region, + credentials.get().secret_key) signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), hashlib.sha256).hexdigest() new_parsed_url = urlsplit(new_url + "&X-Amz-Signature="+signature) @@ -170,9 +168,7 @@ def get_signed_headers(headers): def sign_v4(method, url, region, headers=None, - access_key=None, - secret_key=None, - session_token=None, + credentials=None, content_sha256=None, request_datetime=None): """ @@ -182,18 +178,13 @@ def sign_v4(method, url, region, headers=None, :param url: Final url which needs to be signed. :param region: Region should be set to bucket region. :param headers: Optional headers for the method. - :param access_key: Optional access key, if not - specified no signature is needed. - :param secret_key: Optional secret key, if not - specified no signature is needed. - :param session_token: Optional session token, set - only for temporary credentials. + :param credentials: Optional Credentials object with your AWS s3 account info. :param content_sha256: Optional body sha256. :param request_datetime: Optional request date/time """ # If no access key or secret key is provided return headers. - if not access_key or not secret_key: + if not credentials.get().access_key or not credentials.get().secret_key: return headers if headers is None: @@ -218,8 +209,8 @@ def sign_v4(method, url, region, headers=None, headers['X-Amz-Date'] = request_datetime.strftime("%Y%m%dT%H%M%SZ") headers['X-Amz-Content-Sha256'] = content_sha256 - if session_token: - headers['X-Amz-Security-Token'] = session_token + if credentials.get().session_token is not None: + headers['X-Amz-Security-Token'] = credentials.get().session_token headers_to_sign = headers @@ -232,11 +223,11 @@ def sign_v4(method, url, region, headers=None, string_to_sign = generate_string_to_sign(request_datetime, region, canonical_req) - signing_key = generate_signing_key(request_datetime, region, secret_key) + signing_key = generate_signing_key(request_datetime, region, credentials.get().secret_key) signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), hashlib.sha256).hexdigest() - authorization_header = generate_authorization_header(access_key, + authorization_header = generate_authorization_header(credentials.get().access_key, request_datetime, region, signed_headers, diff --git a/setup.py b/setup.py index eb15d6e5..2d7c44d7 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ packages = [ 'minio', 'minio.select', + 'minio.credentials' ] requires = [ @@ -46,6 +47,7 @@ 'pytz', 'certifi', 'python-dateutil', + 'configparser', ] tests_requires = [ diff --git a/tests/unit/chain_provider_test.py b/tests/unit/chain_provider_test.py new file mode 100644 index 00000000..4354869c --- /dev/null +++ b/tests/unit/chain_provider_test.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest import TestCase + +from minio.credentials.chain import Chain +from minio.credentials.credentials import Value +from minio.credentials.env_aws import EnvAWS +from minio.credentials.env_minio import EnvMinio +from nose.tools import eq_ + +class ChainProviderTest(TestCase): + + def test_chain_retrieve(self): + # clear environment + os.environ.clear() + # prepare env for env_aws provider + os.environ["AWS_ACCESS_KEY_ID"] = "access_aws" + os.environ["AWS_SECRET_ACCESS_KEY"] = "secret_aws" + os.environ["AWS_SESSION_TOKEN"] = "token_aws" + # prepare env for env_minio + os.environ['MINIO_ACCESS_KEY'] = "access_minio" + os.environ["MINIO_SECRET_KEY"] = "secret_minio" + # create chain provider with env_aws and env_minio providers + chain = Chain( + providers=[ + EnvAWS(), + EnvMinio() + ] + ) + # retireve provider (env_aws) has priority + creds = chain.retrieve() + # assert provider credentials + eq_(creds.access_key, "access_aws") + eq_(creds.secret_key, "secret_aws") + eq_(creds.session_token, "token_aws") + # assert is_expired + eq_(chain.is_expired(), False) + + + def test_chain_is_expired(self): + # clear environment + os.environ.clear() + # prepare env for env_aws + os.environ["AWS_ACCESS_KEY_ID"] = "access_aws" + os.environ["AWS_SECRET_ACCESS_KEY"] = "secret_aws" + # create chain provider + chain = Chain( + providers=[EnvAWS()] + ) + # is_expired should be True before retrieve() + eq_(chain.is_expired(), True) + # retieve single env_aws provider + chain.retrieve() + # is_expired should be False after retrieve() + eq_(chain.is_expired(), False) + + def test_chain_with_no_provider(self): + # clear environment + os.environ.clear() + # create empty chain provider + chain = Chain( + providers=[] + ) + # is_expired should be True before retrieve() + eq_(chain.is_expired(), True) diff --git a/tests/unit/credentials_test.py b/tests/unit/credentials_test.py new file mode 100644 index 00000000..50af23b6 --- /dev/null +++ b/tests/unit/credentials_test.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from minio.credentials.credentials import Credentials, Value +from minio.credentials.file_minio_client import FileMinioClient +from unittest import TestCase +from nose.tools import eq_ + +class CredentialsTest(TestCase): + def test_credentials_get(self): + # get credentials + credentials = Credentials( + provider=FileMinioClient('minio/credentials/config.json.sample','play') + ) + # is_expired should be True before get + eq_(credentials.is_expired(), True) + creds = credentials.get() + expected_creds = Value( + access_key='Q3AM3UQ867SPQQA43P2F', + secret_key='zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', + session_token=None + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # is_expired should be False after get + eq_(credentials.is_expired(), False) + + def test_credentials_with_error(self): + pass diff --git a/tests/unit/env_aws_provider_test.py b/tests/unit/env_aws_provider_test.py new file mode 100644 index 00000000..da7e4c38 --- /dev/null +++ b/tests/unit/env_aws_provider_test.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest import TestCase + +from minio.credentials.env_aws import EnvAWS +from minio.credentials.credentials import Value +from nose.tools import eq_ + +class EnvAWSTest(TestCase): + + def test_env_aws_retrieve(self): + # clear environement + os.environ.clear() + # set environment variables + os.environ["AWS_ACCESS_KEY_ID"] = "access" + os.environ["AWS_SECRET_ACCESS_KEY"] = "secret" + os.environ["AWS_SESSION_TOKEN"] = "token" + # init new env_aws provider + provider = EnvAWS() + # assert expired true for newly created provider + eq_(provider.is_expired(), True) + # retrieve provider credentials + creds = provider.retrieve() + # assert expected data + expected_creds = Value( + access_key="access", + secret_key="secret", + session_token="token" + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # assert expired true for retrieved credentials + eq_(provider.is_expired(), False) + + def test_env_aws_retrieve_no_token(self): + # clear environement + os.environ.clear() + # set environment variables + os.environ["AWS_ACCESS_KEY_ID"] = "access" + os.environ["AWS_SECRET_ACCESS_KEY"] = "secret" + # Init new env_aws provider + provider = EnvAWS() + # assert expired true for newly created provider + eq_(provider.is_expired(), True) + # retrieve rpovider credentials + creds = provider.retrieve() + # assert expected data + expected_creds = Value( + access_key="access", + secret_key="secret", + session_token=None + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # assert expired true for retrieved credentials + eq_(provider.is_expired(), False) diff --git a/tests/unit/env_minio_provider_test.py b/tests/unit/env_minio_provider_test.py new file mode 100644 index 00000000..27ceed0a --- /dev/null +++ b/tests/unit/env_minio_provider_test.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest import TestCase + +from minio.credentials.env_minio import EnvMinio +from minio.credentials.credentials import Value +from nose.tools import eq_ + +class EnvMinioTest(TestCase): + + def test_env_minio_retrieve(self): + # clear environment + os.environ.clear() + # set environment variables + os.environ['MINIO_ACCESS_KEY'] = "access" + os.environ["MINIO_SECRET_KEY"] = "secret" + # init new env_minio provider + provider = EnvMinio() + # assert expected true for newly created provider + eq_(provider.is_expired(), True) + # retrieve provider credentials + creds = provider.retrieve() + # assert expected data + expected_creds = Value( + access_key="access", + secret_key="secret", + session_token=None + ) + # assert expired true for retrieved credentials + eq_(provider.is_expired(), False) diff --git a/tests/unit/file_aws_credentials_test.py b/tests/unit/file_aws_credentials_test.py new file mode 100644 index 00000000..aa72e6db --- /dev/null +++ b/tests/unit/file_aws_credentials_test.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest import TestCase + +from minio.credentials.file_aws_credentials import FileAWSCredentials +from minio.credentials.credentials import Value +from nose.tools import eq_ + +class FileAWSCredentialsTest(TestCase): + + def test_file_aws(self): + # clear environment + os.environ.clear() + # get provider + provider = FileAWSCredentials('minio/credentials/credentials.sample') + # is_expired should be True before retrieve + eq_(provider.is_expired(), True) + # retrieve credentials + creds = provider.retrieve() + expected_creds = Value( + access_key='accessKey', + secret_key='secret', + session_token='token' + ) + # assert credentials + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # is_expired should be False before retrieve + eq_(provider.is_expired(), False) + + def test_file_aws_from_env(self): + # clear environment + os.environ.clear() + # set env with aws config file + os.environ['AWS_SHARED_CREDENTIALS_FILE'] = 'minio/credentials/credentials.sample' + # get provider + provider = FileAWSCredentials() + # is_expired should be True before retrieve + eq_(provider.is_expired(), True) + # retieve credentials + creds = provider.retrieve() + expected_creds = Value( + access_key='accessKey', + secret_key='secret', + session_token='token' + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # is_expired should be False after retrieve + eq_(provider.is_expired(), False) + + def test_file_aws_env_profile(self): + # clear environment + os.environ.clear() + # set profile env + os.environ['AWS_PROFILE'] = 'no_token' + # get provider + provider = FileAWSCredentials('minio/credentials/credentials.sample') + # is_expired should be True before retrieve + eq_(provider.is_expired(), True) + # retrieve credentials + creds = provider.retrieve() + expected_creds = Value( + access_key='accessKey', + secret_key='secret', + session_token='' + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # is_expired should be False after retrieve + eq_(provider.is_expired(), False) + + def test_file_aws_arg_profile(self): + # clear environment + os.environ.clear() + # get provider + provider = FileAWSCredentials('minio/credentials/credentials.sample', 'no_token') + # is_expired should be True before retrieve + eq_(provider.is_expired(), True) + # retrieve credentials + creds = provider.retrieve() + expected_creds = Value( + access_key='accessKey', + secret_key='secret', + session_token='' + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # is_expired should be False after retieve + eq_(provider.is_expired(), False) + + def test_file_aws_no_creds(self): + # clear environment + os.environ.clear() + provider = FileAWSCredentials('minio/credentials/credentials.empty', 'no_token') + creds = provider.retrieve() + eq_(creds.access_key, None) + eq_(creds.secret_key, None) + eq_(creds.session_token, None) diff --git a/tests/unit/file_minio_client_test.py b/tests/unit/file_minio_client_test.py new file mode 100644 index 00000000..1c0b9ed7 --- /dev/null +++ b/tests/unit/file_minio_client_test.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from unittest import TestCase +from minio.credentials.file_minio_client import FileMinioClient +from minio.credentials.credentials import Value + +from nose.tools import eq_ + +class FileMinioClientTest(TestCase): + + def test_file_minio_(self): + # clear environment + os.environ.clear() + # get provider + provider = FileMinioClient('minio/credentials/config.json.sample') + # is_expired should be True before retrieve + eq_(provider.is_expired(), True) + # retrieve credentials + creds = provider.retrieve() + expected_creds = Value( + access_key='accessKey', + secret_key='secret', + session_token=None + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # is_expired should be False after retrieve + eq_(provider.is_expired(), False) + + def test_file_minio_env_alias(self): + # clear environment + os.environ.clear() + # set env with minio config file + os.environ['MINIO_ALIAS'] = 'play' + # get provider + provider = FileMinioClient('minio/credentials/config.json.sample') + # is_expired should be True before retrieve + eq_(provider.is_expired(), True) + # retrieve credentials + creds = provider.retrieve() + expected_creds = Value( + access_key='Q3AM3UQ867SPQQA43P2F', + secret_key='zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', + session_token=None + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # is_expired is False after retrieve + eq_(provider.is_expired(), False) + + def test_file_minio_arg_alias(self): + # clear environment + os.environ.clear() + # get provider + provider = FileMinioClient('minio/credentials/config.json.sample','play') + # is_expired is True before retrieve + eq_(provider.is_expired(), True) + # get credentials + creds = provider.retrieve() + expected_creds = Value( + access_key='Q3AM3UQ867SPQQA43P2F', + secret_key='zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', + session_token=None + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # is_expired should be False after retrieve + eq_(provider.is_expired(), False) diff --git a/tests/unit/iam_aws_test.py b/tests/unit/iam_aws_test.py new file mode 100644 index 00000000..093766f4 --- /dev/null +++ b/tests/unit/iam_aws_test.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import json +from unittest import TestCase +from minio.credentials.aws_iam import IamEc2MetaData +from nose.tools import eq_ + +class TestIamEc2MetaData(TestCase): + @mock.patch('urllib3.PoolManager.urlopen') + def test_iam(self, mock_connection): + # get provider + mock_cred_list = { + "status" : 200, + "data" : "test-s3-full-access-for-minio-ec2" + } + request_cred_data = json.dumps({ + "Code" : 'Success', + "Type" : 'AWS-HMAC', + "AccessKeyId" : 'accessKey', + "SecretAccessKey" : 'secret', + "Token" : 'token', + "Expiration" : '2014-12-16T01:51:37Z', + "LastUpdated" : '2009-11-23T0:00:00Z' + }) + mock_request_cred = { + "status" : 200, + "data": request_cred_data + } + mock_connection.side_effect = [mock_cred_list, mock_request_cred] + provider = IamEc2MetaData() + # retrieve credentials + creds = provider.retrieve() + eq_(creds.access_key, "accessKey") + eq_(creds.secret_key, "secret") + eq_(creds.session_token, "token") diff --git a/tests/unit/minio_mocks.py b/tests/unit/minio_mocks.py index 8f1643d6..10a9a18d 100644 --- a/tests/unit/minio_mocks.py +++ b/tests/unit/minio_mocks.py @@ -57,6 +57,10 @@ def stream(self, chunk_size=1, decode_unicode=False): # dummy release connection call. def release_conn(self): return + + def __getitem__(self, key): + if key == "status": + return self.status class MockConnection(object): def __init__(self): @@ -67,11 +71,13 @@ def mock_add_request(self, request): # noinspection PyUnusedLocal def request(self, method, url, headers, redirect=False): - return_request = self.requests.pop(0) + # only pop off matching requests + return_request = self.requests[0] return_request.mock_verify(method, url, headers) - return return_request + return self.requests.pop(0) + # noinspection PyRedeclaration,PyUnusedLocal,PyUnusedLocal - def urlopen(self, method, url, headers, preload_content=False, body=None, + def urlopen(self, method, url, headers={}, preload_content=False, body=None, redirect=False): return self.request(method, url, headers) diff --git a/tests/unit/sign_test.py b/tests/unit/sign_test.py index 8595e74a..8c3ca649 100644 --- a/tests/unit/sign_test.py +++ b/tests/unit/sign_test.py @@ -26,6 +26,7 @@ from minio.error import InvalidArgumentError from minio.compat import urlsplit, urlencode, queryencode from minio.fold_case_dict import FoldCaseDict +from minio.credentials import Credentials, Static from minio.helpers import get_target_url empty_hash = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' @@ -111,18 +112,30 @@ def test_generate_authentication_header(self): class PresignURLTest(TestCase): @raises(InvalidArgumentError) def test_presigned_no_access_key(self): - presign_v4('GET', 'http://localhost:9000/hello', None, None) + credentials = Credentials( + provider=Static() + ) + presign_v4('GET', 'http://localhost:9000/hello', credentials, None) @raises(InvalidArgumentError) def test_presigned_invalid_expires(self): - presign_v4('GET', 'http://localhost:9000/hello', None, None, region=None, headers={}, expires=0) + credentials = Credentials( + provider=Static() + ) + presign_v4('GET', 'http://localhost:9000/hello', credentials, region=None, headers={}, expires=0) class SignV4Test(TestCase): def test_signv4(self): # Construct target url. + credentials = Credentials( + provider=Static( + access_key='minio', + secret_key='minio123' + ) + ) url = get_target_url('http://localhost:9000', bucket_name='testbucket', object_name='~testobject', bucket_region='us-east-1', query={'partID': '1', 'uploadID': '~abcd'}) - hdrs = sign_v4('PUT', url, 'us-east-1', access_key='minio', secret_key='minio123', request_datetime=dt) + hdrs = sign_v4('PUT', url, 'us-east-1', credentials=credentials, request_datetime=dt) eq_(hdrs['Authorization'], 'AWS4-HMAC-SHA256 Credential=minio/20150620/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a2f4546f647981732bd90dfa5a7599c44dca92f44bea48ecc7565df06032c25b') class UnicodeEncodeTest(TestCase): diff --git a/tests/unit/static_test.py b/tests/unit/static_test.py new file mode 100644 index 00000000..0d6fda6f --- /dev/null +++ b/tests/unit/static_test.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from minio.credentials.static import Static +from minio.credentials.credentials import Value +from unittest import TestCase +from nose.tools import eq_ + +class StaticTest(TestCase): + def test_static_credentials(self): + # get provider + provider = Static( + access_key='UXHW', + secret_key='SECRET' + ) + # static is_expired is always False + eq_(provider.is_expired(), False) + # retrieve creds + creds = provider.retrieve() + expected_creds = Value( + access_key='UXHW', + secret_key='SECRET' + ) + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + # static is_expired is always False + eq_(provider.is_expired(), False) + + def test_empty_static_credentials(self): + # get provider + provider = Static() + # static is_expired is always False + eq_(provider.is_expired(), False) + # retrieve credentials + creds = provider.retrieve() + expected_creds = Value() + eq_(creds.access_key, expected_creds.access_key) + eq_(creds.secret_key, expected_creds.secret_key) + eq_(creds.session_token, expected_creds.session_token) + # static is_expired is always False + eq_(provider.is_expired(), False)