mirror of
https://github.com/minio/minio.git
synced 2024-12-26 23:25:54 -05:00
c829e3a13b
With this change, MinIO's ILM supports transitioning objects to a remote tier. This change includes support for Azure Blob Storage, AWS S3 compatible object storage incl. MinIO and Google Cloud Storage as remote tier storage backends. Some new additions include: - Admin APIs remote tier configuration management - Simple journal to track remote objects to be 'collected' This is used by object API handlers which 'mutate' object versions by overwriting/replacing content (Put/CopyObject) or removing the version itself (e.g DeleteObjectVersion). - Rework of previous ILM transition to fit the new model In the new model, a storage class (a.k.a remote tier) is defined by the 'remote' object storage type (one of s3, azure, GCS), bucket name and a prefix. * Fixed bugs, review comments, and more unit-tests - Leverage inline small object feature - Migrate legacy objects to the latest object format before transitioning - Fix restore to particular version if specified - Extend SharedDataDirCount to handle transitioned and restored objects - Restore-object should accept version-id for version-suspended bucket (#12091) - Check if remote tier creds have sufficient permissions - Bonus minor fixes to existing error messages Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io> Co-authored-by: Krishna Srinivas <krishna@minio.io> Signed-off-by: Harshavardhana <harsha@minio.io>
133 lines
4.6 KiB
Python
133 lines
4.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
import json
|
|
# standard.
|
|
import os
|
|
|
|
import certifi
|
|
# Dependencies
|
|
import urllib3
|
|
from botocore.credentials import CredentialProvider, RefreshableCredentials
|
|
from botocore.exceptions import CredentialRetrievalError
|
|
from dateutil.parser import parse
|
|
|
|
from .sts_element import STSElement
|
|
|
|
|
|
class ClientGrantsCredentialProvider(CredentialProvider):
|
|
"""
|
|
ClientGrantsCredentialProvider implements CredentialProvider compatible
|
|
implementation to be used with boto_session
|
|
"""
|
|
METHOD = 'assume-role-client-grants'
|
|
CANONICAL_NAME = 'AssumeRoleClientGrants'
|
|
|
|
def __init__(self, cid, csec,
|
|
idp_ep='http://localhost:8080/auth/realms/minio/protocol/openid-connect/token',
|
|
sts_ep='http://localhost:9000'):
|
|
self.cid = cid
|
|
self.csec = csec
|
|
self.idp_ep = idp_ep
|
|
self.sts_ep = sts_ep
|
|
|
|
# Load CA certificates from SSL_CERT_FILE file if set
|
|
ca_certs = os.environ.get('SSL_CERT_FILE')
|
|
if not ca_certs:
|
|
ca_certs = certifi.where()
|
|
|
|
self._http = urllib3.PoolManager(
|
|
timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
|
|
maxsize=10,
|
|
cert_reqs='CERT_NONE',
|
|
ca_certs=ca_certs,
|
|
retries=urllib3.Retry(
|
|
total=5,
|
|
backoff_factor=0.2,
|
|
status_forcelist=[500, 502, 503, 504]
|
|
)
|
|
)
|
|
|
|
def load(self):
|
|
"""
|
|
Search for credentials with client_grants
|
|
"""
|
|
if self.cid is not None:
|
|
fetcher = self._create_credentials_fetcher()
|
|
return RefreshableCredentials.create_from_metadata(
|
|
metadata=fetcher(),
|
|
refresh_using=fetcher,
|
|
method=self.METHOD,
|
|
)
|
|
else:
|
|
return None
|
|
|
|
def _create_credentials_fetcher(self):
|
|
method = self.METHOD
|
|
|
|
def fetch_credentials():
|
|
# HTTP headers are case insensitive filter out
|
|
# all duplicate headers and pick one.
|
|
headers = {}
|
|
headers['content-type'] = 'application/x-www-form-urlencoded'
|
|
headers['authorization'] = urllib3.make_headers(
|
|
basic_auth='%s:%s' % (self.cid, self.csec))['authorization']
|
|
|
|
response = self._http.urlopen('POST', self.idp_ep,
|
|
body="grant_type=client_credentials",
|
|
headers=headers,
|
|
preload_content=True,
|
|
)
|
|
if response.status != 200:
|
|
message = "Credential refresh failed, response: %s"
|
|
raise CredentialRetrievalError(
|
|
provider=method,
|
|
error_msg=message % response.status,
|
|
)
|
|
|
|
creds = json.loads(response.data)
|
|
|
|
query = {}
|
|
query['Action'] = 'AssumeRoleWithClientGrants'
|
|
query['Token'] = creds['access_token']
|
|
query['DurationSeconds'] = creds['expires_in']
|
|
query['Version'] = '2011-06-15'
|
|
|
|
query_components = []
|
|
for key in query:
|
|
if query[key] is not None:
|
|
query_components.append("%s=%s" % (key, query[key]))
|
|
|
|
query_string = '&'.join(query_components)
|
|
sts_ep_url = self.sts_ep
|
|
if query_string:
|
|
sts_ep_url = self.sts_ep + '?' + query_string
|
|
|
|
response = self._http.urlopen(
|
|
'POST', sts_ep_url, preload_content=True)
|
|
if response.status != 200:
|
|
message = "Credential refresh failed, response: %s"
|
|
raise CredentialRetrievalError(
|
|
provider=method,
|
|
error_msg=message % response.status,
|
|
)
|
|
|
|
return parse_grants_response(response.data)
|
|
|
|
def parse_grants_response(data):
|
|
"""
|
|
Parser for AssumeRoleWithClientGrants response
|
|
|
|
:param data: Response data for AssumeRoleWithClientGrants request
|
|
:return: dict
|
|
"""
|
|
root = STSElement.fromstring(
|
|
'AssumeRoleWithClientGrantsResponse', data)
|
|
result = root.find('AssumeRoleWithClientGrantsResult')
|
|
creds = result.find('Credentials')
|
|
return dict(
|
|
access_key=creds.get_child_text('AccessKeyId'),
|
|
secret_key=creds.get_child_text('SecretAccessKey'),
|
|
token=creds.get_child_text('SessionToken'),
|
|
expiry_time=parse(creds.get_child_text('Expiration')).isoformat())
|
|
|
|
return fetch_credentials
|