S3 - mwicat/personal GitHub Wiki

Create client

import boto3

session = boto3.session.Session()

s3_client = session.client(
    service_name='s3',
    aws_access_key_id=access_key_id,
    aws_secret_access_key=secret_access_key,
    endpoint_url=S3_ENDPOINT_URL,
)

List objects

resp = s3_client.list_objects(Bucket=S3_BUCKET_NAME)

headers = ['name', 'size [MB]']
rows = []

for data in resp['Contents']:
    row = [data['Key'], data['Size'] / 1024 / 1024]
    rows.append(row)

click.echo(tabulate(rows, headers=headers))

Download file

class ProgressPercentage(object):

    def __init__(self, client, bucket, filename):
        self._filename = filename
        self._size = client.head_object(
            Bucket=bucket, Key=filename)['ContentLength']
        self._seen_so_far = 0
        self._lock = threading.Lock()

    def __call__(self, bytes_amount):
        # To simplify, assume this is hooked up to a single filename
        with self._lock:
            self._seen_so_far += bytes_amount
            percentage = (self._seen_so_far / self._size) * 100
            sys.stdout.write(
                "\r%s  %dMB / %dMB  (%.2f%%)" % (
                    self._filename,
                    self._seen_so_far / 1024 / 1024,
                    self._size / 1024 / 1024,
                    percentage))
            sys.stdout.flush()


s3_client.download_file(
    S3_BUCKET_NAME,
    src_file,
    dest_file,
    Callback=ProgressPercentage(s3_client, S3_BUCKET_NAME, src_file))