遍历AWS S3 存储桶中的Key

# -*- coding: utf-8 -*-

import boto3

# 这里输入你存储桶的名字
BUCKET = 'bucket'

s3_client = boto3.client('s3')


# 这个方法可以过滤出 某个存储桶下面的所有key
def get_all_s3_keys(bucket):
    """Get a list of all keys in an S3 bucket."""
    keys = []
    kwargs = {'Bucket': bucket}
    while True:
        resp = s3_client.list_objects_v2(**kwargs)
        for obj in resp['Contents']:
            keys.append(obj['Key'])

        try:
            kwargs['ContinuationToken'] = resp['NextContinuationToken']
        except KeyError:
            break

    return keys


#  这里是一个迭代器.而且可以过滤出你想要的对应的前缀和后缀的key 
def get_matching_s3_keys(bucket, prefix='', suffix=''):
    """
    Generate the keys in an S3 bucket.

    :param bucket: Name of the S3 bucket.
    :param prefix: Only fetch keys that start with this prefix (optional).
    :param suffix: Only fetch keys that end with this suffix (optional).
    """
    kwargs = {'Bucket': bucket, 'Prefix': prefix}
    while True:
        resp = s3_client.list_objects_v2(**kwargs)
        for obj in resp['Contents']:
            key = obj['Key']
            if key.endswith(suffix):
                yield key

        try:
            kwargs['ContinuationToken'] = resp['NextContinuationToken']
        except KeyError:
            break