小知识，大挑战！本文正在参与「程序员必备小知识」创作活动
本文同时参与 「掘力星计划」 ，赢取创作大礼包，挑战创作激励金
minio_s3 官方文档：docs.minio.org.cn/docs/master…
- 安装相关配置包
pip install minio
- 将基础配置添加到yaml文件(filename:minio.yaml)
base:
POINT_URL: xxx.xxx.xxx.xxx
ACCESS_KEY: 你自己的key
SECRET_KEY: 你自己的SECRET_KEY
REGION: 你自己的REGION
SCHEME: http
- 获取配置代码（只给出部分主要代码）
from utils.security.aes import aes_decrypt
_stream = open('minio.yaml', 'r')
class BaseMinioConfig(DBConfig):
point_url = _data['base']['POINT_URL']
access_key = aes_decrypt(_data['base']['ACCESS_KEY'])
secret_key = aes_decrypt(_data['base']['SECRET_KEY'])
region = _data['base']['REGION']
scheme = _data['base']['SCHEME']
minio_conf = {
EnvEnum.base.value: MiniobaseConfig,
}
- 配置settings文件
from config.minio import minio_conf
# MinioS3配置
MINIO_CONF = minio_conf['base']
PUBLIC_BUCKET_NAME = f"demo_base_public_bucket".replace('_', '-')
BUCKET_NAME = f"demo_base_bucket"
MINIO_ACCESS_KEY_ID = MINIO_CONF.access_key
MINIO_SECRET_ACCESS_KEY = MINIO_CONF.secret_key
MINIO_ENDPOINT_URL = MINIO_CONF.point_url
MINIO_REGION = MINIO_CONF.region
MINIO_SCHEME = MINIO_CONF.scheme
- minio_s3调用相关代码
base.py
import uuid
def make_s3_key_with_part(prefix: str, suffix: str, dir: str = '/') -> str:
"""
构建s3 key
:param prefix:
:param suffix:
:param dir:
:return:
"""
if not isinstance(dir, str):
raise ValueError('Please enter a valid dir')
if dir and not dir.endswith('/'):
dir = f'{dir}/'
return '{}{}-{}.{}'.format(dir, prefix, uuid.uuid4().hex, suffix)
def make_s3_key(file_name: str, dir: str = '/') -> str:
"""
构建s3 key(完整)
:param file_name:
:param dir:
:return:
"""
from utils.basic.file import split_file_path
prefix, suffix = split_file_path(file_name)
if not all([prefix, suffix]):
raise ValueError('Please enter a valid file name')
key = make_s3_key_with_part(prefix, suffix, dir)
return key
def is_false(value):
return value == False
def is_false_f(value):
return value[0] == False
def is_none(value):
return not value
class BaseOssclient(object):
def __init__(self,
bucket_name='',
access_key_id='',
secret_access_key='',
endpoint_url='',
region=''
):
self.bucket_name = bucket_name.replace("_", "-")
self.access_key_id = access_key_id
self.secret_access_key = secret_access_key
self.endpoint_url = endpoint_url
self.region = region
@property
def client(self):
raise NotImplementedError()
# 将文件传到s3的指定bucket中
def upload_file(self, file_path, upload_path, add_suffix=True, **kwargs):
"""
:param file_path: 对应s3存储的位置
:param upload_path: 文件的上传位置
:return:
"""
raise NotImplementedError()
def upload_file_obj(self, file, upload_path, **kwargs):
"""
:param file_path:
:param upload_path: 文件的上传位置
:return:
"""
raise NotImplementedError()
def get_signed_url(self, key):
"""
签名
:param key:
:return:
"""
raise NotImplementedError()
def download_file(self, download_path, key):
"""
下载文件到本地
:param download_path:
:param key:
:return:
"""
raise NotImplementedError()
def download_file_obj(self, key):
"""
下载文件到buffer
:param key:
:return:
"""
raise NotImplementedError()
def delete_file(self, key):
"""
删除文件
:param key:
:return:
"""
raise NotImplementedError()
def check_key_is_exist(self, key):
"""
判断文件是否存在
:param key:
:return:
"""
raise NotImplementedError()
def get_pages(self, delimiter='/', prefix=None):
"""
分页
:param delimiter:
:param prefix:
:return:
"""
raise NotImplementedError()
def get_object_list(self, delimiter='/', prefix=None):
"""
获取所有文件
:param delimiter:
:param prefix:
:return:
"""
raise NotImplementedError()
def download_dir(self, delimiter='/', prefix=None, local='/tmp'):
"""
下载整个文件夹
:param delimiter:
:param prefix:
:param local:
:return:
"""
raise NotImplementedError()
def creat_bucket(self):
"""
创建桶
:return:
"""
raise NotImplementedError()
minio_s3.py
import os
from datetime import timedelta
from django.conf import settings
from minio import Minio
from tenacity import stop_after_delay, stop_after_attempt, wait_random, retry_if_result, retry_if_exception_type, retry
from base import make_s3_key ,BaseOssclient, is_false, is_false_f
# 获取S3的url
def get_private_url(key: str) -> str:
"""
:param key: minios3存储的name
:return: url
"""
client = MinioS3()
return client.get_signed_url(key=key)
def get_public_url(key: str) -> str:
url = f"{settings.MINIO_SCHEME}://{settings.MINIO_ENDPOINT_URL}/" \
f"{settings.PUBLIC_BUCKET_NAME.replace('_', '-')}/{key}"
return url
class MinioS3(BaseOssclient):
def __init__(self, bucket_name='',
access_key_id='',
secret_access_key='',
endpoint_url='',
region='',
is_secure: bool = None
):
self.bucket_name = bucket_name.replace("_", "-") or settings.BUCKET_NAME.replace("_", "-")
self.access_key_id = access_key_id or settings.MINIO_ACCESS_KEY_ID
self.secret_access_key = secret_access_key or settings.MINIO_SECRET_ACCESS_KEY
self.endpoint_url = endpoint_url or settings.MINIO_ENDPOINT_URL
self.region = region or settings.MINIO_REGION
if is_secure is None:
self.is_secure = False if settings.MINIO_SCHEME == "http" else True
else:
self.is_secure = is_secure
super().__init__(bucket_name=self.bucket_name,
access_key_id=self.access_key_id,
secret_access_key=self.secret_access_key,
endpoint_url=self.endpoint_url,
region=self.region)
@property
def client(self):
client = Minio(
self.endpoint_url,
access_key=self.access_key_id,
secret_key=self.secret_access_key,
secure=self.is_secure,
region=self.region
)
return client
def creat_bucket(self):
"""
创建桶
:return:
"""
if not self.client.bucket_exists(self.bucket_name):
self.client.make_bucket(self.bucket_name)
return True
return False
# 将文件传到s3的指定bucket中
@retry(reraise=True,
stop=(stop_after_delay(10) | stop_after_attempt(5)),
wait=wait_random(min=0.01, max=0.03),
retry=(retry_if_result(is_false) | retry_if_exception_type(Exception)))
def upload_file(self, file_path, upload_path, add_suffix=True, **kwargs):
"""
:param file_path: 本地
:param upload_path: 远端
:return:
"""
if os.path.exists(file_path):
self.client.fput_object(bucket_name=self.bucket_name,
object_name=make_s3_key(upload_path) if add_suffix else upload_path,
file_path=file_path,
num_parallel_uploads=5
)
return True
else:
return False
@retry(reraise=True, stop=(stop_after_delay(10) | stop_after_attempt(5)), wait=wait_random(min=0.01, max=0.03),
retry=(retry_if_result(is_false) | retry_if_exception_type(Exception)))
def upload_file_obj(self, file, upload_path, **kwargs):
"""
:param file:
:param upload_path: 文件的上传位置
:return:
"""
self.client.put_object(bucket_name=self.bucket_name,
object_name=upload_path,
data=file,
length=file.size,
content_type=file.content_type,
num_parallel_uploads=5)
return True
@retry(reraise=True,
stop=(stop_after_delay(10) | stop_after_attempt(5)),
wait=wait_random(min=0.01, max=0.03),
retry=(retry_if_result(is_false_f) | retry_if_exception_type(Exception)))
def download_file(self, download_path, key):
self.client.fget_object(self.bucket_name, key, download_path)
return True, download_path
@retry(reraise=True, stop=(stop_after_delay(10) | stop_after_attempt(5)), wait=wait_random(min=0.01, max=0.03),
retry=(retry_if_result(is_false_f) | retry_if_exception_type(Exception)))
def download_file_obj(self, key):
data = self.client.get_object(self.bucket_name, key)
from utils.basic.file import content_to_file
buf = content_to_file(data.read())
return True, buf
@retry(reraise=True, stop=(stop_after_delay(10) | stop_after_attempt(5)), wait=wait_random(min=0.01, max=0.03),
retry=(retry_if_result(is_false) | retry_if_exception_type(Exception)))
def delete_file(self, key):
"""
删除文件
:param key:
:return:
"""
self.client.remove_object(self.bucket_name, key)
return True
@retry(reraise=True, stop=(stop_after_delay(10) | stop_after_attempt(5)), wait=wait_random(min=0.01, max=0.03),
retry=(retry_if_result(is_false) | retry_if_exception_type(Exception)))
def get_signed_url(self, key):
if not key:
return ''
url = self.client.presigned_get_object(self.bucket_name, key, expires=timedelta(hours=24))
return url
def public_policy(self, bucket_name):
policy = '{"Version":"","Statement":[{"Effect":"Allow","Principal":' \
'{"AWS": ["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":' \
'["arn:aws:s3:::%s"]},{"Effect":"Allow","Principal":{"AWS":["*"]},' \
'"Action": ["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}' % (
bucket_name, bucket_name)
return policy
