milvus 环境搭建
1.下载 milvus-standalone-docker-compose.yml 配置文件并保存为 docker-compose.yml
wget https://github.com/milvus-io/milvus/releases/download/v2.0.2/milvus-standalone-docker-compose.yml -O docker-compose.yml
or
wget https://assets.artlib.cn/assets/soft/milvus-standalone-docker-compose.yml -O docker-compose.yml
2.安装docker
curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
or
curl -sSL https://get.daocloud.io/docker | sh
3.安装docker-compose
wget https://assets.artlib.cn/assets/soft/docker-composev2.4.1 -O /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
systemctl start docker
4.启动
sudo docker-compose up -d
Attu 搭建
docker run -p 8000:3000 -e HOST_URL=http://{ your machine IP }:8000 -e MILVUS_URL={your machine IP}:19530 zilliz/attu:latest
应用
1.安装anaconda
wget https://repo.anaconda.com/archive/Anaconda3-2022.05-Linux-x86_64.sh
or
wget https://assets.artlib.cn/assets/soft/Anaconda3-2022.05-Linux-x86_64.sh
加入环境变量
vim /etc/profile
export PATH=$PATH:/root/anaconda3/bin
source /etc/profile
# 创建环境
conda create --name py3 python=3.6
# 进入环境
conda activate py3
# source activate py3
# 退出环境
conda deactivate
2.安装python包
cat requirements.txt
absl-py==1.1.0
aiohttp==3.8.1
aiosignal==1.2.0
antlr4-python3-runtime==4.9.3
anyio==3.6.1
argcomplete==1.12.3
asgiref==3.4.1
async-timeout==4.0.2
asynctest==0.13.0
attrs==21.4.0
backports.weakref==1.0rc1
black==22.3.0
bleach==1.5.0
boto3==1.23.10
botocore==1.26.10
cached-property==1.5.2
cachetools==4.2.4
certifi==2021.5.30
charset-normalizer==2.0.12
click==8.0.4
colorlog==6.6.0
contextlib2==21.6.0
contextvars==2.4
cycler==0.11.0
dataclasses==0.8
distlib==0.3.4
fastapi==0.78.0
filelock==3.4.1
flake8==4.0.1
flake8-copyright==0.2.2
Flask==2.0.3
flatbuffers==2.0
frozenlist==1.2.0
fsspec==2022.1.0
future==0.18.2
google-auth==2.7.0
google-auth-oauthlib==0.4.6
grpcio==1.37.1
grpcio-tools==1.37.1
h11==0.13.0
h5py==3.1.0
html5lib==0.9999999
httptools==0.3.0
hydra-core==1.2.0
idna==3.3
idna-ssl==1.1.0
immutables==0.18
importlib-metadata==4.2.0
importlib-resources==5.4.0
isort==5.10.1
itsdangerous==2.0.1
Jinja2==3.0.3
jmespath==0.10.0
kiwisolver==1.3.1
Markdown==3.3.4
MarkupSafe==2.0.1
matplotlib==3.3.4
mccabe==0.6.1
mmh3==3.0.0
multidict==5.2.0
mypy-extensions==0.4.3
nox==2022.1.7
numpy==1.19.5
oauthlib==3.2.0
omegaconf==2.2.2
onnx==1.11.0
onnxruntime==1.10.0
opencv-python==4.6.0.66
packaging==21.3
pandas==1.1.5
pathspec==0.9.0
Pillow==8.4.0
platformdirs==2.4.0
protobuf==3.19.4
py==1.11.0
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycodestyle==2.8.0
pydantic==1.9.1
pyDeprecate==0.3.1
pyflakes==2.4.0
pymilvus==2.0.2
pymilvus-orm==2.0.0rc1
pymilvusdm==2.0
PyMySQL==1.0.2
pyparsing==3.0.9
python-dateutil==2.8.2
python-dotenv==0.20.0
python-multipart==0.0.5
pytorch-lightning==1.5.10
pytouch==0.4.0
pytz==2022.1
PyYAML==6.0
requests==2.27.1
requests-oauthlib==1.3.1
rsa==4.8
ruamel.yaml==0.16.6
ruamel.yaml.clib==0.2.6
s3transfer==0.5.2
scipy==1.5.4
six==1.16.0
sniffio==1.2.0
starlette==0.19.1
tabulate==0.8.9
tensorboard==2.9.1
tensorboard-data-server==0.6.1
tensorboard-plugin-wit==1.8.1
tensorflow==1.2.1
timm==0.5.4
tomli==1.2.3
torch==1.10.1
torchmetrics==0.8.2
torchvision==0.11.2
towhee==0.6.1
towhee.models==0.6.0
tqdm==4.64.0
typed-ast==1.5.4
typing_extensions==4.1.1
ujson==4.3.0
urllib3==1.26.9
uuid==1.30
uvicorn==0.16.0
uvloop==0.14.0
virtualenv==20.14.1
watchgod==0.7
websockets==9.1
Werkzeug==2.0.3
yarl==1.7.2
zipp==3.6.0
3.图片特征提取 resnet50
import torch
import torch.nn as nn
from torchvision import models, transforms
from torch.autograd import Variable
from PIL import Image
# 提取图片特征
def get_resnet50_feature(img_path):
transform1 = transforms.Compose([ # 串联多个图片变换的操作
# transforms.Scale((256, 256)), # 缩放
transforms.Resize((256, 256)),
transforms.CenterCrop((224, 224)), # 中心裁剪
transforms.ToTensor()] # 转换成Tensor
)
img = Image.open(img_path) # 打开图片
img1 = transform1(img) # 对图片进行transform1的各种操作
resnet50_feature_extractor = models.resnet50(pretrained=True) # 导入ResNet50的预训练模型
resnet50_feature_extractor.fc = nn.Linear(2048, 2048) # 重新定义最后一层
# torch.nn.init.eye(resnet50_feature_extractor.fc.weight) # 将二维tensor初始化为单位矩阵
torch.nn.init.eye_(resnet50_feature_extractor.fc.weight)
for param in resnet50_feature_extractor.parameters():
param.requires_grad = False
x = Variable(torch.unsqueeze(img1, dim=0).float(), requires_grad=False)
y = resnet50_feature_extractor(x)
y = y.data.numpy()
return y
4.导入到milvus
def insert_data(img_path, work_id):
y = get_resnet50_feature(img_path)
fields = [
FieldSchema(name="id", dtype=DataType.INT64, is_primary=True, auto_id=True),
FieldSchema(name="d1", dtype=DataType.FLOAT_VECTOR, dim=2048)
]
schema = CollectionSchema(fields)
print("Create collection `hello_milvus`")
hello_milvus = Collection("h2", schema, consistency_level="Strong")
insert_result = hello_milvus.insert([y])
print("insert_result:", insert_result)
print("primary_keys:", insert_result.primary_keys[0])
#
id = insert_result.primary_keys[0]
insert_to_mysql(mid=id, path=img_path, work_id=work_id)