From 3180a4a279f99074f843ed3e0adac52add8c300c Mon Sep 17 00:00:00 2001 From: HuangHai <10402852@qq.com> Date: Thu, 26 Jun 2025 10:37:37 +0800 Subject: [PATCH] 'commit' --- dsRag/Doc/24、向量数据库Milvus.md | 154 ++++++++++++++++++++++++ dsRag/Milvus/X2_create_index.py | 6 +- dsRag/Milvus/X3_insert_data.py | 6 +- dsRag/Milvus/X4_select_all_data.py | 6 +- dsRag/Milvus/X5_search_near_data.py | 6 +- 5 files changed, 166 insertions(+), 12 deletions(-) create mode 100644 dsRag/Doc/24、向量数据库Milvus.md diff --git a/dsRag/Doc/24、向量数据库Milvus.md b/dsRag/Doc/24、向量数据库Milvus.md new file mode 100644 index 00000000..fd0d38e2 --- /dev/null +++ b/dsRag/Doc/24、向量数据库Milvus.md @@ -0,0 +1,154 @@ +### 官网 + +https://milvus.io/zh + + +### 二、安装 + +```shell +# 从嘟嘟鸟下载镜像 +docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/quay.io/coreos/etcd:v3.5.5 +docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/quay.io/coreos/etcd:v3.5.5 quay.io/coreos/etcd:v3.5.5 + +docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/minio/minio:RELEASE.2023-03-20T20-16-18Z +docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/minio/minio:RELEASE.2023-03-20T20-16-18Z docker.io/minio/minio:RELEASE.2023-03-20T20-16-18Z + +docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/milvusdb/milvus:v2.3.12 +docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/milvusdb/milvus:v2.3.12 docker.io/milvusdb/milvus:v2.3.12 +``` + + +> 一般是下载这个脚本： +> wget https://github.com/milvus-io/milvus/releases/download/v2.3.1/milvus-standalone-docker-compose.yml -O docker-compose.yml + +下面的是我修改后的，可以直接使用 + +```shell +cd /usr/local +mkdir milvus +cd milvus +vi docker-compose.yml +``` + +内容如下： + +```shell +services: + etcd: + container_name: milvus-etcd + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd + ports: + - "2379:2379" + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 5s + timeout: 3s + retries: 10 + + minio: + container_name: milvus-minio + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "9001:9001" + - "9000:9000" + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 10 + + standalone: + container_name: milvus-standalone + image: milvusdb/milvus:v2.3.12 + command: ["milvus", "run", "standalone"] + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + start_period: 90s + timeout: 20s + retries: 3 + ports: + - "19530:19530" + - "9091:9091" + depends_on: + - "etcd" + - "minio" + +networks: + default: + name: milvus +``` + + + +```shell +# 创建容器 +docker-compose up -d +# 查看容器 +docker-compose ps +# 停止容器 +docker-compose stop +# 启动容器 +docker-compose start +# 停止并删除容器(特别注意以免误删容器) +docker-compose down + +# 查看日志 +docker logs -f milvus-standalone +``` + + + +参考文档 + +``` +https://developer.aliyun.com/article/1373358 +``` + + + +### 启动$attu$ + +$attu$为$milvus$的一款图形化管理工具，非常方便对$milvus$的一些管理。 + +```shell +docker pull swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/zilliz/attu:v2.5.5 +docker tag swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/zilliz/attu:v2.5.5 docker.io/zilliz/attu:v2.5.5 + +docker run -d --name=attu -p 8000:3000 -e MILVUS_URL=192.168.230.71:19530 docker.io/zilliz/attu:v2.5.5 +``` + +$web$访问地址: + +> http://10.10.14.205:8000/ + + + +到这里我们就完成了整个$standalone$的安装部署过程。 +我们还可以访问$minio$对象存储的管理界面，查看$milvus$存储内容。 + +> http://10.10.14.205:9001/ +> +> 账户和密码:$minioadmin$ + + + diff --git a/dsRag/Milvus/X2_create_index.py b/dsRag/Milvus/X2_create_index.py index 3d06a886..745917de 100644 --- a/dsRag/Milvus/X2_create_index.py +++ b/dsRag/Milvus/X2_create_index.py @@ -1,6 +1,6 @@ -from WxMini.Milvus.Utils.MilvusCollectionManager import MilvusCollectionManager -from WxMini.Milvus.Utils.MilvusConnectionPool import * -from WxMini.Milvus.Config.MulvusConfig import * +from Milvus.Utils.MilvusCollectionManager import MilvusCollectionManager +from Milvus.Utils.MilvusConnectionPool import * +from Milvus.Config.MulvusConfig import * # 1. 使用连接池管理 Milvus 连接 milvus_pool = MilvusConnectionPool(host=MS_HOST, port=MS_PORT, max_connections=MS_MAX_CONNECTIONS) diff --git a/dsRag/Milvus/X3_insert_data.py b/dsRag/Milvus/X3_insert_data.py index c8b68618..bccba082 100644 --- a/dsRag/Milvus/X3_insert_data.py +++ b/dsRag/Milvus/X3_insert_data.py @@ -1,6 +1,6 @@ -from WxMini.Milvus.Config.MulvusConfig import * -from WxMini.Milvus.Utils.MilvusCollectionManager import MilvusCollectionManager -from WxMini.Milvus.Utils.MilvusConnectionPool import * +from Milvus.Config.MulvusConfig import * +from Milvus.Utils.MilvusCollectionManager import MilvusCollectionManager +from Milvus.Utils.MilvusConnectionPool import * from gensim.models import KeyedVectors import jieba diff --git a/dsRag/Milvus/X4_select_all_data.py b/dsRag/Milvus/X4_select_all_data.py index 0a8395ba..2ca5528c 100644 --- a/dsRag/Milvus/X4_select_all_data.py +++ b/dsRag/Milvus/X4_select_all_data.py @@ -1,6 +1,6 @@ -from WxMini.Milvus.Utils.MilvusCollectionManager import MilvusCollectionManager -from WxMini.Milvus.Utils.MilvusConnectionPool import * -from WxMini.Milvus.Config.MulvusConfig import * +from Milvus.Utils.MilvusCollectionManager import MilvusCollectionManager +from Milvus.Utils.MilvusConnectionPool import * +from Milvus.Config.MulvusConfig import * # 1. 使用连接池管理 Milvus 连接 milvus_pool = MilvusConnectionPool(host=MS_HOST, port=MS_PORT, max_connections=MS_MAX_CONNECTIONS) diff --git a/dsRag/Milvus/X5_search_near_data.py b/dsRag/Milvus/X5_search_near_data.py index f75dc867..ea58e1c9 100644 --- a/dsRag/Milvus/X5_search_near_data.py +++ b/dsRag/Milvus/X5_search_near_data.py @@ -1,8 +1,8 @@ import time import jieba # 导入 jieba 分词库 -from WxMini.Milvus.Utils.MilvusCollectionManager import MilvusCollectionManager -from WxMini.Milvus.Utils.MilvusConnectionPool import * -from WxMini.Milvus.Config.MulvusConfig import * +from Milvus.Utils.MilvusCollectionManager import MilvusCollectionManager +from Milvus.Utils.MilvusConnectionPool import * +from Milvus.Config.MulvusConfig import * from gensim.models import KeyedVectors # 1. 加载预训练的 Word2Vec 模型