feat: ES VectorStore

.env.template

@@ -163,6 +163,13 @@ VECTOR_STORE_TYPE=Chroma
 #VECTOR_STORE_TYPE=Weaviate
 #WEAVIATE_URL=https://kt-region-m8hcy0wc.weaviate.network
 
+## ElasticSearch vector db config
+#VECTOR_STORE_TYPE=ElasticSearch
+ElasticSearch_URL=127.0.0.1
+ElasticSearch_PORT=9200
+ElasticSearch_USERNAME=elastic
+ElasticSearch_PASSWORD=i=+iLw9y0Jduq86XTi6W
+
 #*******************************************************************#
 #** WebServer Language Support **#
 #*******************************************************************#

.mypy.ini

@@ -59,6 +59,9 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 follow_imports = skip
 
+[mypy-jieba.*]
+ignore_missing_imports = True
+
 # Storage
 [mypy-msgpack.*]
 ignore_missing_imports = True
@@ -72,6 +75,9 @@ ignore_missing_imports = True
 [mypy-pymilvus.*]
 ignore_missing_imports = True
 
+[mypy-elasticsearch.*]
+ignore_missing_imports = True
+
 [mypy-cryptography.*]
 ignore_missing_imports = True
 

dbgpt/_private/config.py

@@ -207,7 +207,7 @@ def __init__(self) -> None:
  os.getenv("NATIVE_SQL_CAN_RUN_WRITE", "True").lower() == "true"
  )
 
- ###dbgpt meta info database connection configuration
+ ### dbgpt meta info database connection configuration
  self.LOCAL_DB_HOST = os.getenv("LOCAL_DB_HOST")
  self.LOCAL_DB_PATH = os.getenv("LOCAL_DB_PATH", "data/default_sqlite.db")
  self.LOCAL_DB_TYPE = os.getenv("LOCAL_DB_TYPE", "sqlite")
@@ -247,6 +247,11 @@ def __init__(self) -> None:
  self.MILVUS_PORT = os.getenv("MILVUS_PORT", "19530")
  self.MILVUS_USERNAME = os.getenv("MILVUS_USERNAME", None)
  self.MILVUS_PASSWORD = os.getenv("MILVUS_PASSWORD", None)
+ # Elasticsearch Vector Configuration
+ self.ELASTICSEARCH_URL = os.getenv("ELASTICSEARCH_URL", "127.0.0.1")
+ self.ELASTICSEARCH_PORT = os.getenv("ELASTICSEARCH_PORT", "9200")
+ self.ELASTICSEARCH_USERNAME = os.getenv("ELASTICSEARCH_USERNAME", None)
+ self.ELASTICSEARCH_PASSWORD = os.getenv("ELASTICSEARCH_PASSWORD", None)
 
  ## OceanBase Configuration
  self.OB_HOST = os.getenv("OB_HOST", "127.0.0.1")

dbgpt/app/knowledge/service.py

@@ -482,7 +482,13 @@ def delete_document(self, space_name: str, doc_name: str):
  raise Exception(f"there are no or more than one document called {doc_name}")
  vector_ids = documents[0].vector_ids
  if vector_ids is not None:
- config = VectorStoreConfig(name=space_name)
+ embedding_factory = CFG.SYSTEM_APP.get_component(
+ "embedding_factory", EmbeddingFactory
+ )
+ embedding_fn = embedding_factory.create(
+ model_name=EMBEDDING_MODEL_CONFIG[CFG.EMBEDDING_MODEL]
+ )
+ config = VectorStoreConfig(name=space_name, embedding_fn=embedding_fn)
  vector_store_connector = VectorStoreConnector(
  vector_store_type=CFG.VECTOR_STORE_TYPE,
  vector_store_config=config,

dbgpt/storage/vector_store/__init__.py

@@ -32,6 +32,12 @@ def _import_oceanbase() -> Any:
  return OceanBaseStore
 
 
+def _import_elastic() -> Any:
+ from dbgpt.storage.vector_store.elastic_store import ElasticStore
+
+ return ElasticStore
+
+
 def __getattr__(name: str) -> Any:
  if name == "Chroma":
  return _import_chroma()
@@ -43,8 +49,10 @@ def __getattr__(name: str) -> Any:
  return _import_pgvector()
  elif name == "OceanBase":
  return _import_oceanbase()
+ elif name == "ElasticSearch":
+ return _import_elastic()
  else:
  raise AttributeError(f"Could not find: {name}")
 
 
-__all__ = ["Chroma", "Milvus", "Weaviate", "OceanBase", "PGVector"]
+__all__ = ["Chroma", "Milvus", "Weaviate", "OceanBase", "PGVector", "ElasticSearch"]