171 lines
8.2 KiB
Python
171 lines
8.2 KiB
Python
|
import json
|
|||
|
|
import logging
|
|||
|
|
import re
|
|||
|
|
from typing import List, Any, Union
|
|||
|
|
import concurrent
|
|||
|
|
from pydantic import BaseModel, Field
|
|||
|
|
from difflib import SequenceMatcher
|
|||
|
|
from configs import (VECTOR_SEARCH_TOP_K,
|
|||
|
|
SCORE_THRESHOLD,
|
|||
|
|
DEFAULT_POLICY_BASE)
|
|||
|
|
from server.agent.tools import search_internet
|
|||
|
|
from server.chat import utils
|
|||
|
|
from server.knowledge_base.kb_doc_api import search_docs
|
|||
|
|
from server.knowledge_base.kb_service.base import KBServiceFactory
|
|||
|
|
from server.utils import BaseResponse
|
|||
|
|
|
|||
|
|
|
|||
|
|
class KnowledgeKgoInput(BaseModel):
|
|||
|
|
location: str = Field(description="Query for Internet search")
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
def preprocess_text(text: str) -> str:
|
|||
|
|
# 去除空格和特殊符号
|
|||
|
|
text = re.sub(r'[\s\W]', '', text.strip())
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
|
|||
|
|
def knowledge_temperature(a: str, b: str) -> float:
|
|||
|
|
# 使用difflib中的SequenceMatcher计算相似度
|
|||
|
|
return SequenceMatcher(None, a, b).ratio()
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
# def knowledgebase_kgo_iter(query: str,
|
|||
|
|
# fileName: List = [],
|
|||
|
|
# knowledge_base_name: str = DEFAULT_POLICY_BASE,
|
|||
|
|
# top_k: int = VECTOR_SEARCH_TOP_K,
|
|||
|
|
# score_threshold: float = SCORE_THRESHOLD) -> BaseResponse | list[str] | Any:
|
|||
|
|
# kb = KBServiceFactory.get_service_by_name(knowledge_base_name)
|
|||
|
|
# if kb is None:
|
|||
|
|
# return BaseResponse(code=404, msg=f"未找到知识库 {knowledge_base_name}")
|
|||
|
|
# query = query.strip()
|
|||
|
|
|
|||
|
|
# docs = search_docs(fileName=fileName,
|
|||
|
|
# query=query,
|
|||
|
|
# knowledge_base_name=knowledge_base_name,
|
|||
|
|
# top_k=top_k,
|
|||
|
|
# score_threshold=score_threshold)
|
|||
|
|
|
|||
|
|
# # 预处理查询文本
|
|||
|
|
# processed_query = preprocess_text(query).replace("Observ","")
|
|||
|
|
# print("processed_query:", processed_query)
|
|||
|
|
# knowledge_docs = []
|
|||
|
|
# knowledge_content = []
|
|||
|
|
# # 知识库返回的文档与query的相似度
|
|||
|
|
# if docs:
|
|||
|
|
# for enum, doc in enumerate(docs):
|
|||
|
|
# filename = doc.metadata.get("title")
|
|||
|
|
# detail_url = "https://policy.ckcest.cn/detail/" + doc.metadata.get("primary_key") + ".html"
|
|||
|
|
# if filename:
|
|||
|
|
# text = f"""政策类资料[{enum + 1}]: [{filename}]({detail_url})\n"""
|
|||
|
|
# else:
|
|||
|
|
# text = f"""政策类资料[{enum + 1}]: [{"原文地址"}]({detail_url})\n"""
|
|||
|
|
# knowledge_docs.append(text)
|
|||
|
|
# # print("knowledge_docs:", knowledge_docs)
|
|||
|
|
# knowledge_content = [doc.page_content for doc in docs]
|
|||
|
|
# # print("knowledge_content:", knowledge_content)
|
|||
|
|
# # 计算知识库返回的文档与query的相似度
|
|||
|
|
# titles = [doc.metadata.get("title") for doc in docs]
|
|||
|
|
# print("titles:", titles)
|
|||
|
|
# def check_similarity_threshold(titles: List[str], query: str, knowledge_docs: List[str], knowledge_content: List[str]) -> Union[
|
|||
|
|
# List[str], None]:
|
|||
|
|
# # 用于记录是否存在相似度大于0.55的标题
|
|||
|
|
# has_similar_title = False
|
|||
|
|
# for title in titles:
|
|||
|
|
# processed_title = preprocess_text(title)
|
|||
|
|
# similarity = knowledge_temperature(processed_query, processed_title)
|
|||
|
|
# print("processed_title:", processed_title)
|
|||
|
|
# print("similarity:", similarity)
|
|||
|
|
# if similarity >= 0.55:
|
|||
|
|
# has_similar_title = True
|
|||
|
|
# break
|
|||
|
|
# # 如果存在相似度大于0.55的标题，则直接返回 knowledge_docs
|
|||
|
|
# if has_similar_title:
|
|||
|
|
# knowledge = knowledge_content + knowledge_docs
|
|||
|
|
# return knowledge
|
|||
|
|
# # 如果所有标题的相似度都不大于0.55，则返回 None
|
|||
|
|
# return None
|
|||
|
|
|
|||
|
|
# # 在原函数中使用新的函数进行相似度阈值的判断
|
|||
|
|
# similar_docs = check_similarity_threshold(titles, query, knowledge_docs, knowledge_content)
|
|||
|
|
# if similar_docs is None:
|
|||
|
|
# # 如果所有标题的相似度都不大于0.55，则执行搜索引擎查询
|
|||
|
|
# kgo_docs = search_internet(processed_query)
|
|||
|
|
# # print("kgo_docs", kgo_docs)
|
|||
|
|
# return kgo_docs
|
|||
|
|
# else:
|
|||
|
|
# kgo_docs = search_internet(processed_query)
|
|||
|
|
# # print("similar_docs", similar_docs)
|
|||
|
|
# # print("kgo_docs", kgo_docs)
|
|||
|
|
# similar_docs.extend(kgo_docs)
|
|||
|
|
# return similar_docs
|
|||
|
|
# else:
|
|||
|
|
# # 执行搜索引擎查询
|
|||
|
|
# kgo_docs = search_internet(query)
|
|||
|
|
# return kgo_docs
|
|||
|
|
|
|||
|
|
def knowledgebase_kgo_iter(query: str, uid: str) -> BaseResponse | list[str] | Any:
|
|||
|
|
kgo_docs = search_internet(query , uid)
|
|||
|
|
|
|||
|
|
return kgo_docs
|
|||
|
|
def knowledgebase_kgo_search(query: str) -> List[str]:
|
|||
|
|
try:
|
|||
|
|
matches = re.findall(r'\{.*?\}', query)
|
|||
|
|
if len(matches)>=2:
|
|||
|
|
query = matches[0]
|
|||
|
|
parsed_query = json.loads(query)
|
|||
|
|
# 继续使用解析后的查询进行后续操作
|
|||
|
|
time_based_uuid = json.loads(matches[1])["uuid"]
|
|||
|
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|||
|
|
future = executor.submit(knowledgebase_kgo_iter, parsed_query["query"], time_based_uuid)
|
|||
|
|
res = future.result()
|
|||
|
|
# res = knowledgebase_kgo_iter(parsed_query["query"],time_based_uuid)
|
|||
|
|
try:
|
|||
|
|
if type(res[0])==list and len(res[0])>0:
|
|||
|
|
doc_content = "资料内容"
|
|||
|
|
for doc in res[0]:
|
|||
|
|
doc_content += doc
|
|||
|
|
doc_content += "资料来源"
|
|||
|
|
for source in res[1]:
|
|||
|
|
doc_content += source
|
|||
|
|
return f"{doc_content}"
|
|||
|
|
elif type(res[1])==list and len(res[1])>0:
|
|||
|
|
doc_content += "资料来源"
|
|||
|
|
for source in res[1]:
|
|||
|
|
doc_content += source
|
|||
|
|
return "只有标题没有内容，标题为：{doc_content}"
|
|||
|
|
else:
|
|||
|
|
# return "<system>不要再调用工具了</system>"
|
|||
|
|
return f"尝试调整入参重新调用联网思索工具（同一个问题调用超过三次就不要再使用该工具了，浪费时间）"
|
|||
|
|
except Exception as e:
|
|||
|
|
logging.error(f"Error occurred while processing query: {e}")
|
|||
|
|
# return "<system>不要再调用该工具了，根据已有资料或自身能力回答</system>"
|
|||
|
|
return f"尝试调整入参重新调用联网思索工具（同一个问题调用超过三次就不要再使用该工具了，浪费时间）"
|
|||
|
|
else:
|
|||
|
|
logging.error("Invalid JSON format in query.")
|
|||
|
|
# return "<system>不要再调用该工具了</system>"
|
|||
|
|
return f"尝试调整入参重新调用联网思索工具（同一个问题调用超过三次就不要再使用该工具了，浪费时间）"
|
|||
|
|
except json.JSONDecodeError:
|
|||
|
|
# 如果JSON解码失败，则返回错误消息
|
|||
|
|
logging.error("Invalid JSON format in query.")
|
|||
|
|
# return "<system>不要再调用该工具了</system>"
|
|||
|
|
return f"尝试调整入参重新调用联网思索工具（同一个问题调用超过三次就不要再使用该工具了，浪费时间）"
|
|||
|
|
except KeyError:
|
|||
|
|
# 如果解析的JSON对象中缺少必要的键，则返回错误消息
|
|||
|
|
# return "<system>不要再调用该工具了</system>"
|
|||
|
|
return f"尝试调整入参重新调用联网思索工具（同一个问题调用超过三次就不要再使用该工具了，浪费时间）"
|
|||
|
|
except Exception as e:
|
|||
|
|
# 捕获其他所有异常，并返回通用错误消息
|
|||
|
|
logging.error(f"Error occurred while processing query: {e}")
|
|||
|
|
# return f"<system>不要再调用该工具了</system>"
|
|||
|
|
return f"尝试调整入参重新调用联网思索工具（同一个问题调用超过三次就不要再使用该工具了，浪费时间）"
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
result = knowledgebase_kgo_iter("《区块链和分布式记账技术标准体系建设指南》")
|
|||
|
|
print("检索结果:", result)
|