1) FixBug

2) Add Documents abount TASK/TODO Refactor
This commit is contained in:
Liu Zhicong
2024-01-06 13:08:41 -08:00
parent 3663f140fc
commit c975ba2053
22 changed files with 398 additions and 109 deletions
@@ -132,7 +132,7 @@ class MetaDatabase:
create_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
summary = metadata.get("summary", "")
catalogs = json.dumps(metadata.get("catalogs", {}))
catalogs = json.dumps(metadata.get("catalogs", {}),ensure_ascii=False)
title = metadata.get("title","")
tags = ','.join(metadata.get("tags", []))
@@ -151,7 +151,7 @@ class MetaDatabase:
title = meta.get("title", "")
summary = meta.get("summary", "")
catalogs = json.dumps(meta.get("catalogs", {}))
catalogs = json.dumps(meta.get("catalogs", {}),ensure_ascii=False)
tags = ','.join(meta.get("tags", []))
cursor.execute('''
@@ -224,7 +224,7 @@ class MetaDatabase:
def query_docs_by_tag(self, tag: str) -> List[str]:
conn = self._get_conn()
cursor = conn.cursor()
tag_json = json.dumps(tag) # 将标签转换为 JSON 字符串
tag_json = json.dumps(tag,ensure_ascii=False) # 将标签转换为 JSON 字符串
cursor.execute('''
SELECT documents.doc_path
FROM documents
@@ -450,7 +450,7 @@ class ParseLocalDocument:
org_path = meta.get("original_path")
known_obj["original_path"] = org_path
return f"# Known information:\n## Current directory structure:\n{kb_tree}\n## Knowlege Metadata:\n{json.dumps(known_obj)}\n"
return f"# Known information:\n## Current directory structure:\n{kb_tree}\n## Knowlege Metadata:\n{json.dumps(known_obj,ensure_ascii=False)}\n"
def _token_len(self, text: str) -> int:
return CustomAIAgent("", "gpt-4-1106-preview", self.token_limit).token_len(text=text)
@@ -563,7 +563,7 @@ class ParseLocalDocument:
if bookmarks:
catalogs = []
self._parse_pdf_bookmarks(bookmarks,catalogs)
metadata["catalogs"] = json.dumps(catalogs)
metadata["catalogs"] = json.dumps(catalogs,ensure_ascii=False)
except Exception as e:
logger.warn("parse pdf bookmarks failed:%s",e)
@@ -55,7 +55,7 @@ class FilesystemEnvironment(SimpleEnvironment):
item_type = "directory" if is_dir else "file"
items.append({"name": entry.name, "type": item_type})
return json.dumps(items)
return json.dumps(items,ensure_ascii=False)
# inner_function
async def read(self,path:str) -> str:
@@ -120,7 +120,7 @@ class FilesystemEnvironment(SimpleEnvironment):
try:
file_path = self.root_path + path
stat = os.stat(file_path)
return json.dumps(stat)
return json.dumps(stat,ensure_ascii=False)
except Exception as e:
return str(e)
+1 -1
View File
@@ -106,7 +106,7 @@ class Issue:
child = desc_list.pop()
root["child"] = child
root = child
return json.dumps(root)
return json.dumps(root,ensure_ascii=False)
@classmethod
+1 -1
View File
@@ -32,7 +32,7 @@ class Mail:
"date": self.date,
"content": self.content
}
return json.dumps(prompt)
return json.dumps(prompt,ensure_ascii=False)
@classmethod
def prompt_desc(cls) -> dict:
+1 -1
View File
@@ -223,7 +223,7 @@ class OpenAI_ComputeNode(ComputeNode):
max_tokens=result_token,
)
else:
logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions)}")
logger.info(f"call openai {mode_name} prompts: \n\t {prompts} \nfunctions: \n\t{json.dumps(llm_inner_functions,ensure_ascii=False)}")
resp = await client.chat.completions.create(model=mode_name,
messages=prompts,
response_format = response_format,
+4 -4
View File
@@ -156,11 +156,11 @@ class WhisperComputeNode(ComputeNode):
result.result_str = text
result.result = text
elif response_format == "json":
result.result_str = json.dumps({"text": text})
result.result_str = json.dumps({"text": text},ensure_ascii=False)
resp.text = text
result.result = resp
elif response_format == "verbose_json":
result.result_str = json.dumps({"text": text, "segments": results})
result.result_str = json.dumps({"text": text, "segments": results},ensure_ascii=False)
latest_resp.text = text
latest_resp.segments = results
result.result = latest_resp
@@ -190,9 +190,9 @@ class WhisperComputeNode(ComputeNode):
result.set_from_task(task)
result.worker_id = self.node_id
if response_format == "json":
result.result_str = json.dumps({"text": resp.text})
result.result_str = json.dumps({"text": resp.text},ensure_ascii=False)
elif response_format == "verbose_json":
result.result_str = json.dumps({"text": resp.text, "segments": resp.segments})
result.result_str = json.dumps({"text": resp.text, "segments": resp.segments},ensure_ascii=False)
elif response_format == "srt" or response_format == "vtt" or response_format == "text":
result.result_str = resp
else: