diff --git a/aios_shell_history.txt b/aios_shell_history.txt deleted file mode 100644 index 34732fd..0000000 --- a/aios_shell_history.txt +++ /dev/null @@ -1,111 +0,0 @@ - -# 2023-09-19 16:04:55.011656 -+tsukasa - -# 2023-09-19 16:05:27.714815 -+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 - -# 2023-09-20 14:48:06.181582 -+/connect - -# 2023-09-20 14:48:22.678945 -+/connect Jarvis email - -# 2023-09-20 14:54:35.101026 -+/connect Jarvis telegram - -# 2023-09-20 14:54:55.229801 -+token abcd - -# 2023-09-20 16:22:34.229084 -+/knowledge add email - -# 2023-09-20 16:22:46.797842 -+puotosssa@live.com - -# 2023-09-20 16:22:53.622015 -+p19870626 - -# 2023-09-20 16:22:59.186145 -+imap@live.com - -# 2023-09-20 16:23:02.334411 -+993 - -# 2023-09-20 16:40:29.996329 -+/knowledge add email - -# 2023-09-20 16:40:52.860700 -+puotosssa@live.com - -# 2023-09-20 16:40:55.324994 -+fdafda - -# 2023-09-20 16:40:58.167154 -+fdafdaf - -# 2023-09-20 16:40:59.757363 -+993 - -# 2023-09-20 16:42:55.288680 -+/knowledge add email - -# 2023-09-20 16:42:56.916408 -+fdfa - -# 2023-09-20 16:42:57.965232 -+fdfds - -# 2023-09-20 16:42:59.149516 -+fdafd - -# 2023-09-20 16:43:00.949104 -+993 - -# 2023-09-21 14:11:33.500467 -+/knowledge add c:/users/tsukasa/myai/photos - -# 2023-09-21 14:14:31.488794 -+/knowledge add dir - -# 2023-09-21 14:14:49.298635 -+c:/users/tsukasa/myai/photos - -# 2023-09-21 15:27:58.550658 -+/knowledge add dir - -# 2023-09-21 15:28:10.190467 -+c:/users/tsukasa/myai/photos - -# 2023-09-21 15:32:32.431305 -+/knowledge add dir - -# 2023-09-21 15:32:46.695566 -+c:/users/tsukasa/myai/photos - -# 2023-09-21 15:33:36.391465 -+/knowledge add dir - -# 2023-09-21 15:33:47.574853 -+c:/users/tsukasa/myai/photos - -# 2023-09-21 17:44:26.308692 -+/knowledge journals - -# 2023-09-21 17:45:42.925359 -+/knowledge journal - -# 2023-09-21 18:13:08.720054 -+/knowledge query what a see is - -# 2023-09-21 18:20:09.275556 -+/knowledge journal - -# 2023-09-21 18:20:10.746056 -+/knowledge query what a see is - -# 2023-09-21 18:29:48.391223 -+/knowledge journal - -# 2023-09-21 18:29:51.406532 -+/knowledge query what a see is diff --git a/rootfs/agents/Jarvis/agent.toml b/rootfs/agents/Jarvis/agent.toml index 1c56e5f..2da373c 100644 --- a/rootfs/agents/Jarvis/agent.toml +++ b/rootfs/agents/Jarvis/agent.toml @@ -3,6 +3,7 @@ fullname = "Jarvis" llm_model_name = "gpt-3.5-turbo-16k-0613" max_token_size = 16000 #enable_function =["add_event"] +enable_kb = "true" [[prompt]] role = "system" diff --git a/src/aios_kernel/agent.py b/src/aios_kernel/agent.py index 18eeeec..4742003 100644 --- a/src/aios_kernel/agent.py +++ b/src/aios_kernel/agent.py @@ -109,6 +109,7 @@ class AIAgent: self.fullname:str = None self.powerby = None self.enable = True + self.enable_kb = False self.chat_db = None self.unread_msg = Queue() # msg from other agent @@ -154,6 +155,8 @@ class AIAgent: self.max_token_size = config["max_token_size"] if config.get("enable_function") is not None: self.enable_function_list = config["enable_function"] + if config.get("enable_kb") is not None: + self.enable_kb = bool(config["enable_kb"]) return True @@ -300,9 +303,18 @@ class AIAgent: old_content = msg.get("content") msg["content"] = old_content.format_map(self.owner_env) + async def _get_knowlege_prompt(self,input_msg:AgentPrompt) -> AgentPrompt: + if self.enable_kb is False: + return None + + from .knowledge_base import KnowledgeBase + return await KnowledgeBase().query_prompt(input_msg) + async def _process_msg(self,msg:AgentMsg) -> AgentMsg: from .compute_kernel import ComputeKernel from .bus import AIBus + + session_topic = msg.get_sender() + "#" + msg.topic chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db) @@ -311,22 +323,27 @@ class AIAgent: chatsession.append(msg) logger.info(f"agent {self.agent_id} recv a group chat message from {msg.sender},but is not mentioned,ignore!") return None - + + msg_prompt = AgentPrompt() + msg_prompt.messages = [{"role":"user","content":msg.body}] + prompt = AgentPrompt() prompt.append(await self._get_agent_prompt()) + self._format_msg_by_env_value(prompt) + inner_functions,function_token_len = self._get_inner_functions() - # prompt.append(self._get_knowlege_prompt(the_role.get_name())) + system_prompt_len = prompt.get_prompt_token_len() input_len = len(msg.body) history_prmpt,history_token_len = await self._get_prompt_from_session(chatsession,system_prompt_len + function_token_len,input_len) prompt.append(history_prmpt) # chat context - msg_prompt = AgentPrompt() - msg_prompt.messages = [{"role":"user","content":msg.body}] + kb_prompt = await self._get_knowlege_prompt(msg_prompt) + prompt.append(kb_prompt) prompt.append(msg_prompt) - self._format_msg_by_env_value(prompt) + logger.debug(f"Agent {self.agent_id} do llm token static system:{system_prompt_len},function:{function_token_len},history:{history_token_len},input:{input_len},totoal prompt:{system_prompt_len + function_token_len + history_token_len} ") task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,self.llm_model_name,self.max_token_size,inner_functions) final_result = task_result.result_str diff --git a/src/aios_kernel/knowledge_base.py b/src/aios_kernel/knowledge_base.py index 11d6ce7..d2debd8 100644 --- a/src/aios_kernel/knowledge_base.py +++ b/src/aios_kernel/knowledge_base.py @@ -165,8 +165,8 @@ class KnowledgeBase: objects = await self.query_objects(prompt) knowledge_prompt = self.prompt_from_objects(objects) logging.info(f"prompt_from_objects result: {knowledge_prompt.as_str()}") - prompt.append(knowledge_prompt) - return prompt + + return knowledge_prompt async def query_objects(self, prompt: AgentPrompt) -> [ObjectID]: results = [] @@ -204,7 +204,7 @@ class KnowledgeBase: else: results[str(root_object_id)] = [root_object_id, object_id] - content = "I found the following contents described with json format:\n" + content = "*** I have provided the following known information for your reference with json format:\n" result_desc = [] for result in results.values(): # first element in result is the root object @@ -239,7 +239,7 @@ class KnowledgeBase: content += ".\n" prompt = AgentPrompt() - prompt.messages.append({"role": "knowledge", "content": content}) + prompt.messages.append({"role": "user", "content": content}) return prompt diff --git a/src/aios_kernel/knowledge_pipeline.py b/src/aios_kernel/knowledge_pipeline.py index c1f0a72..d1ed109 100644 --- a/src/aios_kernel/knowledge_pipeline.py +++ b/src/aios_kernel/knowledge_pipeline.py @@ -24,6 +24,9 @@ import mailparser import hashlib import json import base64 +import chardet +import aiofiles + from bs4 import BeautifulSoup import requests import os @@ -254,7 +257,16 @@ class KnowledgeDirSource: def path(self): return self.config["path"] - + + @staticmethod + async def read_txt_file(file_path:str)->str: + cur_encode = "utf-8" + async with aiofiles.open(file_path,'rb') as f: + cur_encode = chardet.detect(await f.read())['encoding'] + + async with aiofiles.open(file_path,'r',encoding=cur_encode) as f: + return await f.read() + async def run_once(self): logging.debug(f"knowledge dir source {self.id()} run once") journal_client = KnowledgeJournalClient() @@ -278,11 +290,11 @@ class KnowledgeDirSource: journal_client.insert(KnowledgeJournal("dir", self.id(), rel_path, str(image.calculate_id()), timestamp)) if ext in ['.txt']: logging.info(f"knowledge dir source {self.id()} found text file {file_path}") - with open(file_path, "r", encoding="utf-8") as f: - text = f.read() - document = DocumentObjectBuilder({}, {}, text).build() - await KnowledgeBase().insert_object(document) - journal_client.insert(KnowledgeJournal("dir", self.id(), rel_path, str(document.calculate_id()), timestamp)) + text = await self.read_txt_file(file_path) + + document = DocumentObjectBuilder({}, {}, text).build() + await KnowledgeBase().insert_object(document) + journal_client.insert(KnowledgeJournal("dir", self.id(), rel_path, str(document.calculate_id()), timestamp)) diff --git a/src/requirements.txt b/src/requirements.txt index 0c88c8a..753ce51 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -80,4 +80,5 @@ starlette==0.27.0 sympy==1.12 telegram==0.0.1 tokenizers==0.14.0 -toml==0.10.0 \ No newline at end of file +toml==0.10.0 +chardet \ No newline at end of file