fix text summary build bug

rootfs/agents/Jarvis/agent.toml

@@ -1,7 +1,6 @@
 instance_id = "Jarvis"
 fullname = "Jarvis"
-llm_model_name = "gpt-3.5-turbo-16k-0613"
+max_token_size = 128000
-max_token_size = 16000
 #enable_kb = "true"
 enable_timestamp = "true"
 owner_prompt = "I am your master{name}"

src/aios_kernel/agent.py

@@ -21,6 +21,7 @@ from .contact_manager import ContactManager,Contact,FamilyMember
 from .compute_kernel import ComputeKernel
 from .bus import AIBus
 from .workspace_env import WorkspaceEnvironment
+from .storage import AIStorage
 
 from knowledge import *
 
@@ -53,11 +54,36 @@ DEFAULT_AGENT_GOAL_TO_TODO_PROMPT = """
 """
 
 DEFAULT_AGENT_LEARN_PROMPT = """
-你拥有非常优秀的资料整理技能。我给你一段内容，你会尝试对其进行摘要，并在已有的资料库中找到合适的位置存放该文章。
+我是一名软件工程师，拥有非常优秀的资料学习能力。下面是我学习和整理资料的方法
-1. 结合你的角色和组织的工作目标构建摘要，尽量精简，长度不要超过256个字
+1. 结合我的角色为资料产生长度不超过256个Token的摘要;尝试产生不超过16个tag;
-2. 资料库以文件系统的形式组织，浏览知识库是成本高昂的操作，应尝试从根目录往子目录深入来找到最合适的信息。必要的情况下，你可以在合适的位置创建新的目录。为了方便浏览，每一层目录的文件夹数不超过32个，名称长度不超过16个字符，目录深度不超过6层
+2. 现有资料库以文件系统的形式组织，我未来借助资料的摘要来浏览知识库
-3. 你可以从不同的角度给出最多3个合适的位置
+3. 我将学习过的资料另存在资料库的合适位置（以/开始的完整路径）。保存位置的目录深度不超过5层，文件夹名称长度不超过16个字符。
-4. 返回一个json来保存摘要和建议保存位置信息
+4. 当存在已知信息时，需参考已知信息的内容来思考结果。
+5. 由于LLM的Token限制，我学习的可能只是资料的部分内容，此时我应能产生合适的中间结果，中间结果保存在metadata中。当我决定构建中间结果时，我只构建中间结果。
+6. 当我收到最后一部分内容时，我能结合已知的中间结果产生最终结果。
+7. 总是以json格式返回思考结果，json格式如下
+{
+    think:"$think_result",
+    metadata:{...} , # temp result for long content
+    tags:["tag1","tag2"...],
+    path:["/graphic/opengl","/database/mysql"], # list of directories to save to.
+    title:"$article_title",
+    summary:"$summary",
+    catalogs: [{                     # optional,catalogs is a tree
+            title:"$catalog_name1",
+            pos:"$pos:$length"
+            children:[
+                {
+                    title:"$catalog_name 1.1",
+                    pos:"$pos:$length"  
+                } 
+            ]},
+            {
+            title:"$catalog_name2",
+            pos:"$pos:$length"
+            }
+            ]    
+}
 """
 
 DEFAULT_AGENT_LEARN_LONG_CONENT_PROMPT = """
@@ -125,7 +151,7 @@ class AIAgent:
 
         self.goal_to_todo_prompt = None
 
-        self.learn_token_limit = 500
+        self.learn_token_limit = 4000
         self.learn_prompt = AgentPrompt(DEFAULT_AGENT_LEARN_PROMPT)
 
         self.chat_db = None
@@ -217,6 +243,9 @@ class AIAgent:
         return self.template_id
 
     def get_llm_model_name(self) -> str:
+        if self.llm_model_name is None:
+            return AIStorage.get_instance().get_user_config().get_value("llm_model_name")
+        
         return self.llm_model_name
 
     def get_max_token_size(self) -> int:
@@ -889,22 +918,32 @@ class AIAgent:
             if self.agent_energy <= 0:
                 break
 
-            knowledge = workspace.kb_db.get_knowledge_by_hash(hash)
+            knowledge = workspace.kb_db.get_knowledge(hash)
             if knowledge is None:
                 continue
 
-            if os.path.exists(knowledge.path) is False:
+            full_path = knowledge.get("full_path")
-                logger.warning(f"do_self_learn: knowledge {knowledge.path} is not exists!")
+            if full_path is None:
+                continue
+
+            if os.path.exists(full_path) is False:
+                logger.warning(f"do_self_learn: knowledge {full_path} is not exists!")
                 continue
 
              #TODO 可以用v-db 对不同目录的名字进行选择后，先进行一次快速的插入。有时间再慢慢用LLM整理
-            llm_result = await self._llm_read_article(knowledge)
+            result_obj = await self._llm_read_article(knowledge,full_path)
 
             #根据结果更新knowledge
-            if llm_result is not None:
+            if result_obj is not None:
-                workspace.kb_db.update_knowledge_by_hash(hash,llm_result)
+                workspace.kb_db.set_knowledge_llm_result(hash,result_obj)
                 # 在知识库中创建软链接
-
+                path_list = result_obj.get("path")
+                new_title = result_obj.get("title")
+                if path_list:
+                    for new_path in path_list:
+                        full_new_path = f"/knowledge{new_path}/{new_title}"
+                        await workspace.symlink(full_path,full_new_path)
+                        logger.info(f"create soft link {full_path} -> {full_new_path}")
 
 
             self.agent_energy -= 1
@@ -958,13 +997,11 @@ class AIAgent:
     def parser_learn_llm_result(self,llm_result:LLMResult):
         pass
 
-    async def gen_known_info_for_knowledge_prompt(self,knowledge_item:dict,need_catalogs = False) -> AgentPrompt:
+    async def gen_known_info_for_knowledge_prompt(self,knowledge_item:dict,temp_meta = None,need_catalogs = False) -> AgentPrompt:
-        #已知信息：   
-        #   组织的工作总结（如有）待完成
-        #   现在知识库的结构（注意大小控制）gen_kb_tree_prompt (当为空的时候应该让LLM生成一个合适的初始目录结构)
-        #   原始路径，现在标题，摘要，目录
         workspace =self.get_workspace_by_msg(None)
         kb_tree = await workspace.get_knowledege_catalog()
+
+
         known_obj = {}
         title  = knowledge_item.get("title")
         if title:
@@ -980,36 +1017,40 @@ class AIAgent:
             if catalogs:
                 known_obj["catalogs"] = catalogs
 
-        org_path = knowledge_item.get("path")
+        if temp_meta:       
+            for key in temp_meta.keys():
+                known_obj[key] = temp_meta[key]
+
+        org_path = knowledge_item.get("full_path")
         known_obj["orginal_path"] = org_path
-        know_info_str = f"# Known information\n{json.dumps(known_obj)}\n"
+        know_info_str = f"# Known information:\n## Current directory structure:\n{kb_tree}\n## Knowlege Metadata:\n{json.dumps(known_obj)}\n"
         return AgentPrompt(know_info_str)
 
+    async def _llm_read_article(self,knowledge_item:dict,full_path:str) -> ComputeTaskResult:
+        # Objectives:
+        #   Obtain better titles, abstracts, table of contents (if necessary), tags
+        #   Determine the appropriate place to put it (in line with the organization's goals)
+        # Known information:   
+        #   The reason why the target service's learn_prompt is being sorted
+        #   Summary of the organization's work (if any)
+        #   The current structure of the knowledge base (note the size control) gen_kb_tree_prompt (when empty, LLM should generate an appropriate initial directory structure)
+        #   Original path, current title, abstract, table of contents
 
+        # Sorting long files (general tricks)
+        #   Indicate that the input is part of the content, let LLM generate intermediate results for the task
+        #   Enter the content in sequence, when the last content block is input, LLM gets the result
 
         
-
-
-    async def _llm_read_article(self,knowledge_item:dict) -> ComputeTaskResult:
-        #目标：
-        #   得到更好的标题，摘要，目录 （如有必要）,tags
-        #   应放的合适的位置 （结合组织的目标）
-        #已知信息：   
-        #   整理是为什么目标服务的 learn_prompt
-        #   组织的工作总结（如有）
-        #   现在知识库的结构（注意大小控制）gen_kb_tree_prompt (当为空的时候应该让LLM生成一个合适的初始目录结构)
-        #   原始路径，现在标题，摘要，目录
-
-
-        # 整理长文件（通用技巧）
-        #   告诉输入的是部分内容，让LLM为任务产生中间结果
-        #   依次输入内容，在最后一个内容块输入时，LLM得到结果
-        
         #full_content = item.get_article_full_content()
         workspace = self.get_workspace_by_msg(None)
-        full_content = await workspace.load_knowledge_content(knowledge_item["hash"])
+        full_content = await workspace.load_knowledge_content(full_path)
         if full_content is None:
-            return 
+            return None
+        
+        if len(full_content) < 16:
+            logger.warning(f"llm_read_article: article {knowledge_item['path']} is too short,just read summary!")
+            return None 
+        
         full_content_len = self.token_len(full_content)
         if full_content_len < self.get_llm_learn_token_limit():
             
@@ -1021,9 +1062,9 @@ class AIAgent:
             prompt.append(known_info_prompt)
             content_prompt = AgentPrompt(full_content)
             prompt.append(content_prompt)
-  
+            env_functions = None
-            env_functions = workspace.get_knowledge_base_ai_functions()
+            #env_functions,function_len = workspace.get_knowledge_base_ai_functions()
-            task_result:ComputeTaskResult = await self._do_llm_complection(prompt,env_functions)
+            task_result:ComputeTaskResult = await self._do_llm_complection(prompt,env_functions,None,True)
             if task_result.result_code != ComputeTaskResultCode.OK:
                 result_obj = {}
                 result_obj["error_str"] = task_result.error_str
@@ -1033,11 +1074,42 @@ class AIAgent:
             return result_obj
 
         else:
-            logger.warning(f"llm_read_article: article {knowledge_item['path']} is too long,just read summary!")
+            logger.warning(f"llm_read_article: article {full_path} use LLM loop learn!")
+            pos = 0
+            read_len = int(self.get_llm_learn_token_limit() * 1.5)
+
+            temp_meta_data = {}
+            is_final = False
+            while pos < full_content_len:
+                _content = full_content[pos:pos+read_len]
+                if len(_content) < read_len:
+                    # last chunk
+                    is_final = True
+                    part_content = f"<<Final Part:start at {pos}>>\n{_content}"
+                else:
+                    part_content = f"<<Part:start at {pos}>>\n{_content}"
+                pos = pos + read_len
+
+                prompt = self.get_learn_prompt()
+                known_info_prompt = await self.gen_known_info_for_knowledge_prompt(knowledge_item,temp_meta_data)
+                prompt.append(known_info_prompt)
+                content_prompt = AgentPrompt(part_content)
+                prompt.append(content_prompt)
+                env_functions = None
+                #env_functions,function_len = workspace.get_knowledge_base_ai_functions()
+                task_result:ComputeTaskResult = await self._do_llm_complection(prompt,env_functions,None,True)
+                if task_result.result_code != ComputeTaskResultCode.OK:
                     result_obj = {}
-            result_obj["error_str"] = f"llm_read_article: article {knowledge_item['path']} is too long,just read summary!"
+                    result_obj["error_str"] = task_result.error_str
                     return result_obj
             
+                result_obj = json.loads(task_result.result_str)
+                temp_meta_data = result_obj.get("metadata")
+                if is_final:
+                    return result_obj
+
+            return None
+            
 
     async def do_self_think(self):
         session_id_list = AIChatSession.list_session(self.agent_id,self.chat_db)

src/aios_kernel/simple_kb_db.py

@@ -142,7 +142,7 @@ class SimpleKnowledgeDB:
 
         title = llm_result.get("title", "")
         summary = llm_result.get("summary", "")
-        catalogs = json.dumps(llm_result.get("catalogs", []))
+        catalogs = json.dumps(llm_result.get("catalogs", {}))
         tags = ','.join(llm_result.get("tags", []))
 
         cursor.execute('''
@@ -188,12 +188,15 @@ class SimpleKnowledgeDB:
             return None
         doc_path = row2[0]
     
+
         return {
             "full_path": doc_path,
             "title": row[0],
             "summary": row[1],
-            "catalogs": json.loads(row[2]),
+            "catalogs": row[2],
-            "tags": row[3].split(","),
+            "tags": row[3],
+            "llm_title" : row[4],
+            "llm_summary" : row[5],
         }
 
     def get_knowledge_without_llm_title(self,limit:int=16) -> List[str]:

src/aios_kernel/workspace_env.py

@@ -190,10 +190,13 @@ class WorkspaceEnvironment(Environment):
     # operation or inner_function   
     async def symlink(self,path:str,target:str) -> str:
         try:
-            file_path = self.root_path + path
+            #file_path = self.root_path + path
             target_path = self.root_path + target
-            os.symlink(file_path,target_path)
+            dir_path = os.path.dirname(target_path)
+            os.makedirs(dir_path,exist_ok=True)
+            os.symlink(path,target_path)
         except Exception as e:
+            logger.error("symlink failed:%s",e)
             return str(e)
         
         return None
@@ -435,18 +438,30 @@ class WorkspaceEnvironment(Environment):
 
     # knowledge base system
     def get_knowledge_base_ai_functions(self):
-        func_result = {}
+        all_inner_function = []
 
-        func_result["get_knowledge_catalog"] = SimpleAIFunction("get_knowledge_catalog","get knowledge catalog in tree format",
+        all_inner_function.append(SimpleAIFunction("get_knowledge_catalog","get knowledge catalog in tree format",
                                                                 self.get_knowledege_catalog,
-                                                                {"path":f"catalog path,none is /","depth":"max depth of catalog tree,default is 4"})
+                                                                {"path":f"catalog path,none is /","depth":"max depth of catalog tree,default is 4"}))
-        func_result["get_knowledge"] = SimpleAIFunction("get_knowledge","get knowledge metadata",
+        all_inner_function.append(SimpleAIFunction("get_knowledge","get knowledge metadata",
                                                                 self.get_knowledge,
-                                                                {"path":f"knowledge path"})
+                                                                {"path":f"knowledge path"}))
-        func_result["load_knowledge_content"] = SimpleAIFunction("load_knowledge_content","load knowledge content",
+        all_inner_function.append(SimpleAIFunction("load_knowledge_content","load knowledge content",
                                                                  self.load_knowledge_content,
-                                                                 {"path":f"knowledge path","pos":"start position of content","length":"length of content"})
+                                                                  {"path":f"knowledge path","pos":"start position of content","length":"length of content"}))
-        return func_result
+        result_func = []
+        result_len = 0
+        for inner_func in all_inner_function:
+            func_name = inner_func.get_name()
+
+            this_func = {}
+            this_func["name"] = func_name
+            this_func["description"] = inner_func.get_description()
+            this_func["parameters"] = inner_func.get_parameters()
+            result_len += len(json.dumps(this_func)) / 4
+            result_func.append(this_func)
+
+        return result_func,result_len                                                        
 
     async def get_knowledege_catalog(self,path:str=None,only_dir =True,max_depth:int=5)->str:
         if path:
@@ -499,19 +514,24 @@ class WorkspaceEnvironment(Environment):
         
         return "not found"
 
-    async def load_knowledge_content(self,path:str,pos:int=0,length:int=0) -> str:
+    async def load_knowledge_content(self,path:str,pos:int=0,length:int=None) -> str:
-        full_path = f"{self.root_path}/knowledge/{path}"
+        if path.endswith("pdf"):
-        if os.islink(full_path):
-            org_path = os.readlink(full_path)
-            if full_path.endswith("pdf"):
             logger.info("load_knowledge_content:pdf")
-                return "pdf is not support now!"
+            dir_path = os.path.dirname(path)
+            base_name = os.path.basename(path)
+            text_content_path = f"{dir_path}/.{base_name}.txt"
+            if os.path.exists(text_content_path) is False:
+                return None
+            async with aiofiles.open(path, mode='r', encoding=cur_encode) as f:
+                await f.seek(pos)
+                content = await f.read(length)
+                return content
         else:
-                async with aiofiles.open(full_path,'rb') as f:
+            async with aiofiles.open(path,'rb') as f:
-                    cur_encode = chardet.detect(f.read(1024))['encoding']
+                cur_encode = chardet.detect(await f.read(1024))['encoding']
 
-                async with aiofiles.open(full_path, mode='r', encoding=cur_encode) as f:
+            async with aiofiles.open(path, mode='r', encoding=cur_encode) as f:
-                    f.seek(pos)
+                await f.seek(pos)
                 content = await f.read(length)
                 return content
 
@@ -550,22 +570,38 @@ class WorkspaceEnvironment(Environment):
         return
 
     def _parse_pdf(self,doc_path:str):
-        
         metadata = {}
         with open(doc_path, 'rb') as file:
             reader = PyPDF2.PdfReader(file)
+            try:
                 doc_info = reader.metadata
                 if doc_info:
                     if doc_info.title:
                         metadata["title"] = doc_info.title
                     if doc_info.author:
                         metadata["authors"] = doc_info.author
+            except Exception as e:
+                logger.warn("parse pdf metadata failed:%s",e)
 
+            dir_path = os.path.dirname(doc_path)
+            base_name = os.path.basename(doc_path)
+            text_content_path = f"{dir_path}/.{base_name}.txt"
+            full_text = ""
+
+            for page in reader.pages:
+                text = page.extract_text()
+                full_text += text
+            with open(text_content_path, 'w', encoding='utf-8') as f:
+                f.write(full_text)
+
+            try:
                 bookmarks = reader.outline
                 if bookmarks:
                     catalogs = []
                     self._parse_pdf_bookmarks(bookmarks,catalogs)
                     metadata["catalogs"] = json.dumps(catalogs)
+            except Exception as e:
+                logger.warn("parse pdf bookmarks failed:%s",e)
 
         return metadata
 
@@ -612,11 +648,14 @@ class WorkspaceEnvironment(Environment):
 
         if meta_data.get("title"):
             title = meta_data["title"]
-        
+        logger.info("parse document %s!",doc_path)
         return hash_result,title,meta_data
 
 
     def _support_file(self,file_name:str) -> bool:
+        if file_name.startswith("."):
+            return False
+        
         if file_name.endswith(".pdf"):
             return True
         if file_name.endswith(".md"):