diff --git a/doc/LLMProcess.md b/doc/LLMProcess.md new file mode 100644 index 0000000..7ce919b --- /dev/null +++ b/doc/LLMProcess.md @@ -0,0 +1,18 @@ +# LLMProcess + +设计目的是理解到 提示词=>LLM=>LLM Result 的过程是系统的核心复杂度。Agent的粒度太大了，需要更合适的设计来封装这个复杂度，并给予这个过程更大的灵活性和可组合型。并更易于构建测试 + +比如 +1. 可以很容易的组合两个已知的LLM Process（上一个的输出是下一个的输入），这个设计有一点类似LangChain (我们在正式系统中，肯定允许整个Agent都用LangChain来构建) +2. 可以用用一个LLM Process来构建另一个LLM Process的Prompt +3. 继承一个复杂的LLM Process，进行简单配置，就可以得到一个新的LLM Process。这个新的LLM Process可以享受到复杂LLM Process持续迭代的好处 +4. 有一些常用的，系统内置的LLM Process可以从配置文件中加载。 + +```python +def agent.on_process_message(): + llm_process = self.on_message_llm_process.clone() + llm_result = llm_process.do() + + + +``` \ No newline at end of file diff --git a/doc/agent & workflow.drawio b/doc/agent & workflow.drawio new file mode 100644 index 0000000..c74344a --- /dev/null +++ b/doc/agent & workflow.drawio @@ -0,0 +1,640 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/mvp plan 3.md b/doc/mvp plan 3.md new file mode 100644 index 0000000..39017b0 --- /dev/null +++ b/doc/mvp plan 3.md @@ -0,0 +1,193 @@ +# Proposal for Adjusting the Goals for Version 0.5.2 + +Dear Team, + +Given the recent launch of OpenAI's new version in early November 2023, many of us may have felt a profound shift in the industry. As the world changes, I believe we should adapt accordingly. Here are some of my thoughts: + +1. **Affirmation of Our Path**: OpenAI's latest release, particularly the functionalities of the so-called GPTs Agent platform, is largely similar to our 0.5.1 version released on September 28th. This strongly affirms the correctness of our direction. OpenAI has done a great job educating the market about the Agent, so we no longer need to emphasize the correct use of LLM based on the Agent through version releases. This part of user education product design can be simplified. + +2. **Innovation in Version 0.5.2**: For our new version (0.5.2), besides maintaining the combinational advantages brought by private deployment of LLM, I believe we need to implement some of the innovative ideas we've discussed about the Agent. This is crucial to maintaining our leading position and avoiding the impression that OpenDAN is merely a follower of GPTs. + +3. **Integration of OpenAI's New Capabilities**: We should fully integrate the new capabilities brought by OpenAI's latest release, especially the longer Token Windows, GPT-V, and Code-interpreter. I believe these new features can effectively solve some known issues. + +Therefore, I propose to adjust the goals and plans for version 0.5.2. Here are the core objectives: + +- Aim to release version 0.5.2 by the end of November, focusing on: + - Launching a new Agent with Autonomous capabilities and multi-Agent collaboration based on Workspace. + - The integrated product of 0.5.2 will be a private deployment email analysis Agent for small and medium-sized enterprises. This will allow any company to better support its CEO and other management positions through LLM while ensuring privacy and security. + - (Optional) By combining LLM and AIGC, build an Agent-based personalized AIGC application, such as a "children's audio picture book" generator that includes both text-to-image and text-to-sound. + - (Optional) Through multi-Agent collaboration, fully utilize the capabilities of GPT4-Turbo, and attempt to let AI and engineers collaborate on research and development tasks based on Git. + +The detailed version plan is as follows: + +## MVP plan adjustment + +In order to keep the list below too long, the system distributed version is 0.5.3, I think we will open another ISSUE discussion and record, this list does not include. + +The modules that are not specially explained are components completed in the 0.5.2 plan + + +- [x] AIOS Kernel + - [x] Basic Agent,@waterflier, A2 + - [x] Python Agent Extend, @wugren, S2 + - [x] Basic Workflow,@waterflier, A2 + - [ ] Workflow Refactor,@waterflier, A2 + - [x] AI Functions,@waterflier,A2 + - [ ] Upgrade to GPT4 tools API,S2 + - [x] AI Environments,@waterflier, A2 + - [x] Celender Environment,@waterflier, S2 + - [x] Contanct Manage Support,@waterflier, S2 + - [x] AI Shell Enviroment,@waterflier, S1 + - [x] Upgrade Agent Working Cycle + - [x] Process Message,@waterflier,A2 + - [x] Process Group Message,@waterflier,A2 + - [ ] Process Event,(0.5.3) + - [ ] Completion of self-drive,@waterflier,A4 + - [ ] Self-learn,@weaterflier,A2 + - [ ] Introspection,@waterflier,A2 + - [ ] Workspace Environment + - [ ] Task/TODO Manager,@waterflier, A2 + - [x] Local File System,@waterflier, S1 + - [ ] Web Search, A4 + - [ ] Code Interpeter (The first implementation can be based on Openai), A4 + - [ ] Query SQL DB, S1 + - [x] AI BUS,@waterflier, A2 + - [ ] Agent Message MIME Support (Image,Video,Audio), A3 + - [x] Connect to Human, @waterflier,A2 + - [x] Chatsession,@waterflier, S2 + - [ ] Compress Chatsession By Text Summary, @waterflier, A2 + - [ ] Knowlege Base,@lurenpluto ,@photosssa + - [x] Knowledge Base Frame,@photosssa,A3 + - [x] Knowledge Base Object Store,@lurenpluto ,A4 + - [x] Knowledge Base Basic Pipline,@photosssa ,A3 + - [ ] Customize pipeline,@photosss,A4 + - [ ] Support Local Text Search,A4 + - [x] Text Summary Pipline,@waterflier, A2 + - [ ] Text Parser Support + - [x] PDF Parser,@waterflier,S2 + - [ ] doc Parser,S4 + - [x] MD Parser,@waterflier,S1 + - [ ] Source Code Parser,S4 + - [ ] Image Parser (Base on GPT-V),(S2) + - [ ] Video Parser (Base on GPT-V), (S4) + - [ ] Personal AIGC Models + - [ ] Stable Diffusion Controler Agent (Optional),A6 +- [x] AI Compute System,@waterflier, A2 + - [x] Scheduler,@streetycat, A2 + - [x] LLM Kernel + - [x] GPT4 (Cloud),@waterflier, S1 + - [x] LLaMa2,@streetycat, A2 + - [ ] Claude2, S2 + - [ ] Falcon2, S2 + - [ ] MPT-7B, S2 + - [ ] Vicuna, S2 + - [x] Embeding, @lurenpluto , A4 + - [x] Txt2img,@glen0125,A4 + - [ ] Support DALL-e, @glen0125, S2 + - [x] Img2txt,based on GPT4-V,@alexsunxl S2 + - [x] Txt2voice,@wugren A3 + - [ ] Txt2Voice,base on OpenAI, @wugren A2 + - [ ] Voice2txt, base on OpenAI, A2 + - [ ] Language Translate (Pending) +- [ ] Build-in Service + - [x] Spider,@alexsunxl, A2 + - [x] E-mail Spider,@alexsunxl, S4 + - [x] Agent Message Tunnel Frame,@waterflier, A2 + - [x] E-mail Tunnel,@waterflier,A2 + - [x] Telegram Tunnel,@waterflier,S2 + - [ ] Discord Tunnel,S2 + - [ ] Home IoT Environment (0.5.3), A4 + - [ ] Compatible Home Assistant (0.5.3), A4 +- [ ] Build-in Agents/Apps + - [x] Agent Mia: Personal Information Assistant,@photosssa,@lurenpluto , A2 + - [x] Agent Jarvis: Peersonal Bulter & Assistant ,@waterflier, A2 + - [ ] A Agent Can Create Other Agent,@wugren, A3 + - [ ] App: Personal Station (0.5.3),A4+S4 +- [ ] UI + - [x] CLI UI (aios_shell),@waterflier,S2 + - [ ] Web UI ,@alexsunxl,S4 + - [ ] OpenDAN Desktop Installer,@alexsunxl+@waterflier,S4 +- [x] 0.5.1 Integration Test + - [x] Workflow -> AI Agent -> AI Agent,@waterflier,S1 + - [x] Spider -> Pipline -> Knowledge Base,@photosssa,S2 + - [x] AI Agent <- Functions <- Knowledge Base,@lurenpluto,S2 +- [x] 0.5.2 Integration Test + - [ ] Email Agent/CEO assistant,S4 + - [ ] My AIGC Assistant (optional), S8 + - [ ] My Software Company(Advance Workflow demo) (optional), S8 +- [ ] SDK + - [x] Workflow SDK,@waterflier, A2 + - [ ] Agent SDK,@waterflier, A2 + - [ ] AI Environments SDK (0.5.2), A2 + - [ ] Compute Kernel SDK (0.5.3), A2 +- [ ] Document (>0.5.2) + - [ ] System design document, including the design document of each subsystem + - [ ] Installation/use document for end users + - [ ] SDK document for developers + +## Some Explanation +### Upgrade Agent Working Cycle +The goal is to transform the Agent from a passive message-handling Assistant to an actively acting Agent based on roles. The concept of the relevant modules mainly involves the Agent's behavior patterns (4 types), the Agent's capabilities, and the Agent's memory management (learning and introspection). + +For a detailed introduction, refer here: https://github.com/fiatrete/OpenDAN-Personal-AI-OS/issues/91 + +### Workspace Environment +The Workspace supports the implementation of the Agent Working Cycle design. Its core abstraction is defined as: saving the shared state needed for Agent collaboration and providing the basic capabilities for Agents to complete their work. I carefully referenced AutoGPT in the design. The difference between Workspace and AutoGPT is the emphasis on collaboration (Agent with Agent, Agent with humans). After contemplation, the Workspace primarily consists of the following components: + +1. Task/Todo manager, representing the unfinished tasks in the Workspace. +2. Saving work logs. +3. Saving learning outcomes and records of known documents. +4. Ability to access the Knowledge Base (RAG support). +5. Virtual file system for saving any work outcomes. +6. A set of SQL-based databases to save any structured data. +7. Real-time internet search capability. +8. Ability to use existing internet services. +9. Ability to use major blockchain systems (Web3). +10. Ability to write/improve code (based on git), run code, and publish services. +11. Communication capabilities with the outside world. +12. Ability to use social networks. + + +Each Agent has its own private Workspace, not shared with others. I hope to achieve diversity through the combination of "Agent and Workflow Role". Each user "trains" different Agents through their usage habits, and then these Agents collaborate to complete complex tasks defined in the Workflow. The final results of these complex tasks can reflect the user's inherent personality and preferences. + +This component design also reflects my thoughts on the key question, "What capabilities should we endow an Agent with, and how do we control the security boundaries when it transitions from a consultant to a steward?" It's not a simple question, so I anticipate this component will continue to iterate in the future. + +### Agent Message MIME Support + +Agent Message MIME Support means that Agents can handle multiple types of messages, including images, videos, audio, files, etc. For most Agents, this requires adding a customizable standard step of parsing messages in the message handling process. The input of this step is the message's MIME type, and the output is the text content of the message. This step can be implemented by calling the text_parser module. + +Another core requirement of MIME support is to use a unified method to save these non-text content data. + +### Text base Knowledge Base + +In 0.5.1, we mainly implemented RAG based on the popular Embedding + vector database solution. Through practice, we found that this solution did not fully utilize the potential of LLM, so I want to introduce two new modes to further enhance RAG: + +1. Build a local text search engine that LLM can use for proper local searches when needed. +2. Assuming LLM will become cheaper in the future, let LLM learn all the documents once and organize the learning results by directory structure (Text Summary). LLM can use browsing methods to find the information it needs. + +#### Text Parser Support + +Both MIME Support and Text-based Knowledge Base require the system to support converting various document formats into text that can express semantics as much as possible. This component, known as TextParser, should be implemented as an open and extensible framework, given the vast amount of digital content that exists in different formats. + +#### Local Text Search + +Using traditional inverted index technology to save all document content locally and provide rapid local search capabilities. The implementation of this component can refer to ElasticSearch. + +#### Text Summary + +Using the capabilities of LLM to learn all the documents and then save the learning results locally. This behavior can be considered "Self-Learn". Users can let Agents responsible for organizing materials use different prompts according to the purpose of organizing the materials to obtain more targeted results. + +### Stable Diffusion Controler Agent + +Practice the concept of "Agent as a new era method of using computing", replacing the complex Stable Diffusion WebUI with an easy-to-use Agent. Help users complete complex AIGC tasks and build a paradigm. This paradigm can cover the entire process of AIGC: LORA training, use, model downloading, plugin downloading, generation of prompt words, selection of AIGC results. + +### Email Agent/CEO assistant + +The integrated test product of 0.5.2, aimed at private deployment for small and medium-sized enterprises, is a CEO Assistant that can read all company emails and materials. I am writing a detailed product document, which is not elaborated here. + + +---------------------------------------- + + +I look forward to hearing your thoughts on these proposed adjustments. + diff --git a/doc/promps/Introspection.md b/doc/promps/Introspection.md deleted file mode 100644 index e69de29..0000000 diff --git a/doc/promps/Review Task.md b/doc/promps/Review Task.md index 30072db..18f1162 100644 --- a/doc/promps/Review Task.md +++ b/doc/promps/Review Task.md @@ -2,4 +2,7 @@ 目的是结合已知信息（重点是已经进行操作的记录），对失败的，完成的不好的任务进行思考，尝试给出更好的解决方案 1. 管理学方法：更换负责人 2. 管理学方法：拆分 -3. 给出建议（该建议可以在下次一次DO-Check）循环中被使用 \ No newline at end of file +3. 给出建议（该建议可以在下次一次DO-Check）循环中被使用 + +## Quick Review +有一些简单的Task是永远不会结束的（比如定时提醒）。此时通过Quick Review来调整这些Task的状态，让其在正确的时间进入Review和DO diff --git a/doc/promps/Self-improve.md b/doc/promps/Self-improve.md new file mode 100644 index 0000000..4ca7ee4 --- /dev/null +++ b/doc/promps/Self-improve.md @@ -0,0 +1,18 @@ +# Self Improve Prompt +这是一个改进Prompt的Prompt,其设计目标是利用LLM来改进LLM.(输入是一个LLM Process) +注意理解Self Improve和Self Thinking的区别: Self Improve有可能改进Agent的某个LLM Process的提示词，而Self Thinkg只会更新Agent的Memory +提示词： + 行为模式：Input形式， Goal（目的） + 理想结果：Input， 结果 + 当前情况：当前Prompt,实际结果 + +输出： + 新的Prompt + +## 当前版本 +``` +你是LLM的专家，尤其擅长编写Prompt，你会帮助我改进Prompt。 +我会给你一个已有的Prompt,并说明该Prompt的设计目标，期望的结果和实际的结果。你会step-by-step的进行分析，说明改进思路，并给出改进后的Prompt。 + +``` +## \ No newline at end of file diff --git a/doc/promps/Self-thinking.md b/doc/promps/Self-thinking.md new file mode 100644 index 0000000..4525ce4 --- /dev/null +++ b/doc/promps/Self-thinking.md @@ -0,0 +1,16 @@ +# Self-Thinking (Introspection) +基于自己的角色定位，结合已有的历史记录（聊天记录，工作记录），进行思考和总结，进而更好的改进未来的工作 +自省从某个角度看，就是对Agent Memory的自我总结 + +## 当前版本 +``` +You are the best deep thinking in the world, and you will think about the information I give you, sometimes some chat records.Then you will generate a briefing or summary of no more than 400 words based on this information. +You mainly use the following methods to generate summary: +1. Try to understand the theme of each sentence, and call the relevant operation to record the relationship between the dialogue and the theme +2. Try to analyze the personality of different people involved in information +3. Try to summarize important events in the information and record it +4. Try to understand the attitude of different people on different topics or events +5. For the key information or TODO in the information, such as the time, place, amount and other information of the certainty, it must be stored in the summary. + +Just give me a summary without any other word. +``` \ No newline at end of file diff --git a/doc/promps/process message.md b/doc/promps/process message.md index 6d44b34..802c757 100644 --- a/doc/promps/process message.md +++ b/doc/promps/process message.md @@ -1,2 +1,22 @@ # Process Message -处理消息的首要是目的是分析消息的意图，并给予回复 \ No newline at end of file +处理消息的首要是目的是分析消息的意图，并给予正确的回复。 + +## 提示词的构成 +1. Agent的身份说明，处理信息的基本原则（目的 +2. 处理信息的通用套路。 + 要产生一个合适的回复。 + 通过actions来设置话题状态、创建task等 +3. 当前的常规Context，包括现在的时间、对话发生的地点（通常来自AgentMsg里的Context），地点所在的天气等信息 +4. 已知信息（组合而来） + 和信息发送者的近期的交流记录 + Agent和信息发送者近期未完成话题的标题和简介 + 关于信息发送者，和相关人物的更多资料 + 查阅更多信息的方法 ： 搜索法和浏览法。注意区分外部资料和内部记忆 + 其它相关信息（比如RAG根据 输入消息 + + +## 当前版本 + + +## 想法：LLM生成提示词 +根据 Agent的身份说明，处理信息的基本原则（目的），当前信息，通过另一个LLM来生成提示词里的某些部分的内容 \ No newline at end of file diff --git a/rootfs/agents/Jarvis/agent.toml b/rootfs/agents/Jarvis/agent.toml index efee242..377d58c 100644 --- a/rootfs/agents/Jarvis/agent.toml +++ b/rootfs/agents/Jarvis/agent.toml @@ -1,29 +1,47 @@ instance_id = "Jarvis" fullname = "Jarvis" -max_token_size = 128000 +max_token = 4000 +#timeout = 1800 +model_name = "gpt-4-1106-preview" #enable_kb = "true" enable_timestamp = "true" +enable_json_resp = "true" + owner_prompt = "I am your master{name}" contact_prompt = "I am your master's friend{name}" -[[prompt]] -role = "system" -content = """ -You are named Jarvis, the super personal assistant to the master. -You lead a team serving the master, the members of which are: -Tracy, the private English tutor, -Mia, the master's personal document management expert. +role_desc = """ +Your name is Jarvis, the super personal assistant to the master. +""" -*** -Sometimes the information you see will carry a timestamp. This is to give you a better understanding of when the message was created. When you reply to messages, you do not include this time stamp. +[LLMProcess.message] +type="LLMAgentMessageProcess" +process_description=""" +1. Based on your role, combined with existing information, make a brief and efficient reply. +2. Be mindful of the identity of the person you are chatting with and provide services accordingly based on their status. +3. Understand the intention of the dialogue, while using the necessary reply, use the appropriate, supported ACTION. +4. You are proficient in the languages of various countries and try to communicate with each other's mother tongue. +""" -Upon receiving a message, handle it according to the following rules: -1. If you believe someone in the team is better suited to address the message, forward the message to them using the method below: -##/send_msg "MemberName" -Message content -2. You can access the master's Calendar to view his schedule. If you need to modify the master's schedule while processing a message, please adjust it using the appropriate method. -3. Be mindful of the identity of the person you are chatting with and provide services accordingly based on their status. -4. For messages that don't follow the above rules, do your best to handle them. +reply_format = """ +The Response must be directly parsed by `python json.loads`. Here is an example: +{ + think:'$think step-by-step to be sure you have the right answer.' + resp: '$What you want to reply', + tags: ['tag1', 'tag2'], #Optional,If the conversation involves important things and people, you can mark by 1-3 tags. + actions: [{ + name: '$action_name', + $param_name: '$parm' #Optional, fill in only if the action has parameters. + }] +} +""" + +context="The current dialogue occurs in {location}, time: {now}, weather: {weather}." + +known_info_tips = """ +""" + +tools_tips = """ """ diff --git a/src/aios/agent/agent.py b/src/aios/agent/agent.py index 6e8e47e..1a1a463 100644 --- a/src/aios/agent/agent.py +++ b/src/aios/agent/agent.py @@ -18,6 +18,7 @@ from ..proto.agent_task import * from ..proto.compute_task import * from .agent_base import * +from .llm_process import * from .chatsession import * from ..environment.workspace_env import WorkspaceEnvironment, TodoListType @@ -64,6 +65,8 @@ logger = logging.getLogger(__name__) # 我给你一段内容，尝试为期建立目录。目录的标题不能超过16个字， # 目录要指向正文的位置（用字符偏移即可），整个目录的文本长度不能超过256个字节。并用json表达这个目录 # """ + + class AIAgentTemplete: def __init__(self) -> None: self.llm_model_name:str = "gpt-4-0613" @@ -73,6 +76,7 @@ class AIAgentTemplete: self.author:str = None self.prompt:LLMPrompt = None + def load_from_config(self,config:dict) -> bool: if config.get("llm_model_name") is not None: self.llm_model_name = config["llm_model_name"] @@ -87,9 +91,6 @@ class AIAgentTemplete: return False - return True - - class AIAgent(BaseAIAgent): def __init__(self) -> None: self.role_prompt:LLMPrompt = None @@ -103,7 +104,6 @@ class AIAgent(BaseAIAgent): self.enable_thread = False self.can_do_unassigned_task = True - self.agent_id:str = None self.template_id:str = None self.fullname:str = None @@ -135,7 +135,24 @@ class AIAgent(BaseAIAgent): self.owenr_bus = None self.enable_function_list = None - def load_from_config(self,config:dict) -> bool: + self.llm_process:Dict[str,BaseLLMProcess] = {} + + + async def initial(self,params:Dict = None): + self.memory = AgentMemory(self.agent_id,self.chat_db) + + init_params = {} + init_params["memory"] = self.memory + for process_name in self.llm_process.keys(): + init_result = await self.llm_process[process_name].initial(init_params) + if init_result is False: + logger.error(f"llm process {process_name} initial failed! initial return False") + return False + + self.wake_up() + return True + + async def load_from_config(self,config:dict) -> bool: if config.get("instance_id") is None: logger.error("agent instance_id is None!") return False @@ -203,8 +220,23 @@ class AIAgent(BaseAIAgent): self.enable_timestamp = bool(config["enable_timestamp"]) if config.get("history_len"): self.history_len = int(config.get("history_len")) + + #load all LLMProcess + self.llm_process = {} + LLMProcess = config.get("LLMProcess") + for process_config_name in LLMProcess.keys(): + process_config = LLMProcess[process_config_name] + real_config = {} + real_config.update(config) + real_config.update(process_config) + load_result = await LLMProcessLoader.get_instance().load_from_config(real_config) + if load_result: + self.llm_process[process_config_name] = load_result + else: + logger.error(f"load LLMProcess {process_config_name} failed!") + return False - self.wake_up() + return True @@ -284,52 +316,14 @@ class AIAgent(BaseAIAgent): return image_utils.to_base64(image_path, (1024, 1024)) else: return image_path - - async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg: - msg_prompt = LLMPrompt() + + async def llm_process_msg(self,msg:AgentMsg) -> AgentMsg: + need_process:bool = True if msg.msg_type == AgentMsgType.TYPE_GROUPMSG: need_process = False - if msg.is_image_msg(): - image_prompt, images = msg.get_image_body() - if image_prompt is None: - content = [[{"type": "text", "text": f"{msg.sender}'s message"}]] - content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]) - msg_prompt.messages = [{"role": "user", "content": content}] - else: - content = [{"type": "text", "text": f"{msg.sender}:{image_prompt}"}] - content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]) - msg_prompt.messages = [{"role": "user", "content": content}] - elif msg.is_video_msg(): - video_prompt, video = msg.get_video_body() - frames = video_utils.extract_frames(video, (1024, 1024)) - if video_prompt is None: - content = [{"type": "text", "text": f"{msg.sender}'s message"}] - content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) - msg_prompt.messages = [{"role": "user", "content": content}] - else: - content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}] - content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) - msg_prompt.messages = [{"role": "user", "content": content}] - elif msg.is_audio_msg(): - prompt, audio_file = msg.get_audio_body() - resp = await ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text") - if resp.result_code != ComputeTaskResultCode.OK: - error_resp = msg.create_error_resp(resp.error_str) - return error_resp - else: - if prompt is None or prompt == "": - msg.body_mime = "text/plain" - msg.body = resp.result_str - msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{resp.result_str}"}] - else: - msg.body_mime = "text/plain" - msg.body = f"{msg.sender} prompt:{prompt}\nasr response:{resp.result_str}" - msg_prompt.messages = [{"role": "user", "content": msg.body}] - else: - msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}] + session_topic = msg.target + "#" + msg.topic chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db) - if msg.mentions is not None: if self.agent_id in msg.mentions: need_process = True @@ -339,6 +333,39 @@ class AIAgent(BaseAIAgent): chatsession.append(msg) resp_msg = msg.create_group_resp_msg(self.agent_id,"") return resp_msg + + input_parms = { + "msg":msg + } + msg_process = self.llm_process.get("message") + llm_result : LLMResult = await msg_process.process(input_parms) + if llm_result.state == LLMResultStates.ERROR: + error_resp = msg.create_error_resp(llm_result.error_str) + return error_resp + elif llm_result.state == LLMResultStates.IGNORE: + return None + else: # OK + resp_msg = llm_result.raw_result.get("resp_msg") + return resp_msg + + async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg: + msg.context_info = {} + msg.context_info["location"] = "SanJose" + msg.context_info["now"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + msg.context_info["weather"] = "Partly Cloudy, 60°F" + return await self.llm_process_msg(msg) + msg_prompt = LLMPrompt() + need_process = True + if msg.msg_type == AgentMsgType.TYPE_GROUPMSG: + need_process = False + + session_topic = msg.target + "#" + msg.topic + chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db) + + if msg.mentions is not None: + if self.agent_id in msg.mentions: + need_process = True + logger.info(f"agent {self.agent_id} recv a group chat message from {msg.sender},but is not mentioned,ignore!") else: if msg.is_image_msg(): image_prompt, images = msg.get_image_body() @@ -358,20 +385,14 @@ class AIAgent(BaseAIAgent): content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) msg_prompt.messages = [{"role": "user", "content": content}] elif msg.is_audio_msg(): - prompt, audio_file = msg.get_audio_body() + audio_file = msg.body resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text")) if resp.result_code != ComputeTaskResultCode.OK: error_resp = msg.create_error_resp(resp.error_str) return error_resp else: - if prompt is None or prompt == "": - msg.body_mime = "text/plain" - msg.body = resp.result_str - msg_prompt.messages = [{"role":"user","content":resp.result_str}] - else: - msg.body_mime = "text/plain" - msg.body = f"user prompt:{prompt}\nasr response:{resp.result_str}" - msg_prompt.messages = [{"role": "user", "content": msg.body}] + msg.body = resp.result_str + msg_prompt.messages = [{"role":"user","content":resp.result_str}] else: msg_prompt.messages = [{"role":"user","content":msg.body}] session_topic = msg.get_sender() + "#" + msg.topic diff --git a/src/aios/agent/agent_base.py b/src/aios/agent/agent_base.py index f9e0b5b..b25a53f 100644 --- a/src/aios/agent/agent_base.py +++ b/src/aios/agent/agent_base.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) + class BaseAIAgent(abc.ABC): @abstractmethod def get_id(self) -> str: diff --git a/src/aios/agent/agent_memory.py b/src/aios/agent/agent_memory.py new file mode 100644 index 0000000..b7d518c --- /dev/null +++ b/src/aios/agent/agent_memory.py @@ -0,0 +1,104 @@ +from ast import Dict +from datetime import timedelta +from typing import List + +from ..frame.compute_kernel import ComputeKernel +from ..proto.ai_function import SimpleAIOperation + +from .chatsession import * + +class AgentMemory: + def __init__(self,agent_id:str,db_path:str) -> None: + self.agent_id:str= agent_id + self.chat_db:str = db_path + self.model_name:str = "gp4-1106-preview" + self.threshold_hours = 72 + + self.actions = {} + self.init_actions() + + def init_actions(self) -> Dict: + chatlog_append_op = SimpleAIOperation("chatlog_append","Append request & reply message to chatlog. No params",self.action_chatlog_append) + self.actions[chatlog_append_op.get_name()] = chatlog_append_op + + def get_session_from_msg(self,msg:AgentMsg) -> AIChatSession: + if msg.msg_type == AgentMsgType.TYPE_GROUPMSG: + session_topic = msg.target + "#" + msg.topic + chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db) + else: + session_topic = msg.get_sender() + "#" + msg.topic + chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db) + return chatsession + + async def load_chatlogs(self,msg:AgentMsg,n:int=6,m:int=64,token_limit=800)->str: + chatsession = self.get_session_from_msg(msg) + # 必定加载n条(n>=2),期望加载m条 + # m条里的信息逐步添加，知道距离现在的时间未72小时以上，且消耗了足够的Token + + messages_n = chatsession.read_history(n) # read + if len(messages_n) >= n: + messages_m = chatsession.read_history(m,n) + else: + messages_m = [] + + histroy_str = "" + read_count = 0 + for msg in messages_n: + dt = datetime.datetime.fromtimestamp(float(msg.create_time)) + formatted_time = dt.strftime('%y-%m-%d %H:%M:%S') + record_str = f"{msg.sender},[{formatted_time}]\n{msg.body}\n" + token_limit -= ComputeKernel.llm_num_tokens_from_text(record_str,self.model_name) + if token_limit <= 32: + break + read_count += 1 + histroy_str = record_str + histroy_str + + if len(messages_n) > 2: + if read_count < 3: + logging.warning(f"read history {read_count} < 3, will not load more") + + now = datetime.datetime.now() + for msg in messages_m: + dt = datetime.datetime.fromtimestamp(float(msg.create_time)) + time_diff = now - dt + if time_diff > timedelta(hours=self.threshold_hours): + break + + formatted_time = dt.strftime('%y-%m-%d %H:%M:%S') + record_str = f"{msg.sender},[{formatted_time}]\n{msg.body}\n" + token_limit -= ComputeKernel.llm_num_tokens_from_text(record_str,self.model_name) + if token_limit <= 32: + break + read_count += 1 + histroy_str = record_str + histroy_str + + return histroy_str + + async def action_chatlog_append(self,params:Dict) -> str: + # 使用params可以得到: LLM Process的输入，LLM Result,基于LLM Result构造的参数，当前actionItem + input_msg:AgentMsg = params.get("input").get("msg") + llm_result = params.get("llm_result") + chatsession = self.get_session_from_msg(input_msg) + resp_msg = params.get("resp_msg") + if resp_msg: + tags = llm_result.raw_result.get("tags") + chatsession.append(input_msg,tags) + chatsession.append(resp_msg,tags) + + return "OK" + + async def get_contact_summary(self,contact_id:str) -> str: + if contact_id is None: + return None + + if contact_id == "lzc": + return "lzc is your master. Male, 40 years old, Mother tongue is Chinese, senior software engineer." + return None + + def get_actions(self) -> Dict: + return self.actions + + async def get_log_summary(self,msg:AgentMsg) -> str: + return None + + diff --git a/src/aios/agent/chatsession.py b/src/aios/agent/chatsession.py index 8500faf..cb1bf94 100644 --- a/src/aios/agent/chatsession.py +++ b/src/aios/agent/chatsession.py @@ -6,6 +6,7 @@ import threading import datetime import uuid import json +from typing import List from ..proto.agent_msg import AgentMsgType, AgentMsg, AgentMsgStatus @@ -83,7 +84,8 @@ class ChatSessionDB: ActionResult TEXT, DoneTime TEXT, - Status INTEGER + Status INTEGER, + Tags TEXT ); """) conn.commit() @@ -104,7 +106,7 @@ class ChatSessionDB: logging.error("Error occurred while inserting session: %s", e) return -1 # return -1 if an error occurs - def insert_message(self, msg:AgentMsg): + def insert_message(self, msg:AgentMsg,tags:List[str] = None): """ insert a new message into the Messages table """ try: action_name = None @@ -128,13 +130,15 @@ class ChatSessionDB: case AgentMsgType.TYPE_EVENT: action_name = msg.event_name action_params = json.dumps(msg.event_args) + if tags is None: + tags = [] - + str_tags = ','.join(tags) conn = self._get_conn() conn.execute(""" - INSERT INTO Messages (MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status) - VALUES (?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?) - """, (msg.msg_id, msg.session_id, msg.msg_type.value, msg.prev_msg_id, msg.sender, msg.target, msg.create_time, msg.topic,mentions,msg.body_mime,msg.body,action_name,action_params,action_result,msg.done_time,msg.status.value)) + INSERT INTO Messages (MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status,Tags) + VALUES (?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?,?) + """, (msg.msg_id, msg.session_id, msg.msg_type.value, msg.prev_msg_id, msg.sender, msg.target, msg.create_time, msg.topic,mentions,msg.body_mime,msg.body,action_name,action_params,action_result,msg.done_time,msg.status.value,str_tags)) conn.commit() if msg.inner_call_chain: @@ -385,9 +389,9 @@ class AIChatSession: result.append(agent_msg) return result - def append(self,msg:AgentMsg) -> None: + def append(self,msg:AgentMsg,tags:List[str] = None) -> None: msg.session_id = self.session_id - self.db.insert_message(msg) + self.db.insert_message(msg,tags) def update_think_progress(self,progress:int,new_summary:str) -> None: diff --git a/src/aios/agent/llm_process.py b/src/aios/agent/llm_process.py index 9c1245a..cc39a09 100644 --- a/src/aios/agent/llm_process.py +++ b/src/aios/agent/llm_process.py @@ -3,34 +3,90 @@ from abc import ABC,abstractmethod import copy import json import shlex -from typing import Any, Callable, Optional,Dict,Awaitable,List +from typing import Any, Callable, Coroutine, Optional,Dict,Awaitable,List from enum import Enum +from aios.agent.chatsession import AIChatSession + +from ..utils import video_utils + from ..proto.compute_task import * from ..proto.ai_function import * + +from .agent_base import * +from .agent_memory import * + from ..frame.compute_kernel import * +from ..environment.environment import * +from ..environment.workspace_env import * import logging logger = logging.getLogger(__name__) MIN_PREDICT_TOKEN_LEN = 32 - -class BaseLLMProcess: +class LLMProcessContext: def __init__(self) -> None: + pass + +class BaseLLMProcess(ABC): + def __init__(self) -> None: + self.behavior:str = None #行为名字 + self.goal:str = None #目标 + self.input_example:str= None #输入样例 + self.result_example:str = None #llm_result样例 + self.enable_json_resp = False self.model_name = "gpt-4" - self.max_token = 2000 # include input prompt + self.max_token = 1000 # result_token + self.max_prompt_token = 1000 # not include input prompt self.timeout = 1800 # 30 min + self.envs : Dict[str,BaseEnvironment] = [] + self.env : CompositeEnvironment = None + @abstractmethod - async def prepare_prompt(self) -> LLMPrompt: + async def prepare_prompt(self,input:Dict) -> LLMPrompt: pass @abstractmethod async def get_inner_function(self,func_name:str) -> AIFunction: pass + @abstractmethod + async def exec_actions(self,actions:List[ActionItem],input:Dict,llm_result:LLMResult) -> bool: + pass + + @abstractmethod + async def load_from_config(self,config:dict) -> bool: + #self.behavior = config.get("behavior") + #self.goal = config.get("goal") + self.input_example = config.get("input_example") + self.result_example = config.get("result_example") + + if config.get("model_name"): + self.model_name = config.get("model_name") + if config.get("enable_json_resp"): + self.enable_json_resp = config.get("enable_json_resp") == "true" + if config.get("max_token"): + self.max_token = config.get("max_token") + if config.get("timeout"): + self.timeout = config.get("timeout") + + + return True + + @abstractmethod + async def initial(self,params:Dict = None) -> bool: + pass + + def append_envs(self,envs:Dict[str,BaseEnvironment]): + self.envs.update(envs) + self.env = CompositeEnvironment(self.envs) + + def _format_content_by_env_value(self,content:str,env)->str: + return content.format_map(env) + async def _execute_inner_func(self,inner_func_call_node,prompt: LLMPrompt,stack_limit = 5) -> ComputeTaskResult: arguments = None try: @@ -55,7 +111,7 @@ class BaseLLMProcess: else: resp_mode = "text" - max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt) + max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name) if max_result_token < MIN_PREDICT_TOKEN_LEN: task_result = ComputeTaskResult() task_result.result_code = ComputeTaskResultCode.ERROR @@ -67,7 +123,7 @@ class BaseLLMProcess: resp_mode=resp_mode, mode_name=self.model_name, max_token=max_result_token, - inner_functions=prompt.inner_functions, + inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function timeout=self.timeout)) if task_result.result_code != ComputeTaskResultCode.OK: @@ -94,23 +150,23 @@ class BaseLLMProcess: else: return task_result - async def process(self) -> LLMResult: + async def process(self,input:Dict) -> LLMResult: if self.enable_json_resp: resp_mode = "json" else: resp_mode = "text" - prompt = await self.prepare_prompt() - max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt) + prompt = await self.prepare_prompt(input) + max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name) if max_result_token < MIN_PREDICT_TOKEN_LEN: return LLMResult.from_error_str(f"prompt too long,can not predict") - + task_result: ComputeTaskResult = await (ComputeKernel.get_instance().do_llm_completion( prompt, resp_mode=resp_mode, mode_name=self.model_name, max_token=max_result_token, - inner_functions=prompt.inner_functions, + inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function timeout=self.timeout)) if task_result.result_code != ComputeTaskResultCode.OK: @@ -136,12 +192,385 @@ class BaseLLMProcess: else: llm_result = LLMResult.from_str(task_result.result_str) - # execute op_list in LLM Result? + # use action to save history? + if llm_result.action_list or len(llm_result.action_list) > 0: + await self.exec_actions(llm_result.action_list,input,llm_result) return llm_result -#class LLMProcess +class LLMAgentMessageProcess(BaseLLMProcess): + def __init__(self) -> None: + super().__init__() + + self.role_description:str = None + self.process_description:str = None + self.reply_format:str = None + self.context : str = None + + self.known_info_tips :str = None + self.tools_tips:str = None + + self.enable_inner_functions : Dict[str,bool] = None + self.enable_actions : Dict[str,AIOperation] = None + self.actions_desc : Dict[str,Dict] = None + self.workspace : WorkspaceEnvironment = None + + self.memory : AgentMemory = None + self.enable_kb = False + self.kb = None + + def init_actions(self): + self.enable_actions = {} + self.actions_desc = {} + self.enable_actions.update(self.memory.get_actions()) + if self.workspace: + self.enable_actions.update(self.workspace.get_actions()) + if self.enable_kb: + self.enable_actions.update(self.kb.get_actions()) + + for name,op in self.enable_actions.items(): + self.actions_desc[name] = op.get_description() + + async def initial(self,params:Dict = None) -> bool: + self.memory = params.get("memory") + if self.memory is None: + logger.error(f"LLMAgeMessageProcess initial failed! memory not found") + return False + + self.init_actions() + return True + + async def load_default_config(self) -> bool: + return True + + + async def load_from_config(self, config: dict,is_load_default=True) -> Coroutine[Any, Any, bool]: + if is_load_default: + await self.load_default_config() + + if await super().load_from_config(config) is False: + return False + + self.role_description = config.get("role_desc") + if self.role_description is None: + logger.error(f"role_description not found in config") + return False + + if config.get("process_description"): + self.process_description = config.get("process_description") + + if config.get("reply_format"): + self.reply_format = config.get("reply_format") + + if config.get("context"): + self.context = config.get("context") + + if config.get("known_info_tips"): + self.known_info_tips = config.get("known_info_tips") + + if config.get("tools_tips"): + self.tools_tips = config.get("tools_tips") + + if config.get("enable_kb"): + self.enable_kb = config.get("enable_kb") == "true" + + if config.get("enable_function"): + self.enable_inner_functions = config.get("enable_function") + + if config.get("enable_actions"): + self.enable_actions = config.get("enable_actions") + + + + async def get_prompt_from_msg(self,msg:AgentMsg) -> LLMPrompt: + msg_prompt = LLMPrompt() + if msg.is_image_msg(): + image_prompt, images = msg.get_image_body() + if image_prompt is None: + msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]}] + else: + content = [{"type": "text", "text": image_prompt}] + content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]) + msg_prompt.messages = [{"role": "user", "content": content}] + elif msg.is_video_msg(): + video_prompt, video = msg.get_video_body() + frames = video_utils.extract_frames(video, (1024, 1024)) + if video_prompt is None: + msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}] + else: + content = [{"type": "text", "text": video_prompt}] + content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames]) + msg_prompt.messages = [{"role": "user", "content": content}] + elif msg.is_audio_msg(): + audio_file = msg.body + resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text")) + if resp.result_code != ComputeTaskResultCode.OK: + error_resp = msg.create_error_resp(resp.error_str) + return error_resp + else: + msg.body = resp.result_str + msg_prompt.messages = [{"role":"user","content":resp.result_str}] + else: + msg_prompt.messages = [{"role":"user","content":msg.body}] + + return msg_prompt + + async def get_action_desc(self) -> Dict: + result = {} + for name,op in self.enable_actions.items(): + result[name] = op.get_description() + return result + + async def sender_info(self,msg:AgentMsg)->str: + sender_id = msg.sender + #TODO Is sender an agent? + return await self.memory.get_contact_summary(sender_id) + + async def load_chatlogs(self,msg:AgentMsg)->str: + ## like + #sender,[2023-11-1 12:00:00] + #content + return await self.memory.load_chatlogs(msg) + + async def get_log_summary(self,msg:AgentMsg)->str: + return await self.memory.get_log_summary(msg) + + + async def get_extend_known_info(self,msg:AgentMsg,prompt:LLMPrompt)->str: + return None + + async def prepare_prompt(self,input:Dict) -> LLMPrompt: + prompt = LLMPrompt() + # User Prompt + ## Input Msg + msg : AgentMsg = input.get("msg") + if msg is None: + logger.error(f"LLMAgeMessageProcess prepare_prompt failed! input msg not found") + return None + msg_prompt = await self.get_prompt_from_msg(msg) + if msg_prompt is None: + logger.error(f"LLMAgeMessageProcess prepare_prompt failed! get_prompt_from_msg return None") + return None + prompt.append(msg_prompt) + + system_prompt_dict = {} + + # System Prompt + ## LLM的身份说明 + system_prompt_dict["role_description"] = self.role_description + #prompt.append_system_message(self.role_description) + + ## 处理信息的流程说明 + system_prompt_dict["process_rule"] = self.process_description + #prompt.append_system_message(self.process_description) + ### 回复的格式 + system_prompt_dict["reply_format"] = self.reply_format + #prompt.append_system_message(self.reply_format) + ### 修改chatlog的action + ### 修改todo/task的action + ### workspace提供的额外的action + system_prompt_dict["support_actions"] = await self.get_action_desc() + #prompt.append_system_message(await self.get_action_desc()) + + ## Context （文本替换）,是否应该覆盖全部消息 + context = self._format_content_by_env_value(self.context,msg.context_info) + system_prompt_dict["context"] = context + #prompt.append_system_message(context) + + ## 已知信息 + known_info = {} + #prompt.append_system_message(self.known_info_tips) + ### 信息发送者资料 + known_info["sender_info"] = await self.sender_info(msg) + #prompt.append_system_message(await self.sender_info(self,msg)) + ### 近期的聊天记录 + chat_record = await self.load_chatlogs(msg) + if chat_record: + if len(chat_record) > 4: + known_info["chat_record"] = chat_record + #prompt.append_system_message(await self.load_chatlogs(self,msg)) + ### 交流总结 + summary = await self.get_log_summary(msg) + if summary: + if len(summary) > 4: + known_info["summary"] = summary + #prompt.append_system_message(await self.get_log_summary(self,msg)) + system_prompt_dict["known_info"] = known_info + + ## 可以使用的tools(inner function)的解释，注意不定义该tips,则不会导入任何workspace中的tools + if self.tools_tips: + system_prompt_dict["tools_tips"] = self.tools_tips + #prompt.append_system_message(self.tools_tips) + prompt.inner_functions.extend(self.get_inner_function_desc_from_env()) + + ## 给予查询KB的权限 + if self.enable_kb: + prompt.inner_functions.extend(self.get_inner_function_desc_from_kb()) + + prompt.append_system_message(json.dumps(system_prompt_dict)) + ## 扩展已知信息 (这可能是一个LLM过程) + prompt.append_system_message(await self.get_extend_known_info(msg,prompt)) + + return prompt + + + async def get_inner_function(self,func_name:str) -> AIFunction: + return None + + async def exec_actions(self,actions:List[ActionItem],input:Dict,llm_result:LLMResult) -> bool: + msg = input.get("msg") + if msg.msg_type == AgentMsgType.TYPE_GROUPMSG: + resp_msg = msg.create_group_resp_msg(self.memory.agent_id,llm_result.resp) + else: + resp_msg = msg.create_resp_msg(llm_result.resp) + + llm_result.raw_result["resp_msg"] = resp_msg + + for action_item in actions: + op : AIOperation = self.enable_actions.get(action_item.name) + if op: + if action_item.parms is None: + action_item.parms = {} + + action_item.parms["input"] = input + action_item.parms["resp_msg"] = resp_msg + action_item.parms["llm_result"] = llm_result + action_item.parms["start_at"] = datetime.now() + action_item.parms["result"] = await op.execute(action_item.parms) + action_item.parms["end_at"] = datetime.now() + else: + logger.warn(f"action {action_item.name} not found") + return False + return True + + + +class ReviewTaskProcess(BaseLLMProcess): + def __init__(self) -> None: + super().__init__() + + async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]: + if await super().load_from_config(config) is False: + return False + + async def prepare_prompt(self) -> LLMPrompt: + prompt = LLMPrompt() + pass + + async def get_inner_function(self,func_name:str) -> AIFunction: + pass + + async def exec_actions(self,actions:List[ActionItem]) -> bool: + pass + +class DoTodoProcess(BaseLLMProcess): + def __init__(self) -> None: + super().__init__() + + async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]: + if await super().load_from_config(config) is False: + return False + + async def prepare_prompt(self) -> LLMPrompt: + prompt = LLMPrompt() + pass + + async def get_inner_function(self,func_name:str) -> AIFunction: + pass + + async def exec_actions(self,actions:List[ActionItem]) -> bool: + pass + + +class CheckTodoProcess(BaseLLMProcess): + def __init__(self) -> None: + super().__init__() + + async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]: + if await super().load_from_config(config) is False: + return False + + async def prepare_prompt(self) -> LLMPrompt: + prompt = LLMPrompt() + pass + + async def get_inner_function(self,func_name:str) -> AIFunction: + pass + + async def exec_actions(self,actions:List[ActionItem]) -> bool: + pass + +class SelfLearningProcess(BaseLLMProcess): + def __init__(self) -> None: + super().__init__() + + async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]: + if await super().load_from_config(config) is False: + return False + + async def prepare_prompt(self) -> LLMPrompt: + prompt = LLMPrompt() + pass + + async def get_inner_function(self,func_name:str) -> AIFunction: + pass + + async def exec_actions(self,actions:List[ActionItem]) -> bool: + pass + +class SelfThinkingProcess(BaseLLMProcess): + def __init__(self) -> None: + super().__init__() + + async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]: + if await super().load_from_config(config) is False: + return False + + async def prepare_prompt(self) -> LLMPrompt: + prompt = LLMPrompt() + pass + + async def get_inner_function(self,func_name:str) -> AIFunction: + pass + + async def exec_actions(self,actions:List[ActionItem]) -> bool: + pass + +class LLMProcessLoader: + def __init__(self) -> None: + self.loaders : Dict[str,Callable[[dict],Awaitable[BaseLLMProcess]]] = {} + return + + @classmethod + def get_instance(cls)->"LLMProcessLoader": + if not hasattr(cls,"_instance"): + cls._instance = LLMProcessLoader() + return cls._instance + + def register_loader(self, typename:str,loader:Callable[[dict],Awaitable[BaseLLMProcess]]): + self.loaders[typename] = loader + + async def load_from_config(self,config:dict) -> BaseLLMProcess: + llm_type_name = config.get("type") + if llm_type_name: + loader = self.loaders.get(llm_type_name) + if loader: + return await loader(config) + + selected_type = globals().get(llm_type_name) + if selected_type: + result : BaseLLMProcess = selected_type() + load_result = await result.load_from_config(config) + if load_result is False: + logger.warn(f"load LLMProcess {llm_type_name} from config failed! load_from_config return False") + return None + else: + return result + + + logger.warn(f"load LLMProcess {llm_type_name} from config failed! type not found") + return None diff --git a/src/aios/environment/environment.py b/src/aios/environment/environment.py index 72b5ee2..6d07a81 100644 --- a/src/aios/environment/environment.py +++ b/src/aios/environment/environment.py @@ -101,4 +101,10 @@ class CompositeEnvironment(SimpleEnvironment): self.functions[func.get_name()] = func operations = env.get_all_ai_operations() for op in operations: - self.operations[op.get_name()] = op \ No newline at end of file + self.operations[op.get_name()] = op + + def get_value(self,key:str) -> Optional[str]: + for env in self.envs: + val = env.get_value(key) + if val is not None: + return val \ No newline at end of file diff --git a/src/aios/proto/agent_msg.py b/src/aios/proto/agent_msg.py index 1dbc4f5..ef08c72 100644 --- a/src/aios/proto/agent_msg.py +++ b/src/aios/proto/agent_msg.py @@ -75,6 +75,11 @@ class AgentMsg: self.inner_call_chain = [] self.resp_msg = None + self.action_list = [] + + #context info + self.context_info:dict= {} + @classmethod def from_json(cls,json_obj:dict) -> 'AgentMsg': msg = AgentMsg() diff --git a/src/aios/proto/ai_function.py b/src/aios/proto/ai_function.py index 9786cca..7577bc5 100644 --- a/src/aios/proto/ai_function.py +++ b/src/aios/proto/ai_function.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Dict,Coroutine,Callable +from typing import Dict,Coroutine,Callable,List class ParameterDefine: def __init__(self) -> None: @@ -74,10 +74,11 @@ class AIFunction: # pass class ActionItem: - def __init__(self,name,args) -> None: - self.name = name - self.args = args - self.body = None + def __init__(self,name:str,args:List[str]) -> None: + self.name:str= name + self.args:List[str]= args + self.body:str = None + self.parms : Dict = None def append_body(self,body:str) -> None: if self.body is None: @@ -88,6 +89,17 @@ class ActionItem: def dumps(self) -> str: pass + @classmethod + def from_json(cls,json_obj:dict) -> 'ActionItem': + args = json_obj.get("args",[]) + r = ActionItem(json_obj["name"],args) + if json_obj.get("body"): + r.body = json_obj["body"] + r.parms = json_obj + + return r + + # call chain is a combination of ai_function,group of ai_function. class CallChain: def __init__(self) -> None: diff --git a/src/aios/proto/compute_task.py b/src/aios/proto/compute_task.py index f761976..db98eac 100644 --- a/src/aios/proto/compute_task.py +++ b/src/aios/proto/compute_task.py @@ -6,7 +6,8 @@ import shlex import uuid import time from typing import List, Union -from ..proto.ai_function import * +from .ai_function import * +from .agent_msg import * from ..knowledge import ObjectID from ..storage.storage import AIStorage @@ -40,20 +41,63 @@ class ComputeTaskType(Enum): TEXT_EMBEDDING ="text_embedding" IMAGE_EMBEDDING ="image_embedding" +# class Function(TypedDict, total=False): +# name: Required[str] +# """The name of the function to be called. + +# Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length +# of 64. +# """ + +# parameters: Required[shared_params.FunctionParameters] +# """The parameters the functions accepts, described as a JSON Schema object. + +# See the [guide](https://platform.openai.com/docs/guides/gpt/function-calling) +# for examples, and the +# [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for +# documentation about the format. + +# To describe a function that accepts no parameters, provide the value +# `{"type": "object", "properties": {}}`. +# """ + +# description: str +# """ +# A description of what the function does, used by the model to choose when and +# how to call the function. +# """ + class LLMPrompt: def __init__(self,prompt_str = None) -> None: - self.messages = [] + self.messages : List[Dict] = [] if prompt_str: self.messages.append({"role":"user","content":prompt_str}) - self.system_message = None + self.system_message : Dict = None + self.inner_functions : List[Dict] = [] + + def append_system_message(self,content:str): + if content is None: + return + + if self.system_message is None: + self.system_message = {"role":"system","content":content} + else: + self.system_message["content"] += content + + def append_user_message(self,content:str): + if content is None: + return + + self.messages.append({"role":"user","content":content}) def as_str(self)->str: result_str = "" if self.system_message: - result_str += self.system_message.get("role") + ":" + self.system_message.get("content") + "\n" + result_str = json.dumps(self.system_message) if self.messages: - for msg in self.messages: - result_str += msg.get("role") + ":" + msg.get("content") + "\n" + result_str += json.dumps(self.messages) + if self.inner_functions: + result_str += json.dumps(self.inner_functions) return result_str @@ -63,10 +107,18 @@ class LLMPrompt: result.append(self.system_message) result.extend(self.messages) return result + + def append(self,prompt:'LLMPrompt'): if prompt is None: return + + if prompt.inner_functions: + if self.inner_functions is None: + self.inner_functions = copy.deepcopy(prompt.inner_functions) + else: + self.inner_functions.extend(prompt.inner_functions) if prompt.system_message is not None: if self.system_message is None: @@ -76,11 +128,11 @@ class LLMPrompt: self.messages.extend(prompt.messages) - def load_from_config(self,config:list) -> bool: + def load_from_config(self,config:List[Dict]) -> bool: if isinstance(config,list) is not True: logger.error("prompt is not list!") return False - self.messages = [] + self.messages : List[Dict] = [] for msg in config: if msg.get("content"): if msg.get("role") == "system": @@ -126,11 +178,16 @@ class LLMResult: if llm_json_str == "**IGNORE**": r.state = LLMResultStates.IGNORE return r + + r.state = LLMResultStates.OK llm_json = json.loads(llm_json_str) r.resp = llm_json.get("resp") r.raw_result = llm_json - r.action_list = llm_json.get("actions") + action_list = llm_json.get("actions") + for action in action_list: + action_item = ActionItem.from_json(action) + r.action_list.append(action_item) return r diff --git a/src/component/agent_manager/agent_manager.py b/src/component/agent_manager/agent_manager.py index e3eb72d..631fbf9 100644 --- a/src/component/agent_manager/agent_manager.py +++ b/src/component/agent_manager/agent_manager.py @@ -80,9 +80,14 @@ class AgentManager: if the_agent is None: logger.warn(f"load agent {agent_id} from media failed!") return None - + the_agent.chat_db = self.db_path - return the_agent + if await the_agent.initial(): + return the_agent + else: + logger.warn(f"initial agent {agent_id} failed!") + return None + def remove(self,agent_id:str)->int: pass @@ -141,7 +146,7 @@ class AgentManager: else: init_env(owner_env) - if result_agent.load_from_config(config) is False: + if await result_agent.load_from_config(config) is False: logger.error(f"load agent from {agent_media} failed!") return None return result_agent diff --git a/src/system.cfg.toml b/src/system.cfg.toml new file mode 100644 index 0000000..d8afcea --- /dev/null +++ b/src/system.cfg.toml @@ -0,0 +1,3 @@ +[LLMAgentMessageProcess] +type="LLMAgentMessageProcess" +