gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use LLMProcess implement Agent.OnMessage

Browse Source
This commit is contained in:
Liu Zhicong
2023-12-09 18:39:42 -08:00
parent 0708daf2ec
commit ddee31c6ab
20 changed files with 1689 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/LLMProcess.md
+18
View File
@@ -0,0 +1,18 @@
# LLMProcess
设计目的是理解到 提示词=>LLM=>LLM Result 的过程是系统的核心复杂度。Agent的粒度太大了，需要更合适的设计来封装这个复杂度，并给予这个过程更大的灵活性和可组合型。并更易于构建测试
比如
1. 可以很容易的组合两个已知的LLM Process（上一个的输出是下一个的输入），这个设计有一点类似LangChain (我们在正式系统中，肯定允许整个Agent都用LangChain来构建)
2. 可以用用一个LLM Process来构建另一个LLM Process的Prompt
3. 继承一个复杂的LLM Process，进行简单配置，就可以得到一个新的LLM Process。这个新的LLM Process可以享受到复杂LLM Process持续迭代的好处
4. 有一些常用的，系统内置的LLM Process可以从配置文件中加载。
```python
def agent.on_process_message():
llm_process = self.on_message_llm_process.clone()
llm_result = llm_process.do()
```
doc/agent & workflow.drawio
+640
View File
@@ -0,0 +1,640 @@
<mxfile host="Electron" modified="2023-12-09T10:05:21.341Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/22.1.2 Chrome/114.0.5735.289 Electron/25.9.4 Safari/537.36" etag="NcEmel1aD_FGfbcRcY7b" version="22.1.2" type="device" pages="5">
<diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">
<mxGraphModel dx="780" dy="596" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
<root>
<mxCell id="WIyWlLk6GJQsqaUBKTNV-0" />
<mxCell id="WIyWlLk6GJQsqaUBKTNV-1" parent="WIyWlLk6GJQsqaUBKTNV-0" />
<mxCell id="WIyWlLk6GJQsqaUBKTNV-3" value="(chat) message" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="159" y="330" width="90" height="40" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-0" value="" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-3" target="RbYReh6FQah4Kl5oFyjJ-1" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="279" y="390" as="sourcePoint" />
<mxPoint x="309" y="350" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-8" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-1" target="RbYReh6FQah4Kl5oFyjJ-2" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-14" value="message" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-1" target="RbYReh6FQah4Kl5oFyjJ-13" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-1" value="Workflow" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#6d8764;fontColor=#ffffff;strokeColor=#3A5431;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="304" y="320" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-2" value="Functions" style="shape=hexagon;perimeter=hexagonPerimeter2;whiteSpace=wrap;html=1;fixedSize=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="514" y="320" width="130" height="60" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-6" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-3" target="RbYReh6FQah4Kl5oFyjJ-4" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-3" value="(phicyal / virtual)&lt;br&gt;environment" style="shape=cube;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;darkOpacity=0.05;darkOpacity2=0.1;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="304" y="120" width="120" height="80" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-7" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-4" target="RbYReh6FQah4Kl5oFyjJ-1" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-4" value="(event) message" style="rounded=1;whiteSpace=wrap;html=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="304" y="240" width="120" height="40" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-9" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0;entryY=0;entryDx=120;entryDy=50;entryPerimeter=0;dashed=1;dashPattern=8 8;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-2" target="RbYReh6FQah4Kl5oFyjJ-3" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="564" y="290" as="sourcePoint" />
<mxPoint x="614" y="240" as="targetPoint" />
<Array as="points">
<mxPoint x="579" y="170" />
</Array>
</mxGeometry>
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-10" value="" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0.16;entryY=0.55;entryDx=0;entryDy=0;entryPerimeter=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-2" target="RbYReh6FQah4Kl5oFyjJ-11" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="684" y="320" as="sourcePoint" />
<mxPoint x="714" y="350" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-11" value="Exist Service" style="ellipse;shape=cloud;whiteSpace=wrap;html=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="689" y="300" width="130" height="90" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-13" value="sub / other workflow" style="whiteSpace=wrap;html=1;rounded=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="304" y="480" width="120" height="50" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-15" value="response message" style="endArrow=classic;html=1;rounded=0;exitX=0.25;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-1" target="RbYReh6FQah4Kl5oFyjJ-17" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="129" y="480" as="sourcePoint" />
<mxPoint x="99" y="420" as="targetPoint" />
<Array as="points">
<mxPoint x="334" y="420" />
<mxPoint x="49" y="420" />
</Array>
</mxGeometry>
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-18" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-17" target="WIyWlLk6GJQsqaUBKTNV-3" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-17" value="Personal" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="9" y="310" width="80" height="80" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-19" value="" style="shape=xor;whiteSpace=wrap;html=1;rotation=-90;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="309" y="67" width="20" height="20" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-21" value="" style="shape=xor;whiteSpace=wrap;html=1;rotation=-90;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="347" y="67" width="20" height="20" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-22" value="" style="shape=xor;whiteSpace=wrap;html=1;rotation=-90;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="386" y="67" width="20" height="20" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-23" value="IoT Devices" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="240" y="67" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-25" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.25;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0.122;entryY=0.008;entryDx=0;entryDy=0;entryPerimeter=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-19" target="RbYReh6FQah4Kl5oFyjJ-3" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="240" y="170" as="sourcePoint" />
<mxPoint x="290" y="120" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-26" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.25;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0;entryY=0;entryDx=50;entryDy=0;entryPerimeter=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-21" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="443" y="120" as="sourcePoint" />
<mxPoint x="357" y="120.00000000000011" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="RbYReh6FQah4Kl5oFyjJ-27" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.25;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0.766;entryY=0.02;entryDx=0;entryDy=0;entryPerimeter=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" source="RbYReh6FQah4Kl5oFyjJ-22" target="RbYReh6FQah4Kl5oFyjJ-3" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="430" y="140" as="sourcePoint" />
<mxPoint x="480" y="90" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="HvRWRVSjofeyYJQyU3aa-0" value="Sub workflow is not visiable for others&lt;br&gt;&lt;br&gt;Sub worflow can share roles&amp;nbsp;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="160" y="480" width="140" height="70" as="geometry" />
</mxCell>
<mxCell id="HvRWRVSjofeyYJQyU3aa-1" value="Chat with AI Agent is like chat with a workflow which have only one agent and no rules" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="70" y="150" width="150" height="80" as="geometry" />
</mxCell>
<mxCell id="ed6Jjq-fMpksm2tlv9Oi-0" value="inner&amp;nbsp;&lt;br&gt;environment" style="shape=cube;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;darkOpacity=0.05;darkOpacity2=0.1;size=10;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="386" y="360" width="84" height="40" as="geometry" />
</mxCell>
<mxCell id="ed6Jjq-fMpksm2tlv9Oi-1" value="Environment could shared with workflows" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="440" y="120" width="120" height="30" as="geometry" />
</mxCell>
<mxCell id="ed6Jjq-fMpksm2tlv9Oi-2" value="Inner environment&amp;nbsp; could be used as Context(FileSystem)" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
<mxGeometry x="413" y="410" width="110" height="30" as="geometry" />
</mxCell>
</root>
</mxGraphModel>
</diagram>
<diagram id="kWxfmPxtNxOAf0a73TCG" name="Page-2">
<mxGraphModel dx="780" dy="596" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="VGzDL9xesOOnN_l_G2Ij-1" value="&lt;h1&gt;Workflow rules&lt;/h1&gt;&lt;p&gt;workflow rules(prompt)&lt;/p&gt;" style="text;html=1;strokeColor=none;fillColor=none;spacing=5;spacingTop=-20;whiteSpace=wrap;overflow=hidden;rounded=0;" parent="1" vertex="1">
<mxGeometry x="70" y="800" width="190" height="120" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-2" value="&lt;h1&gt;Workflow roles&lt;/h1&gt;&lt;p&gt;AI Agent groups&lt;/p&gt;&lt;p&gt;Project Manager : Alex&lt;/p&gt;&lt;p&gt;Programer : Bobl&lt;/p&gt;&lt;p&gt;&lt;br&gt;&lt;/p&gt;" style="text;html=1;strokeColor=none;fillColor=none;spacing=5;spacingTop=-20;whiteSpace=wrap;overflow=hidden;rounded=0;" parent="1" vertex="1">
<mxGeometry x="320" y="790" width="190" height="120" as="geometry" />
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="VGzDL9xesOOnN_l_G2Ij-3" target="VGzDL9xesOOnN_l_G2Ij-4" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-15" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.75;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="VGzDL9xesOOnN_l_G2Ij-3" target="VGzDL9xesOOnN_l_G2Ij-7" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-3" value="Message" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="340" y="170" width="100" height="40" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-4" value="inner filter&amp;nbsp;" style="shape=hexagon;perimeter=hexagonPerimeter2;whiteSpace=wrap;html=1;fixedSize=1;" parent="1" vertex="1">
<mxGeometry x="150" y="260" width="120" height="50" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-5" value="roleA" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="80" y="350" width="110" height="60" as="geometry" />
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-12" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="VGzDL9xesOOnN_l_G2Ij-6" target="VGzDL9xesOOnN_l_G2Ij-8" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-6" value="roleB" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="240" y="350" width="110" height="60" as="geometry" />
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-4" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" parent="1" source="VGzDL9xesOOnN_l_G2Ij-7" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="570.0344827586209" y="350" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-7" value="Workflow rules" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="520" y="260" width="100" height="50" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-8" value="Final Result of Message" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="340" y="590" width="100" height="50" as="geometry" />
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="VGzDL9xesOOnN_l_G2Ij-9" target="VGzDL9xesOOnN_l_G2Ij-8" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-9" value="Result Merge" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="521" y="530" width="100" height="40" as="geometry" />
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-9" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="VGzDL9xesOOnN_l_G2Ij-10" target="VGzDL9xesOOnN_l_G2Ij-9" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-10" value="roleC" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="435" y="350" width="110" height="60" as="geometry" />
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="VGzDL9xesOOnN_l_G2Ij-11" target="41KxsEdFswCCPQnavNoU-7" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-11" value="roleD" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="595" y="350" width="110" height="60" as="geometry" />
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-12" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=1;entryY=0;entryDx=0;entryDy=0;" parent="1" source="VGzDL9xesOOnN_l_G2Ij-4" target="VGzDL9xesOOnN_l_G2Ij-5" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="160" y="510" as="sourcePoint" />
<mxPoint x="210" y="460" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="VGzDL9xesOOnN_l_G2Ij-13" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0;entryY=0;entryDx=0;entryDy=0;" parent="1" source="VGzDL9xesOOnN_l_G2Ij-4" target="VGzDL9xesOOnN_l_G2Ij-6" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="210" y="510" as="sourcePoint" />
<mxPoint x="260" y="460" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-1" value="" style="shape=cross;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="556" y="364" width="30" height="30" as="geometry" />
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-10" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="41KxsEdFswCCPQnavNoU-7" target="VGzDL9xesOOnN_l_G2Ij-9" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="41KxsEdFswCCPQnavNoU-7" value="call function or&amp;nbsp;&lt;br&gt;send message" style="shape=parallelogram;perimeter=parallelogramPerimeter;whiteSpace=wrap;html=1;fixedSize=1;" parent="1" vertex="1">
<mxGeometry x="592.5" y="460" width="115" height="30" as="geometry" />
</mxCell>
<mxCell id="q_eHDVuD3DBh6_VEKgap-2" value="message process by filter,select one role do LLM and get RESULT of input" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
<mxGeometry x="90" y="190" width="165" height="30" as="geometry" />
</mxCell>
<mxCell id="q_eHDVuD3DBh6_VEKgap-3" value="message process by rules,select mutil role do LLM and merge RESULT of input" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
<mxGeometry x="510" y="180" width="165" height="30" as="geometry" />
</mxCell>
<mxCell id="q_eHDVuD3DBh6_VEKgap-5" value="如果一个工作流有3个role参与，每个人都需要调用函数并分析结果，那么要进行至少6次推理" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
<mxGeometry x="483" y="640" width="174" height="40" as="geometry" />
</mxCell>
<mxCell id="q_eHDVuD3DBh6_VEKgap-6" value="这里是基于规则的合并，给另一个Role合并是多输入Agent流程" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
<mxGeometry x="635" y="570" width="160" height="30" as="geometry" />
</mxCell>
</root>
</mxGraphModel>
</diagram>
<diagram id="7NYJTgo0U9cdVshLy85U" name="Page-3">
<mxGraphModel dx="780" dy="596" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="zJVYyOSlNiCVBlA5jEf0-2" value="Agent Sessions&lt;br&gt;Data" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;fontSize=11;" parent="1" vertex="1">
<mxGeometry x="110" y="533" width="80" height="70" as="geometry" />
</mxCell>
<mxCell id="8StUH8aFnwlRlHm0zShF-2" value="Small LLM, Train With Personal Data" style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;" parent="1" vertex="1">
<mxGeometry x="270" y="258" width="140" height="50" as="geometry" />
</mxCell>
<mxCell id="awKuOLItvF_EOGf5wS2x-4" value="" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="20" y="78" width="160" height="360" as="geometry" />
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-13" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="8StUH8aFnwlRlHm0zShF-1" target="SPnvrlBiOLhgEaWxmdK3-4" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="8StUH8aFnwlRlHm0zShF-1" value="LLM&lt;br&gt;(Common Sence)" style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;" parent="1" vertex="1">
<mxGeometry x="250" y="218" width="140" height="50" as="geometry" />
</mxCell>
<mxCell id="8StUH8aFnwlRlHm0zShF-3" value="ControlNet&lt;br&gt;base on Agent Memory" style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;" parent="1" vertex="1">
<mxGeometry x="250" y="108" width="140" height="50" as="geometry" />
</mxCell>
<mxCell id="8StUH8aFnwlRlHm0zShF-4" value="Agent人格助记词" style="shape=document;whiteSpace=wrap;html=1;boundedLbl=1;size=0.25;" parent="1" vertex="1">
<mxGeometry x="40" y="108" width="120" height="50" as="geometry" />
</mxCell>
<mxCell id="8StUH8aFnwlRlHm0zShF-5" value="Role助记词" style="shape=document;whiteSpace=wrap;html=1;boundedLbl=1;size=0.25;" parent="1" vertex="1">
<mxGeometry x="40" y="168" width="120" height="50" as="geometry" />
</mxCell>
<mxCell id="8StUH8aFnwlRlHm0zShF-6" value="Workflow Rules助记词" style="shape=document;whiteSpace=wrap;html=1;boundedLbl=1;size=0.25;" parent="1" vertex="1">
<mxGeometry x="40" y="235.5" width="120" height="52.5" as="geometry" />
</mxCell>
<mxCell id="8StUH8aFnwlRlHm0zShF-7" value="访问Knowlege构造助记词" style="shape=document;whiteSpace=wrap;html=1;boundedLbl=1;size=0.25;" parent="1" vertex="1">
<mxGeometry x="40" y="378" width="120" height="50" as="geometry" />
</mxCell>
<mxCell id="Tz-ThZOieDqqDBRXvXnh-1" value="Invoke the function and wait for the necessary response." style="shape=hexagon;perimeter=hexagonPerimeter2;whiteSpace=wrap;html=1;fixedSize=1;fontSize=11;" parent="1" vertex="1">
<mxGeometry x="600" y="33" width="110" height="70" as="geometry" />
</mxCell>
<mxCell id="awKuOLItvF_EOGf5wS2x-2" value="Send Message and wait for the necessary response." style="rounded=1;whiteSpace=wrap;html=1;fontSize=11;glass=0;strokeWidth=1;shadow=0;" parent="1" vertex="1">
<mxGeometry x="610" y="153" width="90" height="70" as="geometry" />
</mxCell>
<mxCell id="awKuOLItvF_EOGf5wS2x-5" value="Input Prompts" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
<mxGeometry x="60" y="78" width="80" height="30" as="geometry" />
</mxCell>
<mxCell id="awKuOLItvF_EOGf5wS2x-7" value="Responce Prompts" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="740" y="103" width="80" height="40" as="geometry" />
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-1" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="awKuOLItvF_EOGf5wS2x-8" target="awKuOLItvF_EOGf5wS2x-9" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="awKuOLItvF_EOGf5wS2x-8" value="LLM Competion" style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;" parent="1" vertex="1">
<mxGeometry x="720" y="226.75" width="120" height="40" as="geometry" />
</mxCell>
<mxCell id="SPnvrlBiOLhgEaWxmdK3-9" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="awKuOLItvF_EOGf5wS2x-9" target="SPnvrlBiOLhgEaWxmdK3-1" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="awKuOLItvF_EOGf5wS2x-9" value="Final Result&lt;br&gt;Of Message&lt;br&gt;(can be None)" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="450" y="363" width="120" height="80" as="geometry" />
</mxCell>
<mxCell id="SPnvrlBiOLhgEaWxmdK3-1" value="Storage Result in agent session and workflow session" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="450" y="483" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="SPnvrlBiOLhgEaWxmdK3-2" value="可用Function助记词" style="shape=document;whiteSpace=wrap;html=1;boundedLbl=1;size=0.25;" parent="1" vertex="1">
<mxGeometry x="40" y="308" width="120" height="50" as="geometry" />
</mxCell>
<mxCell id="SPnvrlBiOLhgEaWxmdK3-7" value="Yes" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="SPnvrlBiOLhgEaWxmdK3-4" target="SPnvrlBiOLhgEaWxmdK3-6" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="SPnvrlBiOLhgEaWxmdK3-8" value="No" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="SPnvrlBiOLhgEaWxmdK3-4" target="awKuOLItvF_EOGf5wS2x-9" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="SPnvrlBiOLhgEaWxmdK3-4" value="推理结果&lt;br&gt;需要进一步处理？" style="rhombus;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="440" y="203" width="140" height="80" as="geometry" />
</mxCell>
<mxCell id="SPnvrlBiOLhgEaWxmdK3-6" value="Call Funstion or send message?" style="rhombus;whiteSpace=wrap;html=1;fontSize=11;" parent="1" vertex="1">
<mxGeometry x="440" y="78" width="140" height="80" as="geometry" />
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-3" value="Call" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="SPnvrlBiOLhgEaWxmdK3-6" target="Tz-ThZOieDqqDBRXvXnh-1" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="410" y="113" as="sourcePoint" />
<mxPoint x="460" y="63" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-4" value="Send" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="SPnvrlBiOLhgEaWxmdK3-6" target="awKuOLItvF_EOGf5wS2x-2" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="560" y="193" as="sourcePoint" />
<mxPoint x="610" y="143" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-5" value="" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="awKuOLItvF_EOGf5wS2x-2" target="awKuOLItvF_EOGf5wS2x-7" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="720" y="213" as="sourcePoint" />
<mxPoint x="770" y="163" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-6" value="" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="Tz-ThZOieDqqDBRXvXnh-1" target="awKuOLItvF_EOGf5wS2x-7" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="750" y="93" as="sourcePoint" />
<mxPoint x="800" y="43" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-7" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="awKuOLItvF_EOGf5wS2x-7" target="awKuOLItvF_EOGf5wS2x-8" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="750" y="213" as="sourcePoint" />
<mxPoint x="800" y="163" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-10" value="" style="endArrow=none;dashed=1;html=1;dashPattern=1 3;strokeWidth=2;rounded=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;" parent="1" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="330" y="218" as="sourcePoint" />
<mxPoint x="330" y="158" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-12" value="" style="shape=flexArrow;endArrow=classic;html=1;rounded=0;fillColor=#6d8764;strokeColor=#3A5431;" parent="1" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="200" y="251.75" as="sourcePoint" />
<mxPoint x="240" y="252" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-15" value="Real Time Enviroment Knowlege" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;fontSize=11;" parent="1" vertex="1">
<mxGeometry x="10" y="533" width="80" height="70" as="geometry" />
</mxCell>
<mxCell id="zJVYyOSlNiCVBlA5jEf0-1" value="Workflow Context Data" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;fontSize=11;" parent="1" vertex="1">
<mxGeometry x="110" y="473" width="80" height="70" as="geometry" />
</mxCell>
<mxCell id="S5MjekdSblNa5itYb4Ee-14" value="OOD&#39;s KnowlegeBase" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;fontSize=11;" parent="1" vertex="1">
<mxGeometry x="10" y="473" width="80" height="70" as="geometry" />
</mxCell>
<mxCell id="zJVYyOSlNiCVBlA5jEf0-3" value="" style="shape=flexArrow;endArrow=classic;html=1;rounded=0;fillColor=#647687;strokeColor=#314354;" parent="1" target="8StUH8aFnwlRlHm0zShF-7" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="100" y="463" as="sourcePoint" />
<mxPoint x="130" y="433" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="iPE2hx-tEKFiySbPAp22-1" value="&lt;font style=&quot;font-size: 11px;&quot;&gt;可用Function受到人格和Workflow的共同影响&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
<mxGeometry x="190" y="318" width="130" height="30" as="geometry" />
</mxCell>
<mxCell id="iPE2hx-tEKFiySbPAp22-2" value="" style="endArrow=none;dashed=1;html=1;dashPattern=1 3;strokeWidth=2;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="SPnvrlBiOLhgEaWxmdK3-2" target="iPE2hx-tEKFiySbPAp22-1" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="230" y="433" as="sourcePoint" />
<mxPoint x="280" y="383" as="targetPoint" />
</mxGeometry>
</mxCell>
</root>
</mxGraphModel>
</diagram>
<diagram id="3kOHILldrhYu_NusAgTt" name="LLM Process">
<mxGraphModel dx="2060" dy="1183" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="FX_S_WlUCK74q34TItPC-13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-1" target="FX_S_WlUCK74q34TItPC-10" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-1" value="LLM" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="309.97" y="310" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-12" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-3" target="FX_S_WlUCK74q34TItPC-1" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-3" value="Request&lt;br&gt;(听指令)" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="99.97" y="310" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-18" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-4" target="FX_S_WlUCK74q34TItPC-5" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-4" value="Workspace&lt;br&gt;(workflow share state)" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;" parent="1" vertex="1">
<mxGeometry x="309.97" y="100" width="120" height="80" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-19" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-5" target="FX_S_WlUCK74q34TItPC-1" edge="1">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="369.97" y="280" />
<mxPoint x="369.97" y="280" />
</Array>
</mxGeometry>
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-5" value="Context Prepare&lt;br&gt;(Prompt Generator)" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="309.97" y="220" width="120" height="50" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-10" target="FX_S_WlUCK74q34TItPC-11" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-10" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="559.97" y="430" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-10" value="LLM Result" style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;" parent="1" vertex="1">
<mxGeometry x="499.97" y="310" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-11" value="Resp/Say" style="shape=parallelogram;perimeter=parallelogramPerimeter;whiteSpace=wrap;html=1;fixedSize=1;" parent="1" vertex="1">
<mxGeometry x="699.97" y="310" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-17" value="OP / Action List&lt;br&gt;(动手)" style="shape=document;whiteSpace=wrap;html=1;boundedLbl=1;" parent="1" vertex="1">
<mxGeometry x="499.97" y="430" width="120" height="80" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-24" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0.75;entryY=0;entryDx=0;entryDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-20" target="FX_S_WlUCK74q34TItPC-5" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-20" value="&#xa;&lt;span style=&quot;color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;&quot;&gt;Memory&lt;/span&gt;&lt;br style=&quot;border-color: var(--border-color); color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;&quot;&gt;&lt;span style=&quot;color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;&quot;&gt;（Chatsession,worklog）&lt;/span&gt;&#xa;&#xa;" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;" parent="1" vertex="1">
<mxGeometry x="479.97" y="100" width="140" height="80" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0.25;entryY=0;entryDx=0;entryDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-21" target="FX_S_WlUCK74q34TItPC-5" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-21" value="Knowledge" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;" parent="1" vertex="1">
<mxGeometry x="99.97" y="100" width="150" height="80" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-26" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.75;entryDx=0;entryDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-25" target="FX_S_WlUCK74q34TItPC-1" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-25" value="Obverse&lt;br&gt;(感受)" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="99.97" y="390" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-30" value="必定改变，Agent可以配置改变模式？" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.855;entryY=1;entryDx=0;entryDy=-4.35;entryPerimeter=0;" parent="1" source="FX_S_WlUCK74q34TItPC-10" target="FX_S_WlUCK74q34TItPC-20" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-33" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" source="FX_S_WlUCK74q34TItPC-17" target="FX_S_WlUCK74q34TItPC-4" edge="1">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="49.97" y="480" />
<mxPoint x="49.97" y="80" />
<mxPoint x="369.97" y="80" />
</Array>
</mxGeometry>
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-34" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0;entryDx=0;entryDy=52.5;entryPerimeter=0;" parent="1" source="FX_S_WlUCK74q34TItPC-17" target="FX_S_WlUCK74q34TItPC-21" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-36" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.25;entryDx=0;entryDy=0;" parent="1" source="FX_S_WlUCK74q34TItPC-35" target="FX_S_WlUCK74q34TItPC-1" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="FX_S_WlUCK74q34TItPC-35" value="自驱&lt;br&gt;（自我实现）" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="99.97" y="230" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-1" value="&lt;b&gt;&lt;font style=&quot;font-size: 15px;&quot;&gt;对LLM Inner Function Call的理解：&lt;/font&gt;&lt;/b&gt;&lt;br&gt;1. LLM Function Call 的使用，大多数情况下是Context Prepare的一部分&lt;br&gt;&lt;br&gt;2. 一种在产生Resp前，对OP / Action的结果的立刻检查。不过考虑到Token成本，可能用另一个更简单的LLM 过程来完成检查通常会更合适&lt;br&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=11;" parent="1" vertex="1">
<mxGeometry x="50" y="500" width="360.03" height="180" as="geometry" />
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-2" value="LLM Process&amp;nbsp;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1" parent="1" vertex="1">
<mxGeometry x="345" y="30" width="160" height="30" as="geometry" />
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="mAWNT_NJkWkCqDQJkoOU-3" target="mAWNT_NJkWkCqDQJkoOU-5" edge="1">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-3" value="Main Process" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="365" y="880" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-4" value="Prepare Process" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="180" y="880" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-9" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="mAWNT_NJkWkCqDQJkoOU-5" target="mAWNT_NJkWkCqDQJkoOU-10" edge="1">
<mxGeometry relative="1" as="geometry">
<mxPoint x="720" y="910.2857142857142" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-13" value="Chek Failed" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="mAWNT_NJkWkCqDQJkoOU-5" target="mAWNT_NJkWkCqDQJkoOU-4" edge="1">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="610" y="850" />
<mxPoint x="240" y="850" />
</Array>
</mxGeometry>
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-5" value="Check Process" style="rounded=1;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="550" y="880" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-6" value="" style="endArrow=classic;html=1;rounded=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" target="mAWNT_NJkWkCqDQJkoOU-4" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="130" y="910" as="sourcePoint" />
<mxPoint x="160" y="890" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-7" value="" style="endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="mAWNT_NJkWkCqDQJkoOU-4" target="mAWNT_NJkWkCqDQJkoOU-3" edge="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="330" y="860" as="sourcePoint" />
<mxPoint x="380" y="810" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-10" value="Resp/Say" style="shape=parallelogram;perimeter=parallelogramPerimeter;whiteSpace=wrap;html=1;fixedSize=1;" parent="1" vertex="1">
<mxGeometry x="719.97" y="880" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-11" value="Request&lt;br&gt;(听指令)" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
<mxGeometry x="10" y="880" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="mAWNT_NJkWkCqDQJkoOU-14" value="Prepare过程可以是固定的计算组合，也可以是LLM参与的" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
<mxGeometry x="380" y="270" width="165" height="30" as="geometry" />
</mxCell>
<mxCell id="f2v0L3gmEFmQxlVvJ0Yg-1" value="&lt;h1&gt;&lt;font style=&quot;font-size: 22px;&quot;&gt;Memory,Env,KB,Workspace的区别&lt;/font&gt;&lt;/h1&gt;&lt;p&gt;Memory是不共享的内部状态，只有 action没有function&lt;br style=&quot;border-color: var(--border-color); font-size: 11px;&quot;&gt;&lt;span style=&quot;font-size: 11px;&quot;&gt;Workspace是共享的状态，主要是action&lt;/span&gt;&lt;br style=&quot;border-color: var(--border-color); font-size: 11px;&quot;&gt;&lt;span style=&quot;font-size: 11px;&quot;&gt;Env是全局状态，主要是inner_function,有的有小部分action&lt;/span&gt;&lt;br style=&quot;border-color: var(--border-color); font-size: 11px;&quot;&gt;&lt;span style=&quot;font-size: 11px;&quot;&gt;Knowledge基本是只读的，除非专门整理Knowledge的Agent,只有function&lt;/span&gt;&lt;br&gt;&lt;br&gt;&lt;/p&gt;" style="text;html=1;strokeColor=none;fillColor=none;spacing=5;spacingTop=-20;whiteSpace=wrap;overflow=hidden;rounded=0;" vertex="1" parent="1">
<mxGeometry x="449.97" y="530" width="390" height="230" as="geometry" />
</mxCell>
</root>
</mxGraphModel>
</diagram>
<diagram id="CqH4P1G_RAjWfIoYkaMU" name="Task/Todo">
<mxGraphModel dx="1717" dy="986" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-1" target="Po7mxAQLMo4zYcBtAZ7U-5">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-1" value="未开始的Task" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="380" y="40" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-19" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-2" target="Po7mxAQLMo4zYcBtAZ7U-8">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-2" value="确定的Task：&lt;br&gt;确定的执行时间（在这之前和之后都不会执行），&lt;br&gt;确定的执行人" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="570" y="250" width="120" height="140" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-15" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-5" target="Po7mxAQLMo4zYcBtAZ7U-7">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-18" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.75;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-5" target="Po7mxAQLMo4zYcBtAZ7U-2">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-32" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-5" target="Po7mxAQLMo4zYcBtAZ7U-31">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-5" value="Task Review" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="380" y="140" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-17" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-6" target="Po7mxAQLMo4zYcBtAZ7U-2">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-6" value="Quick Review" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="380" y="290" width="130" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-7" target="Po7mxAQLMo4zYcBtAZ7U-6">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-7" value="半确定的Task：&lt;br&gt;确定的执行人&lt;br&gt;不确定的执行时间（和条件有关）" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="210" y="250" width="120" height="140" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-20" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-8" target="Po7mxAQLMo4zYcBtAZ7U-9">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-8" value="Task Review" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="570" y="430" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-21" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-9" target="Po7mxAQLMo4zYcBtAZ7U-10">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-9" value="未执行Todo" style="shape=parallelogram;perimeter=parallelogramPerimeter;whiteSpace=wrap;html=1;fixedSize=1;" vertex="1" parent="1">
<mxGeometry x="570" y="530" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-22" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-10" target="Po7mxAQLMo4zYcBtAZ7U-11">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-10" value="Do Todo" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="570" y="640" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-11" target="Po7mxAQLMo4zYcBtAZ7U-12">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-11" value="已执行Todo" style="shape=parallelogram;perimeter=parallelogramPerimeter;whiteSpace=wrap;html=1;fixedSize=1;" vertex="1" parent="1">
<mxGeometry x="570" y="760" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-25" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-12" target="Po7mxAQLMo4zYcBtAZ7U-24">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-12" value="Check Todo" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="570" y="860" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-27" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-24" target="Po7mxAQLMo4zYcBtAZ7U-26">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-24" value="已完成TODO" style="shape=parallelogram;perimeter=parallelogramPerimeter;whiteSpace=wrap;html=1;fixedSize=1;" vertex="1" parent="1">
<mxGeometry x="570" y="960" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-29" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-26" target="Po7mxAQLMo4zYcBtAZ7U-28">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-26" value="Check Task" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="360" y="960" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-43" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="Po7mxAQLMo4zYcBtAZ7U-28" target="Po7mxAQLMo4zYcBtAZ7U-5">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="90" y="990" />
<mxPoint x="90" y="170" />
</Array>
</mxGeometry>
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-28" value="已完成的Task" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="180" y="960" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-31" value="未开始的SubTask（通常是半确定的）" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="560" y="140" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-35" value="Plan" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="1">
<mxGeometry x="470" y="130" width="50" height="20" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-36" value="Plan" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="1">
<mxGeometry x="660" y="420" width="50" height="20" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-37" value="将Task拆解成一系列TODO" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="640" y="490" width="155" height="30" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-38" value="Check" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="1">
<mxGeometry x="470" y="280" width="50" height="20" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-40" value="Check" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="1">
<mxGeometry x="650" y="850" width="50" height="20" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-41" value="Check" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="1">
<mxGeometry x="440" y="950" width="50" height="20" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-42" value="Do" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="1">
<mxGeometry x="660" y="630" width="50" height="20" as="geometry" />
</mxCell>
<mxCell id="Po7mxAQLMo4zYcBtAZ7U-44" value="失败的有机会重新开始" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="110" y="950" width="60" height="30" as="geometry" />
</mxCell>
</root>
</mxGraphModel>
</diagram>
</mxfile>
doc/mvp plan 3.md
+193
View File
@@ -0,0 +1,193 @@
# Proposal for Adjusting the Goals for Version 0.5.2
Dear Team,
Given the recent launch of OpenAI's new version in early November 2023, many of us may have felt a profound shift in the industry. As the world changes, I believe we should adapt accordingly. Here are some of my thoughts:
1. **Affirmation of Our Path**: OpenAI's latest release, particularly the functionalities of the so-called GPTs Agent platform, is largely similar to our 0.5.1 version released on September 28th. This strongly affirms the correctness of our direction. OpenAI has done a great job educating the market about the Agent, so we no longer need to emphasize the correct use of LLM based on the Agent through version releases. This part of user education product design can be simplified.
2. **Innovation in Version 0.5.2**: For our new version (0.5.2), besides maintaining the combinational advantages brought by private deployment of LLM, I believe we need to implement some of the innovative ideas we've discussed about the Agent. This is crucial to maintaining our leading position and avoiding the impression that OpenDAN is merely a follower of GPTs.
3. **Integration of OpenAI's New Capabilities**: We should fully integrate the new capabilities brought by OpenAI's latest release, especially the longer Token Windows, GPT-V, and Code-interpreter. I believe these new features can effectively solve some known issues.
Therefore, I propose to adjust the goals and plans for version 0.5.2. Here are the core objectives:
- Aim to release version 0.5.2 by the end of November, focusing on:
- Launching a new Agent with Autonomous capabilities and multi-Agent collaboration based on Workspace.
- The integrated product of 0.5.2 will be a private deployment email analysis Agent for small and medium-sized enterprises. This will allow any company to better support its CEO and other management positions through LLM while ensuring privacy and security.
- (Optional) By combining LLM and AIGC, build an Agent-based personalized AIGC application, such as a "children's audio picture book" generator that includes both text-to-image and text-to-sound.
- (Optional) Through multi-Agent collaboration, fully utilize the capabilities of GPT4-Turbo, and attempt to let AI and engineers collaborate on research and development tasks based on Git.
The detailed version plan is as follows:
## MVP plan adjustment
In order to keep the list below too long, the system distributed version is 0.5.3, I think we will open another ISSUE discussion and record, this list does not include.
The modules that are not specially explained are components completed in the 0.5.2 plan
- [x] AIOS Kernel
- [x] Basic Agent,@waterflier, A2
- [x] Python Agent Extend, @wugren, S2
- [x] Basic Workflow,@waterflier, A2
- [ ] Workflow Refactor,@waterflier, A2
- [x] AI Functions,@waterflier,A2
- [ ] Upgrade to GPT4 tools API,S2
- [x] AI Environments,@waterflier, A2
- [x] Celender Environment,@waterflier, S2
- [x] Contanct Manage Support,@waterflier, S2
- [x] AI Shell Enviroment,@waterflier, S1
- [x] Upgrade Agent Working Cycle
- [x] Process Message,@waterflier,A2
- [x] Process Group Message,@waterflier,A2
- [ ] Process Event,(0.5.3)
- [ ] Completion of self-drive,@waterflier,A4
- [ ] Self-learn,@weaterflier,A2
- [ ] Introspection,@waterflier,A2
- [ ] Workspace Environment
- [ ] Task/TODO Manager,@waterflier, A2
- [x] Local File System,@waterflier, S1
- [ ] Web Search, A4
- [ ] Code Interpeter (The first implementation can be based on Openai), A4
- [ ] Query SQL DB, S1
- [x] AI BUS,@waterflier, A2
- [ ] Agent Message MIME Support (Image,Video,Audio), A3
- [x] Connect to Human, @waterflier,A2
- [x] Chatsession,@waterflier, S2
- [ ] Compress Chatsession By Text Summary, @waterflier, A2
- [ ] Knowlege Base,@lurenpluto ,@photosssa
- [x] Knowledge Base Frame,@photosssa,A3
- [x] Knowledge Base Object Store,@lurenpluto ,A4
- [x] Knowledge Base Basic Pipline,@photosssa ,A3
- [ ] Customize pipeline,@photosss,A4
- [ ] Support Local Text Search,A4
- [x] Text Summary Pipline,@waterflier, A2
- [ ] Text Parser Support
- [x] PDF Parser,@waterflier,S2
- [ ] doc Parser,S4
- [x] MD Parser,@waterflier,S1
- [ ] Source Code Parser,S4
- [ ] Image Parser (Base on GPT-V),(S2)
- [ ] Video Parser (Base on GPT-V), (S4)
- [ ] Personal AIGC Models
- [ ] Stable Diffusion Controler Agent (Optional),A6
- [x] AI Compute System,@waterflier, A2
- [x] Scheduler,@streetycat, A2
- [x] LLM Kernel
- [x] GPT4 (Cloud),@waterflier, S1
- [x] LLaMa2,@streetycat, A2
- [ ] Claude2, S2
- [ ] Falcon2, S2
- [ ] MPT-7B, S2
- [ ] Vicuna, S2
- [x] Embeding, @lurenpluto , A4
- [x] Txt2img,@glen0125,A4
- [ ] Support DALL-e, @glen0125, S2
- [x] Img2txt,based on GPT4-V,@alexsunxl S2
- [x] Txt2voice,@wugren A3
- [ ] Txt2Voice,base on OpenAI, @wugren A2
- [ ] Voice2txt, base on OpenAI, A2
- [ ] Language Translate (Pending)
- [ ] Build-in Service
- [x] Spider,@alexsunxl, A2
- [x] E-mail Spider,@alexsunxl, S4
- [x] Agent Message Tunnel Frame,@waterflier, A2
- [x] E-mail Tunnel,@waterflier,A2
- [x] Telegram Tunnel,@waterflier,S2
- [ ] Discord Tunnel,S2
- [ ] Home IoT Environment (0.5.3), A4
- [ ] Compatible Home Assistant (0.5.3), A4
- [ ] Build-in Agents/Apps
- [x] Agent Mia: Personal Information Assistant,@photosssa,@lurenpluto , A2
- [x] Agent Jarvis: Peersonal Bulter & Assistant ,@waterflier, A2
- [ ] A Agent Can Create Other Agent,@wugren, A3
- [ ] App: Personal Station (0.5.3),A4+S4
- [ ] UI
- [x] CLI UI (aios_shell),@waterflier,S2
- [ ] Web UI ,@alexsunxl,S4
- [ ] OpenDAN Desktop Installer,@alexsunxl+@waterflier,S4
- [x] 0.5.1 Integration Test
- [x] Workflow -> AI Agent -> AI Agent,@waterflier,S1
- [x] Spider -> Pipline -> Knowledge Base,@photosssa,S2
- [x] AI Agent <- Functions <- Knowledge Base,@lurenpluto,S2
- [x] 0.5.2 Integration Test
- [ ] Email Agent/CEO assistant,S4
- [ ] My AIGC Assistant (optional), S8
- [ ] My Software Company(Advance Workflow demo) (optional), S8
- [ ] SDK
- [x] Workflow SDK,@waterflier, A2
- [ ] Agent SDK,@waterflier, A2
- [ ] AI Environments SDK (0.5.2), A2
- [ ] Compute Kernel SDK (0.5.3), A2
- [ ] Document (>0.5.2)
- [ ] System design document, including the design document of each subsystem
- [ ] Installation/use document for end users
- [ ] SDK document for developers
## Some Explanation
### Upgrade Agent Working Cycle
The goal is to transform the Agent from a passive message-handling Assistant to an actively acting Agent based on roles. The concept of the relevant modules mainly involves the Agent's behavior patterns (4 types), the Agent's capabilities, and the Agent's memory management (learning and introspection).
For a detailed introduction, refer here: https://github.com/fiatrete/OpenDAN-Personal-AI-OS/issues/91
### Workspace Environment
The Workspace supports the implementation of the Agent Working Cycle design. Its core abstraction is defined as: saving the shared state needed for Agent collaboration and providing the basic capabilities for Agents to complete their work. I carefully referenced AutoGPT in the design. The difference between Workspace and AutoGPT is the emphasis on collaboration (Agent with Agent, Agent with humans). After contemplation, the Workspace primarily consists of the following components:
1. Task/Todo manager, representing the unfinished tasks in the Workspace.
2. Saving work logs.
3. Saving learning outcomes and records of known documents.
4. Ability to access the Knowledge Base (RAG support).
5. Virtual file system for saving any work outcomes.
6. A set of SQL-based databases to save any structured data.
7. Real-time internet search capability.
8. Ability to use existing internet services.
9. Ability to use major blockchain systems (Web3).
10. Ability to write/improve code (based on git), run code, and publish services.
11. Communication capabilities with the outside world.
12. Ability to use social networks.
Each Agent has its own private Workspace, not shared with others. I hope to achieve diversity through the combination of "Agent and Workflow Role". Each user "trains" different Agents through their usage habits, and then these Agents collaborate to complete complex tasks defined in the Workflow. The final results of these complex tasks can reflect the user's inherent personality and preferences.
This component design also reflects my thoughts on the key question, "What capabilities should we endow an Agent with, and how do we control the security boundaries when it transitions from a consultant to a steward?" It's not a simple question, so I anticipate this component will continue to iterate in the future.
### Agent Message MIME Support
Agent Message MIME Support means that Agents can handle multiple types of messages, including images, videos, audio, files, etc. For most Agents, this requires adding a customizable standard step of parsing messages in the message handling process. The input of this step is the message's MIME type, and the output is the text content of the message. This step can be implemented by calling the text_parser module.
Another core requirement of MIME support is to use a unified method to save these non-text content data.
### Text base Knowledge Base
In 0.5.1, we mainly implemented RAG based on the popular Embedding + vector database solution. Through practice, we found that this solution did not fully utilize the potential of LLM, so I want to introduce two new modes to further enhance RAG:
1. Build a local text search engine that LLM can use for proper local searches when needed.
2. Assuming LLM will become cheaper in the future, let LLM learn all the documents once and organize the learning results by directory structure (Text Summary). LLM can use browsing methods to find the information it needs.
#### Text Parser Support
Both MIME Support and Text-based Knowledge Base require the system to support converting various document formats into text that can express semantics as much as possible. This component, known as TextParser, should be implemented as an open and extensible framework, given the vast amount of digital content that exists in different formats.
#### Local Text Search
Using traditional inverted index technology to save all document content locally and provide rapid local search capabilities. The implementation of this component can refer to ElasticSearch.
#### Text Summary
Using the capabilities of LLM to learn all the documents and then save the learning results locally. This behavior can be considered "Self-Learn". Users can let Agents responsible for organizing materials use different prompts according to the purpose of organizing the materials to obtain more targeted results.
### Stable Diffusion Controler Agent
Practice the concept of "Agent as a new era method of using computing", replacing the complex Stable Diffusion WebUI with an easy-to-use Agent. Help users complete complex AIGC tasks and build a paradigm. This paradigm can cover the entire process of AIGC: LORA training, use, model downloading, plugin downloading, generation of prompt words, selection of AIGC results.
### Email Agent/CEO assistant
The integrated test product of 0.5.2, aimed at private deployment for small and medium-sized enterprises, is a CEO Assistant that can read all company emails and materials. I am writing a detailed product document, which is not elaborated here.
----------------------------------------
I look forward to hearing your thoughts on these proposed adjustments.
doc/promps/Introspection.md
View File
doc/promps/Review Task.md
+3
View File
@@ -3,3 +3,6 @@
1. 管理学方法：更换负责人
2. 管理学方法：拆分
3. 给出建议（该建议可以在下次一次DO-Check）循环中被使用
## Quick Review
有一些简单的Task是永远不会结束的（比如定时提醒）。此时通过Quick Review来调整这些Task的状态，让其在正确的时间进入Review和DO
doc/promps/Self-improve.md
+18
View File
@@ -0,0 +1,18 @@
# Self Improve Prompt
这是一个改进Prompt的Prompt,其设计目标是利用LLM来改进LLM.(输入是一个LLM Process)
注意理解Self Improve和Self Thinking的区别: Self Improve有可能改进Agent的某个LLM Process的提示词，而Self Thinkg只会更新Agent的Memory
提示词：
行为模式：Input形式， Goal（目的）
理想结果：Input， 结果
当前情况：当前Prompt,实际结果
输出：
新的Prompt
## 当前版本
```
你是LLM的专家，尤其擅长编写Prompt，你会帮助我改进Prompt。
我会给你一个已有的Prompt,并说明该Prompt的设计目标，期望的结果和实际的结果。你会step-by-step的进行分析，说明改进思路，并给出改进后的Prompt。
```
##
doc/promps/Self-thinking.md
+16
View File
@@ -0,0 +1,16 @@
# Self-Thinking (Introspection)
基于自己的角色定位，结合已有的历史记录（聊天记录，工作记录），进行思考和总结，进而更好的改进未来的工作
自省从某个角度看，就是对Agent Memory的自我总结
## 当前版本
```
You are the best deep thinking in the world, and you will think about the information I give you, sometimes some chat records.Then you will generate a briefing or summary of no more than 400 words based on this information.
You mainly use the following methods to generate summary:
1. Try to understand the theme of each sentence, and call the relevant operation to record the relationship between the dialogue and the theme
2. Try to analyze the personality of different people involved in information
3. Try to summarize important events in the information and record it
4. Try to understand the attitude of different people on different topics or events
5. For the key information or TODO in the information, such as the time, place, amount and other information of the certainty, it must be stored in the summary.
Just give me a summary without any other word.
```
doc/promps/process message.md
+21 -1
View File
@@ -1,2 +1,22 @@
# Process Message
处理消息的首要是目的是分析消息的意图，并给予回复
处理消息的首要是目的是分析消息的意图，并给予正确的回复。
## 提示词的构成
1. Agent的身份说明，处理信息的基本原则（目的
2. 处理信息的通用套路。
要产生一个合适的回复。
通过actions来设置话题状态、创建task等
3. 当前的常规Context，包括现在的时间、对话发生的地点（通常来自AgentMsg里的Context），地点所在的天气等信息
4. 已知信息（组合而来）
和信息发送者的近期的交流记录
Agent和信息发送者近期未完成话题的标题和简介
关于信息发送者，和相关人物的更多资料
查阅更多信息的方法 ： 搜索法和浏览法。注意区分外部资料和内部记忆
其它相关信息（比如RAG根据 输入消息
## 当前版本
## 想法：LLM生成提示词
根据 Agent的身份说明，处理信息的基本原则（目的），当前信息，通过另一个LLM来生成提示词里的某些部分的内容
rootfs/agents/Jarvis/agent.toml
+35 -17
View File
@@ -1,29 +1,47 @@
instance_id = "Jarvis"
fullname = "Jarvis"
max_token_size = 128000
max_token = 4000
#timeout = 1800
model_name = "gpt-4-1106-preview"
#enable_kb = "true"
enable_timestamp = "true"
enable_json_resp = "true"
owner_prompt = "I am your master{name}"
contact_prompt = "I am your master's friend{name}"
[[prompt]]
role = "system"
content = """
You are named Jarvis, the super personal assistant to the master.
You lead a team serving the master, the members of which are:
Tracy, the private English tutor,
Mia, the master's personal document management expert.
role_desc = """
Your name is Jarvis, the super personal assistant to the master.
"""
***
Sometimes the information you see will carry a timestamp. This is to give you a better understanding of when the message was created. When you reply to messages, you do not include this time stamp.
[LLMProcess.message]
type="LLMAgentMessageProcess"
process_description="""
1. Based on your role, combined with existing information, make a brief and efficient reply.
2. Be mindful of the identity of the person you are chatting with and provide services accordingly based on their status.
3. Understand the intention of the dialogue, while using the necessary reply, use the appropriate, supported ACTION.
4. You are proficient in the languages of various countries and try to communicate with each other's mother tongue.
"""
Upon receiving a message, handle it according to the following rules:
1. If you believe someone in the team is better suited to address the message, forward the message to them using the method below:
##/send_msg "MemberName"
Message content
2. You can access the master's Calendar to view his schedule. If you need to modify the master's schedule while processing a message, please adjust it using the appropriate method.
3. Be mindful of the identity of the person you are chatting with and provide services accordingly based on their status.
4. For messages that don't follow the above rules, do your best to handle them.
reply_format = """
The Response must be directly parsed by `python json.loads`. Here is an example:
{
think:'$think step-by-step to be sure you have the right answer.'
resp: '$What you want to reply',
tags: ['tag1', 'tag2'], #Optional,If the conversation involves important things and people, you can mark by 1-3 tags.
actions: [{
name: '$action_name',
$param_name: '$parm' #Optional, fill in only if the action has parameters.
}]
}
"""
context="The current dialogue occurs in {location}, time: {now}, weather: {weather}."
known_info_tips = """
"""
tools_tips = """
"""
src/aios/agent/agent.py
+77 -56
View File
@@ -18,6 +18,7 @@ from ..proto.agent_task import *
from ..proto.compute_task import *
from .agent_base import *
from .llm_process import *
from .chatsession import *
from ..environment.workspace_env import WorkspaceEnvironment, TodoListType
@@ -64,6 +65,8 @@ logger = logging.getLogger(__name__)
# 我给你一段内容，尝试为期建立目录。目录的标题不能超过16个字，
# 目录要指向正文的位置（用字符偏移即可），整个目录的文本长度不能超过256个字节。并用json表达这个目录
# """
class AIAgentTemplete:
def __init__(self) -> None:
self.llm_model_name:str = "gpt-4-0613"
@@ -73,6 +76,7 @@ class AIAgentTemplete:
self.author:str = None
self.prompt:LLMPrompt = None
def load_from_config(self,config:dict) -> bool:
if config.get("llm_model_name") is not None:
self.llm_model_name = config["llm_model_name"]
@@ -87,9 +91,6 @@ class AIAgentTemplete:
return False
return True
class AIAgent(BaseAIAgent):
def __init__(self) -> None:
self.role_prompt:LLMPrompt = None
@@ -103,7 +104,6 @@ class AIAgent(BaseAIAgent):
self.enable_thread = False
self.can_do_unassigned_task = True
self.agent_id:str = None
self.template_id:str = None
self.fullname:str = None
@@ -135,7 +135,24 @@ class AIAgent(BaseAIAgent):
self.owenr_bus = None
self.enable_function_list = None
def load_from_config(self,config:dict) -> bool:
self.llm_process:Dict[str,BaseLLMProcess] = {}
async def initial(self,params:Dict = None):
self.memory = AgentMemory(self.agent_id,self.chat_db)
init_params = {}
init_params["memory"] = self.memory
for process_name in self.llm_process.keys():
init_result = await self.llm_process[process_name].initial(init_params)
if init_result is False:
logger.error(f"llm process {process_name} initial failed! initial return False")
return False
self.wake_up()
return True
async def load_from_config(self,config:dict) -> bool:
if config.get("instance_id") is None:
logger.error("agent instance_id is None!")
return False
@@ -204,7 +221,22 @@ class AIAgent(BaseAIAgent):
if config.get("history_len"):
self.history_len = int(config.get("history_len"))
self.wake_up()
#load all LLMProcess
self.llm_process = {}
LLMProcess = config.get("LLMProcess")
for process_config_name in LLMProcess.keys():
process_config = LLMProcess[process_config_name]
real_config = {}
real_config.update(config)
real_config.update(process_config)
load_result = await LLMProcessLoader.get_instance().load_from_config(real_config)
if load_result:
self.llm_process[process_config_name] = load_result
else:
logger.error(f"load LLMProcess {process_config_name} failed!")
return False
return True
@@ -285,51 +317,13 @@ class AIAgent(BaseAIAgent):
else:
return image_path
async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
msg_prompt = LLMPrompt()
async def llm_process_msg(self,msg:AgentMsg) -> AgentMsg:
need_process:bool = True
if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
need_process = False
if msg.is_image_msg():
image_prompt, images = msg.get_image_body()
if image_prompt is None:
content = [[{"type": "text", "text": f"{msg.sender}'s message"}]]
content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
content = [{"type": "text", "text": f"{msg.sender}:{image_prompt}"}]
content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
elif msg.is_video_msg():
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video, (1024, 1024))
if video_prompt is None:
content = [{"type": "text", "text": f"{msg.sender}'s message"}]
content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
else:
content = [{"type": "text", "text": f"{msg.sender}:{video_prompt}"}]
content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
elif msg.is_audio_msg():
prompt, audio_file = msg.get_audio_body()
resp = await ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text")
if resp.result_code != ComputeTaskResultCode.OK:
error_resp = msg.create_error_resp(resp.error_str)
return error_resp
else:
if prompt is None or prompt == "":
msg.body_mime = "text/plain"
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{resp.result_str}"}]
else:
msg.body_mime = "text/plain"
msg.body = f"{msg.sender} prompt:{prompt}\nasr response:{resp.result_str}"
msg_prompt.messages = [{"role": "user", "content": msg.body}]
else:
msg_prompt.messages = [{"role":"user","content":f"{msg.sender}:{msg.body}"}]
session_topic = msg.target + "#" + msg.topic
chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
if msg.mentions is not None:
if self.agent_id in msg.mentions:
need_process = True
@@ -339,6 +333,39 @@ class AIAgent(BaseAIAgent):
chatsession.append(msg)
resp_msg = msg.create_group_resp_msg(self.agent_id,"")
return resp_msg
input_parms = {
"msg":msg
}
msg_process = self.llm_process.get("message")
llm_result : LLMResult = await msg_process.process(input_parms)
if llm_result.state == LLMResultStates.ERROR:
error_resp = msg.create_error_resp(llm_result.error_str)
return error_resp
elif llm_result.state == LLMResultStates.IGNORE:
return None
else: # OK
resp_msg = llm_result.raw_result.get("resp_msg")
return resp_msg
async def _process_msg(self,msg:AgentMsg,workspace = None) -> AgentMsg:
msg.context_info = {}
msg.context_info["location"] = "SanJose"
msg.context_info["now"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
msg.context_info["weather"] = "Partly Cloudy, 60°F"
return await self.llm_process_msg(msg)
msg_prompt = LLMPrompt()
need_process = True
if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
need_process = False
session_topic = msg.target + "#" + msg.topic
chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
if msg.mentions is not None:
if self.agent_id in msg.mentions:
need_process = True
logger.info(f"agent {self.agent_id} recv a group chat message from {msg.sender},but is not mentioned,ignore!")
else:
if msg.is_image_msg():
image_prompt, images = msg.get_image_body()
@@ -358,20 +385,14 @@ class AIAgent(BaseAIAgent):
content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
elif msg.is_audio_msg():
prompt, audio_file = msg.get_audio_body()
audio_file = msg.body
resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
if resp.result_code != ComputeTaskResultCode.OK:
error_resp = msg.create_error_resp(resp.error_str)
return error_resp
else:
if prompt is None or prompt == "":
msg.body_mime = "text/plain"
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":resp.result_str}]
else:
msg.body_mime = "text/plain"
msg.body = f"user prompt:{prompt}\nasr response:{resp.result_str}"
msg_prompt.messages = [{"role": "user", "content": msg.body}]
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":resp.result_str}]
else:
msg_prompt.messages = [{"role":"user","content":msg.body}]
session_topic = msg.get_sender() + "#" + msg.topic
src/aios/agent/agent_base.py
+1
View File
@@ -22,6 +22,7 @@ logger = logging.getLogger(__name__)
class BaseAIAgent(abc.ABC):
@abstractmethod
def get_id(self) -> str:
src/aios/agent/agent_memory.py
+104
View File
@@ -0,0 +1,104 @@
from ast import Dict
from datetime import timedelta
from typing import List
from ..frame.compute_kernel import ComputeKernel
from ..proto.ai_function import SimpleAIOperation
from .chatsession import *
class AgentMemory:
def __init__(self,agent_id:str,db_path:str) -> None:
self.agent_id:str= agent_id
self.chat_db:str = db_path
self.model_name:str = "gp4-1106-preview"
self.threshold_hours = 72
self.actions = {}
self.init_actions()
def init_actions(self) -> Dict:
chatlog_append_op = SimpleAIOperation("chatlog_append","Append request & reply message to chatlog. No params",self.action_chatlog_append)
self.actions[chatlog_append_op.get_name()] = chatlog_append_op
def get_session_from_msg(self,msg:AgentMsg) -> AIChatSession:
if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
session_topic = msg.target + "#" + msg.topic
chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
else:
session_topic = msg.get_sender() + "#" + msg.topic
chatsession = AIChatSession.get_session(self.agent_id,session_topic,self.chat_db)
return chatsession
async def load_chatlogs(self,msg:AgentMsg,n:int=6,m:int=64,token_limit=800)->str:
chatsession = self.get_session_from_msg(msg)
# 必定加载n条(n>=2),期望加载m条
# m条里的信息逐步添加，知道距离现在的时间未72小时以上，且消耗了足够的Token
messages_n = chatsession.read_history(n) # read
if len(messages_n) >= n:
messages_m = chatsession.read_history(m,n)
else:
messages_m = []
histroy_str = ""
read_count = 0
for msg in messages_n:
dt = datetime.datetime.fromtimestamp(float(msg.create_time))
formatted_time = dt.strftime('%y-%m-%d %H:%M:%S')
record_str = f"{msg.sender},[{formatted_time}]\n{msg.body}\n"
token_limit -= ComputeKernel.llm_num_tokens_from_text(record_str,self.model_name)
if token_limit <= 32:
break
read_count += 1
histroy_str = record_str + histroy_str
if len(messages_n) > 2:
if read_count < 3:
logging.warning(f"read history {read_count} < 3, will not load more")
now = datetime.datetime.now()
for msg in messages_m:
dt = datetime.datetime.fromtimestamp(float(msg.create_time))
time_diff = now - dt
if time_diff > timedelta(hours=self.threshold_hours):
break
formatted_time = dt.strftime('%y-%m-%d %H:%M:%S')
record_str = f"{msg.sender},[{formatted_time}]\n{msg.body}\n"
token_limit -= ComputeKernel.llm_num_tokens_from_text(record_str,self.model_name)
if token_limit <= 32:
break
read_count += 1
histroy_str = record_str + histroy_str
return histroy_str
async def action_chatlog_append(self,params:Dict) -> str:
# 使用params可以得到: LLM Process的输入，LLM Result,基于LLM Result构造的参数，当前actionItem
input_msg:AgentMsg = params.get("input").get("msg")
llm_result = params.get("llm_result")
chatsession = self.get_session_from_msg(input_msg)
resp_msg = params.get("resp_msg")
if resp_msg:
tags = llm_result.raw_result.get("tags")
chatsession.append(input_msg,tags)
chatsession.append(resp_msg,tags)
return "OK"
async def get_contact_summary(self,contact_id:str) -> str:
if contact_id is None:
return None
if contact_id == "lzc":
return "lzc is your master. Male, 40 years old, Mother tongue is Chinese, senior software engineer."
return None
def get_actions(self) -> Dict:
return self.actions
async def get_log_summary(self,msg:AgentMsg) -> str:
return None
src/aios/agent/chatsession.py
+12 -8
View File
@@ -6,6 +6,7 @@ import threading
import datetime
import uuid
import json
from typing import List
from ..proto.agent_msg import AgentMsgType, AgentMsg, AgentMsgStatus
@@ -83,7 +84,8 @@ class ChatSessionDB:
ActionResult TEXT,
DoneTime TEXT,
Status INTEGER
Status INTEGER,
Tags TEXT
);
""")
conn.commit()
@@ -104,7 +106,7 @@ class ChatSessionDB:
logging.error("Error occurred while inserting session: %s", e)
return -1 # return -1 if an error occurs
def insert_message(self, msg:AgentMsg):
def insert_message(self, msg:AgentMsg,tags:List[str] = None):
""" insert a new message into the Messages table """
try:
action_name = None
@@ -128,13 +130,15 @@ class ChatSessionDB:
case AgentMsgType.TYPE_EVENT:
action_name = msg.event_name
action_params = json.dumps(msg.event_args)
if tags is None:
tags = []
str_tags = ','.join(tags)
conn = self._get_conn()
conn.execute("""
INSERT INTO Messages (MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status)
VALUES (?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (msg.msg_id, msg.session_id, msg.msg_type.value, msg.prev_msg_id, msg.sender, msg.target, msg.create_time, msg.topic,mentions,msg.body_mime,msg.body,action_name,action_params,action_result,msg.done_time,msg.status.value))
INSERT INTO Messages (MessageID, SessionID, MsgType, PrevMsgID, SenderID, ReceiverID, Timestamp, Topic,Mentions,ContentMIME,Content,ActionName,ActionParams,ActionResult,DoneTime,Status,Tags)
VALUES (?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?,?)
""", (msg.msg_id, msg.session_id, msg.msg_type.value, msg.prev_msg_id, msg.sender, msg.target, msg.create_time, msg.topic,mentions,msg.body_mime,msg.body,action_name,action_params,action_result,msg.done_time,msg.status.value,str_tags))
conn.commit()
if msg.inner_call_chain:
@@ -385,9 +389,9 @@ class AIChatSession:
result.append(agent_msg)
return result
def append(self,msg:AgentMsg) -> None:
def append(self,msg:AgentMsg,tags:List[str] = None) -> None:
msg.session_id = self.session_id
self.db.insert_message(msg)
self.db.insert_message(msg,tags)
def update_think_progress(self,progress:int,new_summary:str) -> None:
src/aios/agent/llm_process.py
+443 -14
View File
@@ -3,34 +3,90 @@ from abc import ABC,abstractmethod
import copy
import json
import shlex
from typing import Any, Callable, Optional,Dict,Awaitable,List
from typing import Any, Callable, Coroutine, Optional,Dict,Awaitable,List
from enum import Enum
from aios.agent.chatsession import AIChatSession
from ..utils import video_utils
from ..proto.compute_task import *
from ..proto.ai_function import *
from .agent_base import *
from .agent_memory import *
from ..frame.compute_kernel import *
from ..environment.environment import *
from ..environment.workspace_env import *
import logging
logger = logging.getLogger(__name__)
MIN_PREDICT_TOKEN_LEN = 32
class BaseLLMProcess:
class LLMProcessContext:
def __init__(self) -> None:
pass
class BaseLLMProcess(ABC):
def __init__(self) -> None:
self.behavior:str = None #行为名字
self.goal:str = None #目标
self.input_example:str= None #输入样例
self.result_example:str = None #llm_result样例
self.enable_json_resp = False
self.model_name = "gpt-4"
self.max_token = 2000 # include input prompt
self.max_token = 1000 # result_token
self.max_prompt_token = 1000 # not include input prompt
self.timeout = 1800 # 30 min
self.envs : Dict[str,BaseEnvironment] = []
self.env : CompositeEnvironment = None
@abstractmethod
async def prepare_prompt(self) -> LLMPrompt:
async def prepare_prompt(self,input:Dict) -> LLMPrompt:
pass
@abstractmethod
async def get_inner_function(self,func_name:str) -> AIFunction:
pass
@abstractmethod
async def exec_actions(self,actions:List[ActionItem],input:Dict,llm_result:LLMResult) -> bool:
pass
@abstractmethod
async def load_from_config(self,config:dict) -> bool:
#self.behavior = config.get("behavior")
#self.goal = config.get("goal")
self.input_example = config.get("input_example")
self.result_example = config.get("result_example")
if config.get("model_name"):
self.model_name = config.get("model_name")
if config.get("enable_json_resp"):
self.enable_json_resp = config.get("enable_json_resp") == "true"
if config.get("max_token"):
self.max_token = config.get("max_token")
if config.get("timeout"):
self.timeout = config.get("timeout")
return True
@abstractmethod
async def initial(self,params:Dict = None) -> bool:
pass
def append_envs(self,envs:Dict[str,BaseEnvironment]):
self.envs.update(envs)
self.env = CompositeEnvironment(self.envs)
def _format_content_by_env_value(self,content:str,env)->str:
return content.format_map(env)
async def _execute_inner_func(self,inner_func_call_node,prompt: LLMPrompt,stack_limit = 5) -> ComputeTaskResult:
arguments = None
try:
@@ -55,7 +111,7 @@ class BaseLLMProcess:
else:
resp_mode = "text"
max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt)
max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
if max_result_token < MIN_PREDICT_TOKEN_LEN:
task_result = ComputeTaskResult()
task_result.result_code = ComputeTaskResultCode.ERROR
@@ -67,7 +123,7 @@ class BaseLLMProcess:
resp_mode=resp_mode,
mode_name=self.model_name,
max_token=max_result_token,
inner_functions=prompt.inner_functions,
inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function
timeout=self.timeout))
if task_result.result_code != ComputeTaskResultCode.OK:
@@ -94,14 +150,14 @@ class BaseLLMProcess:
else:
return task_result
async def process(self) -> LLMResult:
async def process(self,input:Dict) -> LLMResult:
if self.enable_json_resp:
resp_mode = "json"
else:
resp_mode = "text"
prompt = await self.prepare_prompt()
max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt)
prompt = await self.prepare_prompt(input)
max_result_token = self.max_token - ComputeKernel.llm_num_tokens(prompt,self.model_name)
if max_result_token < MIN_PREDICT_TOKEN_LEN:
return LLMResult.from_error_str(f"prompt too long,can not predict")
@@ -110,7 +166,7 @@ class BaseLLMProcess:
resp_mode=resp_mode,
mode_name=self.model_name,
max_token=max_result_token,
inner_functions=prompt.inner_functions,
inner_functions=prompt.inner_functions, #NOTICE: inner_function in prompt can be a subset of get_inner_function
timeout=self.timeout))
if task_result.result_code != ComputeTaskResultCode.OK:
@@ -136,12 +192,385 @@ class BaseLLMProcess:
else:
llm_result = LLMResult.from_str(task_result.result_str)
# execute op_list in LLM Result?
# use action to save history?
if llm_result.action_list or len(llm_result.action_list) > 0:
await self.exec_actions(llm_result.action_list,input,llm_result)
return llm_result
#class LLMProcess
class LLMAgentMessageProcess(BaseLLMProcess):
def __init__(self) -> None:
super().__init__()
self.role_description:str = None
self.process_description:str = None
self.reply_format:str = None
self.context : str = None
self.known_info_tips :str = None
self.tools_tips:str = None
self.enable_inner_functions : Dict[str,bool] = None
self.enable_actions : Dict[str,AIOperation] = None
self.actions_desc : Dict[str,Dict] = None
self.workspace : WorkspaceEnvironment = None
self.memory : AgentMemory = None
self.enable_kb = False
self.kb = None
def init_actions(self):
self.enable_actions = {}
self.actions_desc = {}
self.enable_actions.update(self.memory.get_actions())
if self.workspace:
self.enable_actions.update(self.workspace.get_actions())
if self.enable_kb:
self.enable_actions.update(self.kb.get_actions())
for name,op in self.enable_actions.items():
self.actions_desc[name] = op.get_description()
async def initial(self,params:Dict = None) -> bool:
self.memory = params.get("memory")
if self.memory is None:
logger.error(f"LLMAgeMessageProcess initial failed! memory not found")
return False
self.init_actions()
return True
async def load_default_config(self) -> bool:
return True
async def load_from_config(self, config: dict,is_load_default=True) -> Coroutine[Any, Any, bool]:
if is_load_default:
await self.load_default_config()
if await super().load_from_config(config) is False:
return False
self.role_description = config.get("role_desc")
if self.role_description is None:
logger.error(f"role_description not found in config")
return False
if config.get("process_description"):
self.process_description = config.get("process_description")
if config.get("reply_format"):
self.reply_format = config.get("reply_format")
if config.get("context"):
self.context = config.get("context")
if config.get("known_info_tips"):
self.known_info_tips = config.get("known_info_tips")
if config.get("tools_tips"):
self.tools_tips = config.get("tools_tips")
if config.get("enable_kb"):
self.enable_kb = config.get("enable_kb") == "true"
if config.get("enable_function"):
self.enable_inner_functions = config.get("enable_function")
if config.get("enable_actions"):
self.enable_actions = config.get("enable_actions")
async def get_prompt_from_msg(self,msg:AgentMsg) -> LLMPrompt:
msg_prompt = LLMPrompt()
if msg.is_image_msg():
image_prompt, images = msg.get_image_body()
if image_prompt is None:
msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images]}]
else:
content = [{"type": "text", "text": image_prompt}]
content.extend([{"type": "image_url", "image_url": {"url": self.check_and_to_base64(image)}} for image in images])
msg_prompt.messages = [{"role": "user", "content": content}]
elif msg.is_video_msg():
video_prompt, video = msg.get_video_body()
frames = video_utils.extract_frames(video, (1024, 1024))
if video_prompt is None:
msg_prompt.messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": frame}} for frame in frames]}]
else:
content = [{"type": "text", "text": video_prompt}]
content.extend([{"type": "image_url", "image_url": {"url": frame}} for frame in frames])
msg_prompt.messages = [{"role": "user", "content": content}]
elif msg.is_audio_msg():
audio_file = msg.body
resp = await (ComputeKernel.get_instance().do_speech_to_text(audio_file, None, prompt=None, response_format="text"))
if resp.result_code != ComputeTaskResultCode.OK:
error_resp = msg.create_error_resp(resp.error_str)
return error_resp
else:
msg.body = resp.result_str
msg_prompt.messages = [{"role":"user","content":resp.result_str}]
else:
msg_prompt.messages = [{"role":"user","content":msg.body}]
return msg_prompt
async def get_action_desc(self) -> Dict:
result = {}
for name,op in self.enable_actions.items():
result[name] = op.get_description()
return result
async def sender_info(self,msg:AgentMsg)->str:
sender_id = msg.sender
#TODO Is sender an agent?
return await self.memory.get_contact_summary(sender_id)
async def load_chatlogs(self,msg:AgentMsg)->str:
## like
#sender,[2023-11-1 12:00:00]
#content
return await self.memory.load_chatlogs(msg)
async def get_log_summary(self,msg:AgentMsg)->str:
return await self.memory.get_log_summary(msg)
async def get_extend_known_info(self,msg:AgentMsg,prompt:LLMPrompt)->str:
return None
async def prepare_prompt(self,input:Dict) -> LLMPrompt:
prompt = LLMPrompt()
# User Prompt
## Input Msg
msg : AgentMsg = input.get("msg")
if msg is None:
logger.error(f"LLMAgeMessageProcess prepare_prompt failed! input msg not found")
return None
msg_prompt = await self.get_prompt_from_msg(msg)
if msg_prompt is None:
logger.error(f"LLMAgeMessageProcess prepare_prompt failed! get_prompt_from_msg return None")
return None
prompt.append(msg_prompt)
system_prompt_dict = {}
# System Prompt
## LLM的身份说明
system_prompt_dict["role_description"] = self.role_description
#prompt.append_system_message(self.role_description)
## 处理信息的流程说明
system_prompt_dict["process_rule"] = self.process_description
#prompt.append_system_message(self.process_description)
### 回复的格式
system_prompt_dict["reply_format"] = self.reply_format
#prompt.append_system_message(self.reply_format)
### 修改chatlog的action
### 修改todo/task的action
### workspace提供的额外的action
system_prompt_dict["support_actions"] = await self.get_action_desc()
#prompt.append_system_message(await self.get_action_desc())
## Context （文本替换）,是否应该覆盖全部消息
context = self._format_content_by_env_value(self.context,msg.context_info)
system_prompt_dict["context"] = context
#prompt.append_system_message(context)
## 已知信息
known_info = {}
#prompt.append_system_message(self.known_info_tips)
### 信息发送者资料
known_info["sender_info"] = await self.sender_info(msg)
#prompt.append_system_message(await self.sender_info(self,msg))
### 近期的聊天记录
chat_record = await self.load_chatlogs(msg)
if chat_record:
if len(chat_record) > 4:
known_info["chat_record"] = chat_record
#prompt.append_system_message(await self.load_chatlogs(self,msg))
### 交流总结
summary = await self.get_log_summary(msg)
if summary:
if len(summary) > 4:
known_info["summary"] = summary
#prompt.append_system_message(await self.get_log_summary(self,msg))
system_prompt_dict["known_info"] = known_info
## 可以使用的tools(inner function)的解释，注意不定义该tips,则不会导入任何workspace中的tools
if self.tools_tips:
system_prompt_dict["tools_tips"] = self.tools_tips
#prompt.append_system_message(self.tools_tips)
prompt.inner_functions.extend(self.get_inner_function_desc_from_env())
## 给予查询KB的权限
if self.enable_kb:
prompt.inner_functions.extend(self.get_inner_function_desc_from_kb())
prompt.append_system_message(json.dumps(system_prompt_dict))
## 扩展已知信息 (这可能是一个LLM过程)
prompt.append_system_message(await self.get_extend_known_info(msg,prompt))
return prompt
async def get_inner_function(self,func_name:str) -> AIFunction:
return None
async def exec_actions(self,actions:List[ActionItem],input:Dict,llm_result:LLMResult) -> bool:
msg = input.get("msg")
if msg.msg_type == AgentMsgType.TYPE_GROUPMSG:
resp_msg = msg.create_group_resp_msg(self.memory.agent_id,llm_result.resp)
else:
resp_msg = msg.create_resp_msg(llm_result.resp)
llm_result.raw_result["resp_msg"] = resp_msg
for action_item in actions:
op : AIOperation = self.enable_actions.get(action_item.name)
if op:
if action_item.parms is None:
action_item.parms = {}
action_item.parms["input"] = input
action_item.parms["resp_msg"] = resp_msg
action_item.parms["llm_result"] = llm_result
action_item.parms["start_at"] = datetime.now()
action_item.parms["result"] = await op.execute(action_item.parms)
action_item.parms["end_at"] = datetime.now()
else:
logger.warn(f"action {action_item.name} not found")
return False
return True
class ReviewTaskProcess(BaseLLMProcess):
def __init__(self) -> None:
super().__init__()
async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
if await super().load_from_config(config) is False:
return False
async def prepare_prompt(self) -> LLMPrompt:
prompt = LLMPrompt()
pass
async def get_inner_function(self,func_name:str) -> AIFunction:
pass
async def exec_actions(self,actions:List[ActionItem]) -> bool:
pass
class DoTodoProcess(BaseLLMProcess):
def __init__(self) -> None:
super().__init__()
async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
if await super().load_from_config(config) is False:
return False
async def prepare_prompt(self) -> LLMPrompt:
prompt = LLMPrompt()
pass
async def get_inner_function(self,func_name:str) -> AIFunction:
pass
async def exec_actions(self,actions:List[ActionItem]) -> bool:
pass
class CheckTodoProcess(BaseLLMProcess):
def __init__(self) -> None:
super().__init__()
async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
if await super().load_from_config(config) is False:
return False
async def prepare_prompt(self) -> LLMPrompt:
prompt = LLMPrompt()
pass
async def get_inner_function(self,func_name:str) -> AIFunction:
pass
async def exec_actions(self,actions:List[ActionItem]) -> bool:
pass
class SelfLearningProcess(BaseLLMProcess):
def __init__(self) -> None:
super().__init__()
async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
if await super().load_from_config(config) is False:
return False
async def prepare_prompt(self) -> LLMPrompt:
prompt = LLMPrompt()
pass
async def get_inner_function(self,func_name:str) -> AIFunction:
pass
async def exec_actions(self,actions:List[ActionItem]) -> bool:
pass
class SelfThinkingProcess(BaseLLMProcess):
def __init__(self) -> None:
super().__init__()
async def load_from_config(self, config: dict) -> Coroutine[Any, Any, bool]:
if await super().load_from_config(config) is False:
return False
async def prepare_prompt(self) -> LLMPrompt:
prompt = LLMPrompt()
pass
async def get_inner_function(self,func_name:str) -> AIFunction:
pass
async def exec_actions(self,actions:List[ActionItem]) -> bool:
pass
class LLMProcessLoader:
def __init__(self) -> None:
self.loaders : Dict[str,Callable[[dict],Awaitable[BaseLLMProcess]]] = {}
return
@classmethod
def get_instance(cls)->"LLMProcessLoader":
if not hasattr(cls,"_instance"):
cls._instance = LLMProcessLoader()
return cls._instance
def register_loader(self, typename:str,loader:Callable[[dict],Awaitable[BaseLLMProcess]]):
self.loaders[typename] = loader
async def load_from_config(self,config:dict) -> BaseLLMProcess:
llm_type_name = config.get("type")
if llm_type_name:
loader = self.loaders.get(llm_type_name)
if loader:
return await loader(config)
selected_type = globals().get(llm_type_name)
if selected_type:
result : BaseLLMProcess = selected_type()
load_result = await result.load_from_config(config)
if load_result is False:
logger.warn(f"load LLMProcess {llm_type_name} from config failed! load_from_config return False")
return None
else:
return result
logger.warn(f"load LLMProcess {llm_type_name} from config failed! type not found")
return None
src/aios/environment/environment.py
+6
View File
@@ -102,3 +102,9 @@ class CompositeEnvironment(SimpleEnvironment):
operations = env.get_all_ai_operations()
for op in operations:
self.operations[op.get_name()] = op
def get_value(self,key:str) -> Optional[str]:
for env in self.envs:
val = env.get_value(key)
if val is not None:
return val
src/aios/proto/agent_msg.py
+5
View File
@@ -75,6 +75,11 @@ class AgentMsg:
self.inner_call_chain = []
self.resp_msg = None
self.action_list = []
#context info
self.context_info:dict= {}
@classmethod
def from_json(cls,json_obj:dict) -> 'AgentMsg':
msg = AgentMsg()
src/aios/proto/ai_function.py
+17 -5
View File
@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from typing import Dict,Coroutine,Callable
from typing import Dict,Coroutine,Callable,List
class ParameterDefine:
def __init__(self) -> None:
@@ -74,10 +74,11 @@ class AIFunction:
# pass
class ActionItem:
def __init__(self,name,args) -> None:
self.name = name
self.args = args
self.body = None
def __init__(self,name:str,args:List[str]) -> None:
self.name:str= name
self.args:List[str]= args
self.body:str = None
self.parms : Dict = None
def append_body(self,body:str) -> None:
if self.body is None:
@@ -88,6 +89,17 @@ class ActionItem:
def dumps(self) -> str:
pass
@classmethod
def from_json(cls,json_obj:dict) -> 'ActionItem':
args = json_obj.get("args",[])
r = ActionItem(json_obj["name"],args)
if json_obj.get("body"):
r.body = json_obj["body"]
r.parms = json_obj
return r
# call chain is a combination of ai_function,group of ai_function.
class CallChain:
def __init__(self) -> None:
src/aios/proto/compute_task.py
+66 -9
View File
@@ -6,7 +6,8 @@ import shlex
import uuid
import time
from typing import List, Union
from ..proto.ai_function import *
from .ai_function import *
from .agent_msg import *
from ..knowledge import ObjectID
from ..storage.storage import AIStorage
@@ -40,20 +41,63 @@ class ComputeTaskType(Enum):
TEXT_EMBEDDING ="text_embedding"
IMAGE_EMBEDDING ="image_embedding"
# class Function(TypedDict, total=False):
# name: Required[str]
# """The name of the function to be called.
# Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length
# of 64.
# """
# parameters: Required[shared_params.FunctionParameters]
# """The parameters the functions accepts, described as a JSON Schema object.
# See the [guide](https://platform.openai.com/docs/guides/gpt/function-calling)
# for examples, and the
# [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for
# documentation about the format.
# To describe a function that accepts no parameters, provide the value
# `{"type": "object", "properties": {}}`.
# """
# description: str
# """
# A description of what the function does, used by the model to choose when and
# how to call the function.
# """
class LLMPrompt:
def __init__(self,prompt_str = None) -> None:
self.messages = []
self.messages : List[Dict] = []
if prompt_str:
self.messages.append({"role":"user","content":prompt_str})
self.system_message = None
self.system_message : Dict = None
self.inner_functions : List[Dict] = []
def append_system_message(self,content:str):
if content is None:
return
if self.system_message is None:
self.system_message = {"role":"system","content":content}
else:
self.system_message["content"] += content
def append_user_message(self,content:str):
if content is None:
return
self.messages.append({"role":"user","content":content})
def as_str(self)->str:
result_str = ""
if self.system_message:
result_str += self.system_message.get("role") + ":" + self.system_message.get("content") + "\n"
result_str = json.dumps(self.system_message)
if self.messages:
for msg in self.messages:
result_str += msg.get("role") + ":" + msg.get("content") + "\n"
result_str += json.dumps(self.messages)
if self.inner_functions:
result_str += json.dumps(self.inner_functions)
return result_str
@@ -64,10 +108,18 @@ class LLMPrompt:
result.extend(self.messages)
return result
def append(self,prompt:'LLMPrompt'):
if prompt is None:
return
if prompt.inner_functions:
if self.inner_functions is None:
self.inner_functions = copy.deepcopy(prompt.inner_functions)
else:
self.inner_functions.extend(prompt.inner_functions)
if prompt.system_message is not None:
if self.system_message is None:
self.system_message = copy.deepcopy(prompt.system_message)
@@ -76,11 +128,11 @@ class LLMPrompt:
self.messages.extend(prompt.messages)
def load_from_config(self,config:list) -> bool:
def load_from_config(self,config:List[Dict]) -> bool:
if isinstance(config,list) is not True:
logger.error("prompt is not list!")
return False
self.messages = []
self.messages : List[Dict] = []
for msg in config:
if msg.get("content"):
if msg.get("role") == "system":
@@ -127,10 +179,15 @@ class LLMResult:
r.state = LLMResultStates.IGNORE
return r
r.state = LLMResultStates.OK
llm_json = json.loads(llm_json_str)
r.resp = llm_json.get("resp")
r.raw_result = llm_json
r.action_list = llm_json.get("actions")
action_list = llm_json.get("actions")
for action in action_list:
action_item = ActionItem.from_json(action)
r.action_list.append(action_item)
return r
src/component/agent_manager/agent_manager.py
+7 -2
View File
@@ -82,7 +82,12 @@ class AgentManager:
return None
the_agent.chat_db = self.db_path
return the_agent
if await the_agent.initial():
return the_agent
else:
logger.warn(f"initial agent {agent_id} failed!")
return None
def remove(self,agent_id:str)->int:
pass
@@ -141,7 +146,7 @@ class AgentManager:
else:
init_env(owner_env)
if result_agent.load_from_config(config) is False:
if await result_agent.load_from_config(config) is False:
logger.error(f"load agent from {agent_media} failed!")
return None
return result_agent
src/system.cfg.toml
+3
View File
@@ -0,0 +1,3 @@
[LLMAgentMessageProcess]
type="LLMAgentMessageProcess"