Generator prompt from session histroy base on token limit.

2023-09-20 14:45:54 -07:00
parent 9f01a1aaf7
commit bde91771a8
5 changed files with 79 additions and 17 deletions
@@ -1,6 +1,7 @@
 instance_id = "Jarvis"
 fullname = "Jarvis"
 llm_model_name = "gpt-3.5-turbo-16k-0613"
+max_token_size = 16000

 [[prompt]]
 role = "system"
@@ -19,27 +19,57 @@ logger = logging.getLogger(__name__)
 class AgentPrompt:
    def __init__(self) -> None:
        self.messages = []
+        self.system_message = None

    def as_str(self)->str:
        result_str = "" 
+        if self.system_message:
+            result_str += self.system_message.get("role") + ":" + self.system_message.get("content") + "\n"
        if self.messages:
            for msg in self.messages:
                result_str += msg.get("role") + ":" + msg.get("content") + "\n"

        return result_str
    
+    def to_message_list(self):
+        result = []
+        if self.system_message:
+            result.append(self.system_message)
+        result.extend(self.messages)
+        return result
+    
    def append(self,prompt):
        if prompt is None:
            return
        
+        if prompt.system_message is not None:
+            if self.system_message is None:
+                self.system_message = prompt.system_message
+            else:
+                self.system_message["content"] += prompt.system_message.get("content")
+
        self.messages.extend(prompt.messages)

+    def get_prompt_token_len(self):
+        result = 0
+
+        if self.system_message:
+            result += len(self.system_message.get("content"))
+        for msg in self.messages:
+            result += len(msg.get("content"))
+
+        return result
+
    def load_from_config(self,config:list) -> bool:
        if isinstance(config,list) is not True:
            logger.error("prompt is not list!")
            return False
-        
-        self.messages = config
+        self.messages = []
+        for msg in config:
+            if msg.get("role") == "system":
+                self.system_message = msg
+            else:
+                self.messages.append(msg)
        return True


@@ -203,16 +233,18 @@ class AIAgent:
            return None
        
        result_func = []
+        result_len = 0
        for inner_func in all_inner_function:
            this_func = {}
            this_func["name"] = inner_func.get_name()
            this_func["description"] = inner_func.get_description()
            this_func["parameters"] = inner_func.get_parameters()
+            result_len += len(json.dumps(this_func)) / 4
            result_func.append(this_func)

-        return result_func 
+        return result_func,result_len

-    async def _execute_func(self,inenr_func_call_node:dict,prompt:AgentPrompt,org_msg:AgentMsg) -> str:
+    async def _execute_func(self,inenr_func_call_node:dict,prompt:AgentPrompt,org_msg:AgentMsg,stack_limit = 5) -> str:
        from .compute_kernel import ComputeKernel

        func_name = inenr_func_call_node.get("name")
@@ -231,7 +263,7 @@ class AIAgent:
            logger.error(f"llm execute inner func:{func_name} error:{e}")
            

-        inner_functions = self._get_inner_functions()
+        inner_functions,inner_function_len = self._get_inner_functions()
        prompt.messages.append({"role":"function","content":result_str,"name":func_name})
        task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,self.llm_model_name,self.max_token_size,inner_functions)
        
@@ -239,9 +271,11 @@ class AIAgent:
        ineternal_call_record.done_time = time.time()
        org_msg.inner_call_chain.append(ineternal_call_record)

-        inner_func_call_node = task_result.result_message.get("function_call")
+        if stack_limit > 0:
+            inner_func_call_node = task_result.result_message.get("function_call")
+            
        if inner_func_call_node:
-            return await self._execute_func(inner_func_call_node,prompt,org_msg)      
+            return await self._execute_func(inner_func_call_node,prompt,org_msg,stack_limit-1)      
        else:
            return task_result.result_str

@@ -270,16 +304,20 @@ class AIAgent:
            
            prompt = AgentPrompt()
            prompt.append(await self._get_agent_prompt())
+            inner_functions,function_token_len = self._get_inner_functions()
            # prompt.append(self._get_knowlege_prompt(the_role.get_name()))
-            prompt.append(await self._get_prompt_from_session(chatsession)) # chat context
+            system_prompt_len = prompt.get_prompt_token_len()
+            input_len = len(msg.body)
+            
+            history_prmpt,history_token_len = await self._get_prompt_from_session(chatsession,system_prompt_len + function_token_len,input_len)
+            prompt.append(history_prmpt) # chat context
            
            msg_prompt = AgentPrompt()
            msg_prompt.messages = [{"role":"user","content":msg.body}]
            prompt.append(msg_prompt)

            self._format_msg_by_env_value(prompt)
-            inner_functions = self._get_inner_functions()
-
+            logger.info(f"Agent {self.agent_id} do llm token static system:{system_prompt_len},function:{function_token_len},history:{history_token_len},input:{input_len}")
            task_result:ComputeTaskResult = await ComputeKernel.get_instance().do_llm_completion(prompt,self.llm_model_name,self.max_token_size,inner_functions)
            final_result = task_result.result_str

@@ -332,14 +370,26 @@ class AIAgent:
    def get_max_token_size(self) -> int:
        return self.max_token_size
    
-    async def _get_prompt_from_session(self,chatsession:AIChatSession,is_groupchat=False) -> AgentPrompt:
+    async def _get_prompt_from_session(self,chatsession:AIChatSession,system_token_len,input_token_len,is_groupchat=False) -> AgentPrompt:
        # TODO: get prompt from group chat is different from single chat
+        history_len = (self.max_token_size * 0.7) - system_token_len - input_token_len
        messages = chatsession.read_history() # read 
+        result_token_len = 0
        result_prompt = AgentPrompt()
+        read_history_msg = 0
        for msg in reversed(messages):
+            read_history_msg += 1
            if msg.sender == self.agent_id:
                result_prompt.messages.append({"role":"assistant","content":msg.body})
+                
            else:
                result_prompt.messages.append({"role":"user","content":msg.body})
-        return result_prompt
+
+            history_len -= len(msg.body)
+            result_token_len += len(msg.body)
+            if history_len < 0:
+                logger.warning(f"_get_prompt_from_session reach limit of token,just read {read_history_msg} history message.")
+                break
+
+        return result_prompt,result_token_len

@@ -41,7 +41,7 @@ class ComputeTask:
        self.create_time = time.time()
        self.task_id = uuid.uuid4().hex
        self.callchain_id = callchain_id
-        self.params["prompts"] = prompts.messages
+        self.params["prompts"] = prompts.to_message_list()
        if model_name is not None:
            self.params["model_name"] = model_name
        else:
@@ -78,7 +78,7 @@ class ComputeTaskResult:
        self.result_code: int = 0
        self.result_str: str = None # easy to use,can read from result
        self.result_message: dict = {}
-        self.result_refers: dict = None
+        self.result_refers: dict = {}
        self.pading_data: bytearray = None

    def set_from_task(self, task: ComputeTask):
@@ -100,18 +100,20 @@ class OpenAI_ComputeNode(ComputeNode):
                if max_token_size is None:
                    max_token_size = 4000

+                result_token = int(max_token_size * 0.4)
+                    
                logger.info(f"call openai {mode_name} prompts: {prompts}")

                if llm_inner_functions is None:
                    resp = openai.ChatCompletion.create(model=mode_name,
                                                    messages=prompts,
-                                                    max_tokens=max_token_size,
+                                                    max_tokens=result_token,
                                                    temperature=0.7)
                else:
                    resp = openai.ChatCompletion.create(model=mode_name,
                                                        messages=prompts,
                                                        functions=llm_inner_functions,
-                                                        max_tokens=max_token_size,
+                                                        max_tokens=result_token,
                                                        temperature=0.7) # TODO: add temperature to task params?

            
@@ -121,6 +123,7 @@ class OpenAI_ComputeNode(ComputeNode):
                result.set_from_task(task)

                status_code = resp["choices"][0]["finish_reason"]
+                token_usage = resp.get("usage")
                match status_code:
                    case "function_call":
                        task.state = ComputeTaskState.DONE
@@ -134,6 +137,8 @@ class OpenAI_ComputeNode(ComputeNode):
                result.worker_id = self.node_id
                result.result_str = resp["choices"][0]["message"]["content"]
                result.result_message = resp["choices"][0]["message"]
+                if token_usage:
+                    result.result_refers["token_usage"] = token_usage
                return result
            case _:
                task.state = ComputeTaskState.ERROR
@@ -106,7 +106,7 @@ class CalenderEnvironment(Environment):
                VALUES (?, ?, ?, ?, ?, ?);
            """, (title, start_time, end_time, participants, location, details))
            await db.commit()
-            return "Add event ok"
+            return f"execute add_event OK,event '{title}' already add to calender!"

    async def _search_events(self,query):
        async with aiosqlite.connect(self.db_file) as db:
@@ -137,7 +137,9 @@ class CalenderEnvironment(Environment):
            rows = await cursor.fetchall()

            result = {}
+            have_result = False
            for row in rows:
+                have_result = True
                _event = {}
                _event["title"] = row[1]
                _event["start_time"] = row[2]
@@ -146,6 +148,10 @@ class CalenderEnvironment(Environment):
                _event["location"] = row[5]
                _event["details"] = row[6]
                result[row[0]] = _event
+            
+            if not have_result:
+                return "No event."
+            
            return json.dumps(result, indent=4, sort_keys=True)
       
    async def _update_event(self,event_id, new_title=None, new_participants=None, new_location=None, new_details=None ,start_time=None, end_time=None):