From 0a4feb1daf881e273907a4233ae85e2744c1b948 Mon Sep 17 00:00:00 2001 From: fiatrete Date: Fri, 16 Jun 2023 15:58:57 +0800 Subject: [PATCH] use gpt function --- agent_jarvis/jarvis/ai_agent/gpt_agent.py | 165 ++++++------------ agent_jarvis/jarvis/ai_agent/webui_agent.py | 4 +- .../functional_modules/functional_module.py | 28 ++- agent_jarvis/jarvis/gpt/ai_function.py | 5 +- agent_jarvis/jarvis/gpt/gpt.py | 23 ++- agent_jarvis/jarvis/gpt/token_counter.py | 9 +- .../demo_modules/google_calendar.module.py | 22 ++- .../demo_modules/stable_diffusion.module.py | 13 +- .../demo_modules/twitter.module.py | 8 +- .../demo_modules/youtube.module.py | 28 ++- example_modules/light_switch/switch.module.py | 11 +- 11 files changed, 181 insertions(+), 135 deletions(-) diff --git a/agent_jarvis/jarvis/ai_agent/gpt_agent.py b/agent_jarvis/jarvis/ai_agent/gpt_agent.py index a6d8368..8b7011b 100644 --- a/agent_jarvis/jarvis/ai_agent/gpt_agent.py +++ b/agent_jarvis/jarvis/ai_agent/gpt_agent.py @@ -17,27 +17,10 @@ from jarvis.logger import logger def _generate_first_prompt(): - return """Since now, every your response should satisfy the following JSON format, a 'function' must be chosen: -``` -{ - "thoughts": { - "text": "", - "reasoning": "", - "speak": "" - }, - "function": { - "name": "", - "args": { - "arg name": "" - } - } -} -``` - -I will ask you questions or ask you to do something. You should: -First, you should determine if you know the answer of the question or you can accomplish the task directly. -If so, you should response directly. -If not, you should try to complete the task by calling the functions below. + return """I will ask you questions or ask you to do something. You should: +First, determine if you know the answer of the question or you can accomplish the task directly. +If so, response directly. +If not, try to complete the task by calling the functions below. If you can't accomplish the task by yourself and no function is able to accomplish the task, say "Dear master, sorry, I'm not able to do that." Your setup: @@ -46,27 +29,6 @@ Your setup: "author": "OpenDAN", "name": "Jarvis", } -``` -Available functions: -``` -""" + moduleRegistry.to_prompt() + """ -``` -Example: -``` -me: generate a picture of me. -you: { - "thoughts": { - "text": "You need a picture of 'me'", - "reasoning": "stable_diffusion is able to generate pictures", - "speak": "Ok, I will do that" - }, - "function": { - "name": "stable_diffusion", - "args": { - "prompt": "me" - } - } -} ```""" @@ -79,47 +41,25 @@ class GptAgent(BaseAgent): super().__init__(caller_context) self._system_prompt = _generate_first_prompt() logger.debug(f"Using GptAgent, system prompt is: {self._system_prompt}") + logger.debug(f"{json.dumps(moduleRegistry.to_json_schema())}") async def _feed_prompt_to_get_response(self, prompt): - assistant_reply = await self._chat_with_ai( + reply_type, assistant_reply = await self._chat_with_ai( self._system_prompt, prompt, CFG.token_limit, ) - reply = { - "thoughts": None, - "reasoning": None, - "speak": None, - "function": None, - "arguments": None, - } - - if must_not_be_valid_json(assistant_reply): - raise Exception(f"AI replied an invalid response: {assistant_reply}!") - else: - assistant_reply_json = await fix_json_using_multiple_techniques(assistant_reply) - - # Print Assistant thoughts - if assistant_reply_json != {}: - validate_json(assistant_reply_json, "llm_response_format_1") - try: - get_thoughts(reply, assistant_reply_json) - get_function(reply, assistant_reply_json) - except Exception as e: - logger.error(f"AI replied an invalid response: {assistant_reply}. Error: {str(e)}") - raise e - else: - raise Exception(f"AI replied an invalid response: {assistant_reply}!") - - function_name = reply["function"] - if function_name is None or function_name == '': - raise Exception(f"Missing a function") - arguments = reply["arguments"] - - if not isinstance(arguments, dict): - raise Exception(f"Invalid arguments, it MUST be a dict") - return reply + if reply_type == "content": + return { + "speak": assistant_reply, + } + elif reply_type == "function_call": + # TODO: Check arguments + return { + "function": assistant_reply["name"], + "arguments": json.loads(assistant_reply["arguments"]) + } async def feed_prompt(self, prompt): # Send message to AI, get response @@ -146,35 +86,36 @@ class GptAgent(BaseAgent): return # Execute function - function_name: str = reply["function"] - arguments: Dict = reply["arguments"] + function_name: str = reply.get("function") + if function_name is None: + await self._caller_context.reply_text(reply["speak"]) + pass + else: + arguments: Dict = reply["arguments"] - await self._caller_context.reply_text(reply["speak"]) - execute_error = None - try: - function_result = await execute_function(self._caller_context, function_name, **arguments) - except Exception as e: function_result = "Failed" - execute_error = e - result = f"Function {function_name} returned: " f"{function_result}" + try: + function_result = await execute_function(self._caller_context, function_name, **arguments) + finally: + result = f"{function_result}" - if function_name is not None: - # Check if there's a result from the function append it to the message - # history - if result is not None: - self._caller_context.append_history_message("system", result) - logger.debug(f"SYSTEM: {result}") - else: - self._caller_context.append_history_message("system", "Unable to execute function") - logger.debug("SYSTEM: Unable to execute function") - - if execute_error is not None: - raise execute_error + # Check if there's a result from the function append it to the message + # history + if result is not None: + self.append_history_message_raw({"role": "function", "name": function_name, "content": result}) + logger.debug(f"function: {result}") + else: + self.append_history_message_raw({"role": "function", "name": function_name, "content": "Unable to execute function"}) + logger.debug("function: Unable to execute function") def append_history_message(self, role: str, content: str): self._full_message_history.append({'role': role, 'content': content}) self._message_tokens.append(-1) + def append_history_message_raw(self, msg: dict): + self._full_message_history.append(msg) + self._message_tokens.append(-1) + def clear_history_messages(self): self._full_message_history.clear() self._message_tokens.clear() @@ -216,7 +157,7 @@ class GptAgent(BaseAgent): ) = await self._generate_context(prompt, model) current_tokens_used += await token_counter.count_message_tokens( - [create_chat_message("user", user_input)], model + [{"role": "user", "content": user_input}], model ) # Account for user input (appended later) while next_message_to_add_index >= 0: @@ -237,7 +178,7 @@ class GptAgent(BaseAgent): next_message_to_add_index -= 1 # Append user input, the length of this is accounted for above - current_context.extend([create_chat_message("user", user_input)]) + current_context.extend([{"role": "user", "content": user_input}]) # Calculate remaining tokens tokens_remaining = token_limit - current_tokens_used @@ -248,19 +189,29 @@ class GptAgent(BaseAgent): await self._caller_context.push_notification( f'Thinking timeout{", retry" if will_retry else ", give up"}.') - assistant_reply = await gpt.acreate_chat_completion( + + reply_type, assistant_reply = await gpt.acreate_chat_completion( model=model, messages=current_context, temperature=CFG.temperature, max_tokens=tokens_remaining, - on_single_request_timeout=on_single_chat_timeout + on_single_request_timeout=on_single_chat_timeout, + functions=moduleRegistry.to_json_schema() ) # Update full message history - self._caller_context.append_history_message("user", user_input) - self._caller_context.append_history_message("assistant", assistant_reply) + if reply_type == "content": + self.append_history_message("user", user_input) + self.append_history_message("assistant", assistant_reply) + pass + elif reply_type == "function_call": + self.append_history_message("user", user_input) + self.append_history_message_raw({"role": "assistant", "function_call": assistant_reply, "content": None}) + pass + else: + assert False, "Unexpected reply type" - return assistant_reply + return reply_type, assistant_reply except RateLimitError: # TODO: When we switch to langchain, or something else this is built in print("Error: ", "API Rate Limit Reached. Waiting 10 seconds...") @@ -271,10 +222,8 @@ class GptAgent(BaseAgent): timestamp = time.time() + time.timezone + self._caller_context.get_tz_offset() * 3600 time_str = time.strftime('%c', time.localtime(timestamp)) current_context = [ - create_chat_message("system", prompt), - create_chat_message( - "system", f"The current time and date is {time_str}" - ) + {"role": "system", "content": prompt}, + {"role": "system", "content": f"The current time and date is {time_str}"}, ] # Add messages from the full message history until we reach the token limit diff --git a/agent_jarvis/jarvis/ai_agent/webui_agent.py b/agent_jarvis/jarvis/ai_agent/webui_agent.py index abffc16..09bf5a7 100644 --- a/agent_jarvis/jarvis/ai_agent/webui_agent.py +++ b/agent_jarvis/jarvis/ai_agent/webui_agent.py @@ -182,10 +182,10 @@ class WebuiAgent(BaseAgent): if function_name is not None: if result is not None: - self._caller_context.append_history_message("system", result) + self.append_history_message("system", result) logger.debug(f"SYSTEM: {result}") else: - self._caller_context.append_history_message("system", "Unable to execute function") + self.append_history_message("system", "Unable to execute function") logger.debug("SYSTEM: Unable to execute function") if execute_error is not None: diff --git a/agent_jarvis/jarvis/functional_modules/functional_module.py b/agent_jarvis/jarvis/functional_modules/functional_module.py index 2ba8e0f..65e0434 100644 --- a/agent_jarvis/jarvis/functional_modules/functional_module.py +++ b/agent_jarvis/jarvis/functional_modules/functional_module.py @@ -12,7 +12,7 @@ class FunctionalModule: name: str description: str method: Callable[..., Any] - signature: dict[str, str] + signature: dict[str, dict] def __init__(self, name, description, method, signature): self.name = name @@ -39,8 +39,8 @@ class FunctionalModuleRegistry: })) @staticmethod - def _signature_to_string(signature: dict[str, str]): - return ", ".join([f"{k}: <{v}>" for k, v in signature.items()]) + def _signature_to_string(signature: dict[str, dict]): + return ", ".join([f"{k}: <{v['description']}>" for k, v in signature.items()]) def to_prompt(self): text = "" @@ -56,6 +56,26 @@ class FunctionalModuleRegistry: return text + def to_json_schema(self): + return [ + { + "name": module.name, + "description": module.description, + "parameters": { + "type": "object", + "properties": { + key: { + k: v for k, v in value.items() if k != "required" + } + for key, value in module.signature.items() + }, + "required": [key for key, value in module.signature.items() if value.get("required") != False] + } + + } + for module in sorted(self._modules.values(), key=lambda cmd: cmd.name) + ] + async def execute_function(self, context: CallerContext, function_name: str, **kwargs): cmd = self._modules.get(function_name) if cmd is not None: @@ -68,7 +88,7 @@ moduleRegistry = FunctionalModuleRegistry() def functional_module(name: str, description: str, - signature=None): + signature: dict[str, dict] = None): if signature is None: signature = {} diff --git a/agent_jarvis/jarvis/gpt/ai_function.py b/agent_jarvis/jarvis/gpt/ai_function.py index 1bcdcb3..e5a2753 100644 --- a/agent_jarvis/jarvis/gpt/ai_function.py +++ b/agent_jarvis/jarvis/gpt/ai_function.py @@ -37,4 +37,7 @@ async def acall_ai_function(function: str, args: list, description: str, model: logger.debug(str(messages)) - return await gpt.acreate_chat_completion(model=model, messages=messages, temperature=0) + msg_type, msg_content = await gpt.acreate_chat_completion(model=model, messages=messages, temperature=0) + if msg_type == "content": + return msg_content + return 'failed' diff --git a/agent_jarvis/jarvis/gpt/gpt.py b/agent_jarvis/jarvis/gpt/gpt.py index 3d13031..1ca8ac2 100644 --- a/agent_jarvis/jarvis/gpt/gpt.py +++ b/agent_jarvis/jarvis/gpt/gpt.py @@ -21,6 +21,7 @@ async def acreate_chat_completion_once( max_tokens: int | None = None, deployment_id=None, request_timeout=40, + **kwargs ) -> str: """ Create a chat completion and update the cost. @@ -39,7 +40,8 @@ async def acreate_chat_completion_once( messages=messages, temperature=temperature, max_tokens=max_tokens, - request_timeout=request_timeout + request_timeout=request_timeout, + **kwargs ) else: response = await openai.ChatCompletion.acreate( @@ -47,7 +49,8 @@ async def acreate_chat_completion_once( messages=messages, temperature=temperature, max_tokens=max_tokens, - request_timeout=request_timeout + request_timeout=request_timeout, + **kwargs ) if CFG.debug_mode: logger.debug(f"Response: {response}") @@ -65,12 +68,13 @@ async def acreate_chat_completion( max_tokens: int = None, request_timeout: int = 40, num_retries=3, - on_single_request_timeout: Callable = None + on_single_request_timeout: Callable = None, + **kwargs ): """Create a chat completion using the OpenAI API Args: - messages (List[Message]): The messages to send to the chat completion + messages (List[dict]): The messages to send to the chat completion model (str, optional): The model to use. Defaults to None. temperature (float, optional): The temperature to use. Defaults to 0.9. max_tokens (int, optional): The max tokens to use. Defaults to None. @@ -100,6 +104,7 @@ async def acreate_chat_completion( temperature=temperature, max_tokens=max_tokens, request_timeout=request_timeout, + **kwargs ) else: response = await acreate_chat_completion_once( @@ -108,6 +113,7 @@ async def acreate_chat_completion( temperature=temperature, max_tokens=max_tokens, request_timeout=request_timeout, + **kwargs ) break except RateLimitError: @@ -129,5 +135,10 @@ async def acreate_chat_completion( if response is None: logger.error(f"Failed to get response from GPT after {num_retries} retries") raise RuntimeError(f"Failed to get response after {num_retries} retries") - resp = response.choices[0].message["content"] - return resp + + choice_message = response.choices[0].message + content = choice_message.get("content") + if content is None: + return "function_call", {k: v for k, v in choice_message["function_call"].items()} + else: + return "content", content diff --git a/agent_jarvis/jarvis/gpt/token_counter.py b/agent_jarvis/jarvis/gpt/token_counter.py index 2e661d5..9494912 100644 --- a/agent_jarvis/jarvis/gpt/token_counter.py +++ b/agent_jarvis/jarvis/gpt/token_counter.py @@ -2,6 +2,7 @@ from __future__ import annotations from typing import List +import json import tiktoken_async @@ -33,7 +34,8 @@ async def count_message_tokens( elif model == "gpt-4": # !Note: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.") return await count_message_tokens(messages, model="gpt-4-0314") - elif model == "gpt-3.5-turbo-0301": + # TODO: OpenAI has not mention how to count tokens for 0613, thus, we use the former method + elif model == "gpt-3.5-turbo-0301" or model == "gpt-3.5-turbo-0613" or model == "gpt-3.5-turbo-16k-0613": tokens_per_message = ( 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n ) @@ -51,6 +53,11 @@ async def count_message_tokens( for message in messages: num_tokens += tokens_per_message for key, value in message.items(): + if not isinstance(value, str): + # TODO: Since openai does not mentioned how to count tokens of 'funciton_call', + # and only string is countable, thus, if the value is not a `str` (`function_call` + # field of a message), we convert it into json + value = json.dumps(value) num_tokens += len(encoding.encode(value)) if key == "name": num_tokens += tokens_per_name diff --git a/example_modules/demo_modules/google_calendar.module.py b/example_modules/demo_modules/google_calendar.module.py index f5074f6..944e5f3 100644 --- a/example_modules/demo_modules/google_calendar.module.py +++ b/example_modules/demo_modules/google_calendar.module.py @@ -18,7 +18,18 @@ def reg_or_not(): @functional_module( name="add_alarm", description="Create an alarm", - signature={"date": "The alarm date, 'YYYY-mm-dd HH:MM:SS format'", "desc": "The event description"}) + signature={ + "date": { + "description": "The alarm date, 'YYYY-mm-dd HH:MM:SS format'", + "type": "string", + "required": True + }, + "desc": { + "description": "The event description", + "type": "string", + "required": True + } + }) async def add_alarm(context: CallerContext, date, desc): # date = "2023-05-10 14:56:59" now = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S").timestamp() @@ -43,7 +54,14 @@ def reg_or_not(): @functional_module( name="delete_alarm", description="delete all alarms whose ID is in the list", - signature={"IDs": "A list of alarm IDs to"}) + signature={ + "IDs": { + "type": "array", + "items": { "type": "string" }, + "description": "A list of alarm IDs to delete", + "required": True + } + }) async def delete_alarm(context: CallerContext, IDs: List[str]): # get all tasks result = await do_get(google_calendar_service_address + "/tasks") diff --git a/example_modules/demo_modules/stable_diffusion.module.py b/example_modules/demo_modules/stable_diffusion.module.py index 6cd0461..2a76864 100644 --- a/example_modules/demo_modules/stable_diffusion.module.py +++ b/example_modules/demo_modules/stable_diffusion.module.py @@ -113,7 +113,7 @@ Sometimes you maybe asked to generate a pic of myself. That means you MUST add ' - Add unique touches to each output, making it lengthy, detailed, and stylized. - Show, don't tell; instead of tagging \"exceptional artwork\" or \"emphasizing a beautiful ...\" provide - precise details. - Ensure the output is placed inside a beautiful and stylized markdown. -- The prompt you return MUST be English. The lenth of prompt MUST less than 150. +- The prompt you return MUST be English. The tokens of prompt MUST less than 70. """ OTHER_SD_PARAMS_NAME = "other" @@ -153,7 +153,7 @@ NOTE: Just reply using these information, don't ask me anything. sys_prompt = {'role': 'system', 'content': gpt_system_prompt} messages = [sys_prompt, {'role': 'user', 'content': prompt}] model = CFG.small_llm_model - resp = await acreate_chat_completion( + _, resp = await acreate_chat_completion( messages, model, temperature=0, @@ -207,7 +207,7 @@ NOTE: Just reply using these information, don't ask me anything. messages = [sys_prompt, {'role': 'user', 'content': "Generation " + origin_str}] model = CFG.small_llm_model try: - resp = await acreate_chat_completion( + _, resp = await acreate_chat_completion( messages, model, temperature=0, @@ -239,7 +239,12 @@ NOTE: Just reply using these information, don't ask me anything. @functional_module( name="stable_diffusion", description="Generate a picture.", - signature={'prompt': 'the description I told you'}) + signature={ + 'prompt': { + "type": "string", + "description": 'the description I told you' + } + }) async def stable_diffusion(context: CallerContext, prompt: str): await context.reply_text("I'm generating the image, this may take a while.") style = await determine_style(prompt) diff --git a/example_modules/demo_modules/twitter.module.py b/example_modules/demo_modules/twitter.module.py index 97afa56..b1acb2d 100644 --- a/example_modules/demo_modules/twitter.module.py +++ b/example_modules/demo_modules/twitter.module.py @@ -14,8 +14,12 @@ def reg_or_not(): @functional_module( name="post_tweet", description="post a tweet", - signature={"content": "the content of the tweet"} - ) + signature={ + "content": { + "type": "string", + "description": "the content of the tweet" + } + }) async def post_tweet(context: CallerContext, content: str): response = await do_post(twitter_service_address + "/twitter/tweet_post", '', {"content": content}) logger.info(f"response: {response}") diff --git a/example_modules/demo_modules/youtube.module.py b/example_modules/demo_modules/youtube.module.py index 9d89543..6432211 100644 --- a/example_modules/demo_modules/youtube.module.py +++ b/example_modules/demo_modules/youtube.module.py @@ -14,7 +14,12 @@ def reg_or_not(): @functional_module( name="youtube_video_brief", description="Get the brief content of a youtube video", - signature={"url": "The address of the video"}) + signature={ + "url": { + "type": "string", + "description": "The address of the video" + } + }) async def youtube_video_brief(context: CallerContext, url: str): await context.push_notification(f"One second... I'm watching this video: {url}") if not url.startswith('https://www.youtube.com/watch?'): @@ -38,7 +43,12 @@ def reg_or_not(): @functional_module( name="youtube_video_brief_vid", description="Get the brief content of a youtube video identified by video id", - signature={"video_id": "The video id of the video"}) + signature={ + "video_id": { + "type": "string", + "description": "The video id of the video" + } + }) async def youtube_video_brief_vid(context: CallerContext, video_id: str): url = f'https://www.youtube.com/watch?v={video_id}' await context.push_notification(f"One second... I'm watching this video: {url}") @@ -70,7 +80,12 @@ def reg_or_not(): @functional_module( name="youtube_x_video_info", description="Get the basic information of a youtube user's newest videos, when the summary of videos are not required, you should use this function", - signature={"username": "The username"}) + signature={ + "username": { + "type": "string", + "description": "The username" + } + }) async def youtube_x_video_info(context: CallerContext, username: str): response = await youtube_latest_video_info_of(context, username, False) result = f'The brief content of the latest videos of {username} are:\n' @@ -86,7 +101,12 @@ def reg_or_not(): @functional_module( name="youtube_notify_new", description="Watching an Youtuber, push an notification when the youtuber published a new video", - signature={"username": "The username"}) + signature={ + "username": { + "type": "string", + "description": "The username" + } + }) async def youtube_notify_new(context: CallerContext, username: str): if username.startswith('@'): username = username[1:] diff --git a/example_modules/light_switch/switch.module.py b/example_modules/light_switch/switch.module.py index 58f1f2c..6a67b79 100644 --- a/example_modules/light_switch/switch.module.py +++ b/example_modules/light_switch/switch.module.py @@ -4,7 +4,16 @@ from jarvis.functional_modules.functional_module import functional_module, Calle @functional_module( name="toggle_light", description="Turn on/off the light.", - signature={"room": "The room name", "on": "Turn on or off, bool type>"}) + signature={ + "room": { + "type": "string", + "description": "The room name" + }, + "on": { + "type": "boolean", + "description": "Turn on or off" + } + }) async def light_switch(context: CallerContext, room: str, on: bool): # Do the actual control here, something like this # room_id = convert_room_name_to_id(room)