use gpt function

agent_jarvis/jarvis/ai_agent/gpt_agent.py

@@ -17,27 +17,10 @@ from jarvis.logger import logger
 
 
 def _generate_first_prompt():
-    return """Since now, every your response should satisfy the following JSON format, a 'function' must be chosen:
+    return """I will ask you questions or ask you to do something. You should:
-```
+First, determine if you know the answer of the question or you can accomplish the task directly. 
-{
+If so, response directly. 
-    "thoughts": {
+If not, try to complete the task by calling the functions below.
-        "text": "<Your thought>",
-        "reasoning": "<Your reasoning>",
-        "speak": "<what you want to say to me>"
-    },
-    "function": {
-        "name": "<mandatory, one of listed functions>",
-        "args": {
-            "arg name": "<value>"
-        }
-    }
-}
-```
-
-I will ask you questions or ask you to do something. You should:
-First, you should determine if you know the answer of the question or you can accomplish the task directly. 
-If so, you should response directly. 
-If not, you should try to complete the task by calling the functions below.
 If you can't accomplish the task by yourself and no function is able to accomplish the task, say "Dear master, sorry, I'm not able to do that."
 
 Your setup:
@@ -46,27 +29,6 @@ Your setup:
     "author": "OpenDAN",
     "name": "Jarvis",
 }
-```
-Available functions:
-```
-""" + moduleRegistry.to_prompt() + """
-```
-Example:
-```
-me: generate a picture of me.
-you: {
-    "thoughts": {
-        "text": "You need a picture of 'me'",
-        "reasoning": "stable_diffusion is able to generate pictures",
-        "speak": "Ok, I will do that"
-    },
-    "function": {
-        "name": "stable_diffusion",
-        "args": {
-            "prompt": "me"
-        }
-    }
-}
 ```"""
 
 
@@ -79,47 +41,25 @@ class GptAgent(BaseAgent):
         super().__init__(caller_context)
         self._system_prompt = _generate_first_prompt()
         logger.debug(f"Using GptAgent, system prompt is: {self._system_prompt}")
+        logger.debug(f"{json.dumps(moduleRegistry.to_json_schema())}")
 
     async def _feed_prompt_to_get_response(self, prompt):
-        assistant_reply = await self._chat_with_ai(
+        reply_type, assistant_reply = await self._chat_with_ai(
             self._system_prompt,
             prompt,
             CFG.token_limit,
         )
 
-        reply = {
+        if reply_type == "content":
-            "thoughts": None,
+            return {
-            "reasoning": None,
+                "speak": assistant_reply,
-            "speak": None,
+            }
-            "function": None,
+        elif reply_type == "function_call":
-            "arguments": None,
+            # TODO: Check arguments
+            return {
+                "function": assistant_reply["name"],
+                "arguments": json.loads(assistant_reply["arguments"])
             }
-
-        if must_not_be_valid_json(assistant_reply):
-            raise Exception(f"AI replied an invalid response: {assistant_reply}!")
-        else:
-            assistant_reply_json = await fix_json_using_multiple_techniques(assistant_reply)
-
-        # Print Assistant thoughts
-        if assistant_reply_json != {}:
-            validate_json(assistant_reply_json, "llm_response_format_1")
-            try:
-                get_thoughts(reply, assistant_reply_json)
-                get_function(reply, assistant_reply_json)
-            except Exception as e:
-                logger.error(f"AI replied an invalid response: {assistant_reply}. Error: {str(e)}")
-                raise e
-        else:
-            raise Exception(f"AI replied an invalid response: {assistant_reply}!")
-
-        function_name = reply["function"]
-        if function_name is None or function_name == '':
-            raise Exception(f"Missing a function")
-        arguments = reply["arguments"]
-
-        if not isinstance(arguments, dict):
-            raise Exception(f"Invalid arguments, it MUST be a dict")
-        return reply
 
     async def feed_prompt(self, prompt):
         # Send message to AI, get response
@@ -146,35 +86,36 @@ class GptAgent(BaseAgent):
             return
 
         # Execute function
-        function_name: str = reply["function"]
+        function_name: str = reply.get("function")
+        if function_name is None:
+            await self._caller_context.reply_text(reply["speak"])
+            pass
+        else:
             arguments: Dict = reply["arguments"]
 
-        await self._caller_context.reply_text(reply["speak"])
+            function_result = "Failed"
-        execute_error = None
             try:
                 function_result = await execute_function(self._caller_context, function_name, **arguments)
-        except Exception as e:
+            finally:
-            function_result = "Failed"
+                result = f"{function_result}"
-            execute_error = e
-        result = f"Function {function_name} returned: " f"{function_result}"
 
-        if function_name is not None:
                 # Check if there's a result from the function append it to the message
                 # history
                 if result is not None:
-                self._caller_context.append_history_message("system", result)
+                    self.append_history_message_raw({"role": "function", "name": function_name, "content": result})
-                logger.debug(f"SYSTEM: {result}")
+                    logger.debug(f"function: {result}")
                 else:
-                self._caller_context.append_history_message("system", "Unable to execute function")
+                    self.append_history_message_raw({"role": "function", "name": function_name, "content": "Unable to execute function"})
-                logger.debug("SYSTEM: Unable to execute function")
+                    logger.debug("function: Unable to execute function")
-
-        if execute_error is not None:
-            raise execute_error
 
     def append_history_message(self, role: str, content: str):
         self._full_message_history.append({'role': role, 'content': content})
         self._message_tokens.append(-1)
 
+    def append_history_message_raw(self, msg: dict):
+        self._full_message_history.append(msg)
+        self._message_tokens.append(-1)
+
     def clear_history_messages(self):
         self._full_message_history.clear()
         self._message_tokens.clear()
@@ -216,7 +157,7 @@ class GptAgent(BaseAgent):
                 ) = await self._generate_context(prompt, model)
 
                 current_tokens_used += await token_counter.count_message_tokens(
-                    [create_chat_message("user", user_input)], model
+                    [{"role": "user", "content": user_input}], model
                 )  # Account for user input (appended later)
 
                 while next_message_to_add_index >= 0:
@@ -237,7 +178,7 @@ class GptAgent(BaseAgent):
                     next_message_to_add_index -= 1
 
                 # Append user input, the length of this is accounted for above
-                current_context.extend([create_chat_message("user", user_input)])
+                current_context.extend([{"role": "user", "content": user_input}])
 
                 # Calculate remaining tokens
                 tokens_remaining = token_limit - current_tokens_used
@@ -248,19 +189,29 @@ class GptAgent(BaseAgent):
                     await self._caller_context.push_notification(
                         f'Thinking timeout{", retry" if will_retry else ", give up"}.')
 
-                assistant_reply = await gpt.acreate_chat_completion(
+
+                reply_type, assistant_reply = await gpt.acreate_chat_completion(
                     model=model,
                     messages=current_context,
                     temperature=CFG.temperature,
                     max_tokens=tokens_remaining,
-                    on_single_request_timeout=on_single_chat_timeout
+                    on_single_request_timeout=on_single_chat_timeout,
+                    functions=moduleRegistry.to_json_schema()
                 )
 
                 # Update full message history
-                self._caller_context.append_history_message("user", user_input)
+                if reply_type == "content":
-                self._caller_context.append_history_message("assistant", assistant_reply)
+                    self.append_history_message("user", user_input)
+                    self.append_history_message("assistant", assistant_reply)
+                    pass
+                elif reply_type == "function_call":
+                    self.append_history_message("user", user_input)
+                    self.append_history_message_raw({"role": "assistant", "function_call": assistant_reply, "content": None})
+                    pass
+                else:
+                    assert False, "Unexpected reply type"
 
-                return assistant_reply
+                return reply_type, assistant_reply
             except RateLimitError:
                 # TODO: When we switch to langchain, or something else this is built in
                 print("Error: ", "API Rate Limit Reached. Waiting 10 seconds...")
@@ -271,10 +222,8 @@ class GptAgent(BaseAgent):
         timestamp = time.time() + time.timezone + self._caller_context.get_tz_offset() * 3600
         time_str = time.strftime('%c', time.localtime(timestamp))
         current_context = [
-            create_chat_message("system", prompt),
+            {"role": "system", "content": prompt},
-            create_chat_message(
+            {"role": "system", "content": f"The current time and date is {time_str}"},
-                "system", f"The current time and date is {time_str}"
-            )
         ]
 
         # Add messages from the full message history until we reach the token limit

agent_jarvis/jarvis/ai_agent/webui_agent.py

@@ -182,10 +182,10 @@ class WebuiAgent(BaseAgent):
 
         if function_name is not None:
             if result is not None:
-                self._caller_context.append_history_message("system", result)
+                self.append_history_message("system", result)
                 logger.debug(f"SYSTEM: {result}")
             else:
-                self._caller_context.append_history_message("system", "Unable to execute function")
+                self.append_history_message("system", "Unable to execute function")
                 logger.debug("SYSTEM: Unable to execute function")
 
         if execute_error is not None:

agent_jarvis/jarvis/functional_modules/functional_module.py

@@ -12,7 +12,7 @@ class FunctionalModule:
     name: str
     description: str
     method: Callable[..., Any]
-    signature: dict[str, str]
+    signature: dict[str, dict]
 
     def __init__(self, name, description, method, signature):
         self.name = name
@@ -39,8 +39,8 @@ class FunctionalModuleRegistry:
             }))
 
     @staticmethod
-    def _signature_to_string(signature: dict[str, str]):
+    def _signature_to_string(signature: dict[str, dict]):
-        return ", ".join([f"{k}: <{v}>" for k, v in signature.items()])
+        return ", ".join([f"{k}: <{v['description']}>" for k, v in signature.items()])
 
     def to_prompt(self):
         text = ""
@@ -56,6 +56,26 @@ class FunctionalModuleRegistry:
 
         return text
 
+    def to_json_schema(self):
+        return [
+            {
+                "name": module.name,
+                "description": module.description,
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        key: {
+                            k: v for k, v in value.items() if k != "required"
+                        }
+                        for key, value in module.signature.items()
+                    },
+                    "required": [key for key, value in module.signature.items() if value.get("required") != False]
+                }
+
+            }
+            for module in sorted(self._modules.values(), key=lambda cmd: cmd.name)
+        ]
+
     async def execute_function(self, context: CallerContext, function_name: str, **kwargs):
         cmd = self._modules.get(function_name)
         if cmd is not None:
@@ -68,7 +88,7 @@ moduleRegistry = FunctionalModuleRegistry()
 
 def functional_module(name: str,
                       description: str,
-                      signature=None):
+                      signature: dict[str, dict] = None):
     if signature is None:
         signature = {}
 

agent_jarvis/jarvis/gpt/ai_function.py

@@ -37,4 +37,7 @@ async def acall_ai_function(function: str, args: list, description: str, model:
 
     logger.debug(str(messages))
 
-    return await gpt.acreate_chat_completion(model=model, messages=messages, temperature=0)
+    msg_type, msg_content = await gpt.acreate_chat_completion(model=model, messages=messages, temperature=0)
+    if msg_type == "content":
+        return msg_content
+    return 'failed'

agent_jarvis/jarvis/gpt/gpt.py

@@ -21,6 +21,7 @@ async def acreate_chat_completion_once(
         max_tokens: int | None = None,
         deployment_id=None,
         request_timeout=40,
+        **kwargs
 ) -> str:
     """
     Create a chat completion and update the cost.
@@ -39,7 +40,8 @@ async def acreate_chat_completion_once(
             messages=messages,
             temperature=temperature,
             max_tokens=max_tokens,
-            request_timeout=request_timeout
+            request_timeout=request_timeout,
+            **kwargs
         )
     else:
         response = await openai.ChatCompletion.acreate(
@@ -47,7 +49,8 @@ async def acreate_chat_completion_once(
             messages=messages,
             temperature=temperature,
             max_tokens=max_tokens,
-            request_timeout=request_timeout
+            request_timeout=request_timeout,
+            **kwargs
         )
     if CFG.debug_mode:
         logger.debug(f"Response: {response}")
@@ -65,12 +68,13 @@ async def acreate_chat_completion(
         max_tokens: int = None,
         request_timeout: int = 40,
         num_retries=3,
-        on_single_request_timeout: Callable = None
+        on_single_request_timeout: Callable = None,
+        **kwargs
 ):
     """Create a chat completion using the OpenAI API
 
     Args:
-        messages (List[Message]): The messages to send to the chat completion
+        messages (List[dict]): The messages to send to the chat completion
         model (str, optional): The model to use. Defaults to None.
         temperature (float, optional): The temperature to use. Defaults to 0.9.
         max_tokens (int, optional): The max tokens to use. Defaults to None.
@@ -100,6 +104,7 @@ async def acreate_chat_completion(
                     temperature=temperature,
                     max_tokens=max_tokens,
                     request_timeout=request_timeout,
+                    **kwargs
                 )
             else:
                 response = await acreate_chat_completion_once(
@@ -108,6 +113,7 @@ async def acreate_chat_completion(
                     temperature=temperature,
                     max_tokens=max_tokens,
                     request_timeout=request_timeout,
+                    **kwargs
                 )
             break
         except RateLimitError:
@@ -129,5 +135,10 @@ async def acreate_chat_completion(
     if response is None:
         logger.error(f"Failed to get response from GPT after {num_retries} retries")
         raise RuntimeError(f"Failed to get response after {num_retries} retries")
-    resp = response.choices[0].message["content"]
+
-    return resp
+    choice_message = response.choices[0].message
+    content = choice_message.get("content")
+    if content is None:
+        return "function_call", {k: v for k, v in choice_message["function_call"].items()}
+    else:
+        return "content", content

agent_jarvis/jarvis/gpt/token_counter.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 from typing import List
+import json
 
 import tiktoken_async
 
@@ -33,7 +34,8 @@ async def count_message_tokens(
     elif model == "gpt-4":
         # !Note: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.")
         return await count_message_tokens(messages, model="gpt-4-0314")
-    elif model == "gpt-3.5-turbo-0301":
+    # TODO: OpenAI has not mention how to count tokens for 0613, thus, we use the former method
+    elif model == "gpt-3.5-turbo-0301" or model == "gpt-3.5-turbo-0613" or model == "gpt-3.5-turbo-16k-0613":
         tokens_per_message = (
             4  # every message follows <|start|>{role/name}\n{content}<|end|>\n
         )
@@ -51,6 +53,11 @@ async def count_message_tokens(
     for message in messages:
         num_tokens += tokens_per_message
         for key, value in message.items():
+            if not isinstance(value, str):
+                # TODO: Since openai does not mentioned how to count tokens of 'funciton_call',
+                #   and only string is countable, thus, if the value is not a `str` (`function_call`
+                #   field of a message), we convert it into json
+                value = json.dumps(value)
             num_tokens += len(encoding.encode(value))
             if key == "name":
                 num_tokens += tokens_per_name

example_modules/demo_modules/google_calendar.module.py

@@ -18,7 +18,18 @@ def reg_or_not():
     @functional_module(
         name="add_alarm",
         description="Create an alarm",
-        signature={"date": "The alarm date, 'YYYY-mm-dd HH:MM:SS format'", "desc": "The event description"})
+        signature={
+            "date": {
+                "description": "The alarm date, 'YYYY-mm-dd HH:MM:SS format'",
+                "type": "string",
+                "required": True
+            },
+            "desc": {
+                "description": "The event description",
+                "type": "string",
+                "required": True
+            }
+        })
     async def add_alarm(context: CallerContext, date, desc):
         # date = "2023-05-10 14:56:59"
         now = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S").timestamp()
@@ -43,7 +54,14 @@ def reg_or_not():
     @functional_module(
         name="delete_alarm",
         description="delete all alarms whose ID is in the list",
-        signature={"IDs": "A list of alarm IDs to"})
+        signature={
+            "IDs": {
+                "type": "array",
+                "items": { "type": "string" },
+                "description": "A list of alarm IDs to delete",
+                "required": True
+            }
+        })
     async def delete_alarm(context: CallerContext, IDs: List[str]):
         # get all tasks
         result = await do_get(google_calendar_service_address + "/tasks")

example_modules/demo_modules/stable_diffusion.module.py

@@ -113,7 +113,7 @@ Sometimes you maybe asked to generate a pic of myself. That means you MUST add '
 - Add unique touches to each output, making it lengthy, detailed, and stylized.
 - Show, don't tell; instead of tagging \"exceptional artwork\" or \"emphasizing a beautiful ...\" provide - precise details.
 - Ensure the output is placed inside a beautiful and stylized markdown.
-- The prompt you return  MUST be English. The lenth of prompt MUST less than 150.
+- The prompt you return  MUST be English. The tokens of prompt MUST less than 70.
 """
 
     OTHER_SD_PARAMS_NAME = "other"
@@ -153,7 +153,7 @@ NOTE: Just reply using these information, don't ask me anything.
         sys_prompt = {'role': 'system', 'content': gpt_system_prompt}
         messages = [sys_prompt, {'role': 'user', 'content': prompt}]
         model = CFG.small_llm_model
-        resp = await acreate_chat_completion(
+        _, resp = await acreate_chat_completion(
             messages,
             model,
             temperature=0,
@@ -207,7 +207,7 @@ NOTE: Just reply using these information, don't ask me anything.
         messages = [sys_prompt, {'role': 'user', 'content': "Generation " + origin_str}]
         model = CFG.small_llm_model
         try:
-            resp = await acreate_chat_completion(
+            _, resp = await acreate_chat_completion(
                 messages,
                 model,
                 temperature=0,
@@ -239,7 +239,12 @@ NOTE: Just reply using these information, don't ask me anything.
     @functional_module(
         name="stable_diffusion",
         description="Generate a picture.",
-        signature={'prompt': 'the description I told you'})
+        signature={
+            'prompt': {
+                "type": "string",
+                "description": 'the description I told you'
+            }
+        })
     async def stable_diffusion(context: CallerContext, prompt: str):
         await context.reply_text("I'm generating the image, this may take a while.")
         style = await determine_style(prompt)

example_modules/demo_modules/twitter.module.py

@@ -14,8 +14,12 @@ def reg_or_not():
     @functional_module(
         name="post_tweet",
         description="post a tweet",
-        signature={"content": "the content of the tweet"}
+        signature={
-    )
+            "content": {
+                "type": "string",
+                "description": "the content of the tweet"
+            }
+        })
     async def post_tweet(context: CallerContext, content: str):
         response = await do_post(twitter_service_address + "/twitter/tweet_post", '', {"content": content})
         logger.info(f"response: {response}")

example_modules/demo_modules/youtube.module.py

@@ -14,7 +14,12 @@ def reg_or_not():
     @functional_module(
         name="youtube_video_brief",
         description="Get the brief content of a youtube video",
-        signature={"url": "The address of the video"})
+        signature={
+            "url": {
+                "type": "string",
+                "description": "The address of the video" 
+            }
+        })
     async def youtube_video_brief(context: CallerContext, url: str):
         await context.push_notification(f"One second... I'm watching this video: {url}")
         if not url.startswith('https://www.youtube.com/watch?'):
@@ -38,7 +43,12 @@ def reg_or_not():
     @functional_module(
         name="youtube_video_brief_vid",
         description="Get the brief content of a youtube video identified by video id",
-        signature={"video_id": "The video id of the video"})
+        signature={
+            "video_id": {
+                "type": "string",
+                "description": "The video id of the video"
+            }
+        })
     async def youtube_video_brief_vid(context: CallerContext, video_id: str):
         url = f'https://www.youtube.com/watch?v={video_id}'
         await context.push_notification(f"One second... I'm watching this video: {url}")
@@ -70,7 +80,12 @@ def reg_or_not():
     @functional_module(
         name="youtube_x_video_info",
         description="Get the basic information of a youtube user's newest videos, when the summary of videos are not required, you should use this function",
-        signature={"username": "The username"})
+        signature={
+            "username": {
+                "type": "string",
+                "description": "The username"
+            }
+        })
     async def youtube_x_video_info(context: CallerContext, username: str):
         response = await youtube_latest_video_info_of(context, username, False)
         result = f'The brief content of the latest videos of {username} are:\n'
@@ -86,7 +101,12 @@ def reg_or_not():
     @functional_module(
         name="youtube_notify_new",
         description="Watching an Youtuber, push an notification when the youtuber published a new video",
-        signature={"username": "The username"})
+        signature={
+            "username": {
+                "type": "string",
+                "description": "The username"
+            }
+        })
     async def youtube_notify_new(context: CallerContext, username: str):
         if username.startswith('@'):
             username = username[1:]

example_modules/light_switch/switch.module.py

@@ -4,7 +4,16 @@ from jarvis.functional_modules.functional_module import functional_module, Calle
 @functional_module(
     name="toggle_light",
     description="Turn on/off the light.",
-    signature={"room": "The room name", "on": "Turn on or off, bool type>"})
+    signature={
+        "room": {
+            "type": "string",
+            "description": "The room name"
+        },
+        "on": {
+            "type": "boolean",
+            "description": "Turn on or off"
+        }
+    })
 async def light_switch(context: CallerContext, room: str, on: bool):
     # Do the actual control here, something like this
     # room_id = convert_room_name_to_id(room)