Adjust the directory structure to prepare for merging into Master.

PoC/agent_jarvis/jarvis/CFG.py

@@ -0,0 +1,97 @@
+import logging
+import os
+import dotenv
+
+dotenv.load_dotenv()
+
+
+# ==== Utils
+def _string_to_bool(s: str | None):
+    if s is None:
+        return None
+    s = s.lower()
+    if s in ['y', 'yes', 't', 'true']:
+        return True
+    if s in ['n', 'no', 'f', 'false']:
+        return False
+    raise Exception(f"Invalid argument '{s}', should be a bool value: y/yes/n/no/t/true/f/false.")
+
+
+def _string_to_log_level(s: str | None):
+    if s is None:
+        return None
+    s = s.lower()
+    if s in ['debug', 'd']:
+        return logging.DEBUG
+    if s in ['info', 'i']:
+        return logging.INFO
+    if s in ['w', 'warn', 'warning']:
+        return logging.WARNING
+    if s in ['error', 'e', 'err']:
+        return logging.ERROR
+    if s in ['fatal', 'critical']:
+        return logging.FATAL
+    raise Exception(f"Invalid argument '{s}', should be a log level: debug, info, warn, error, fatal")
+
+
+def _get_env_str(name: str, must_not_empty: bool = False):
+    v = os.getenv(name)
+    if must_not_empty and (v is None or v == ''):
+        raise Exception(f"Environment variable '{name}' is required!")
+    return v
+
+
+def _get_env_bool(name: str): return _string_to_bool(os.getenv(name))
+
+
+def _get_env_int(name: str): return int(os.getenv(name))
+
+
+def _get_env_float(name: str): return float(os.getenv(name))
+
+
+def _get_env_log_level(name: str): return _string_to_log_level(os.getenv(name))
+
+
+# The config
+
+# DO NOT use it, it's still not mature yet
+use_private_ai = _get_env_bool("JARVIS_USE_PRIVATE_AI") or False
+private_ai_address = _get_env_str("JARVIS_PRIVATE_AI_URL", use_private_ai)
+
+is_server_mode = _get_env_bool("JARVIS_SERVER_MODE") or False
+# The port used in server mode
+server_mode_port = _get_env_int("JARVIS_SERVER_MODE_PORT") or 1000
+# Jarvis can also connect to a server as a client.
+# This is the server's address
+bot_server_url = _get_env_str("JARVIS_BOT_SERVER_URL") or "http://localhost:8081"
+
+# The directory where the chat history should be stored,
+# By storing the chat history, each time Jarvis starts up, the chat context is restored
+chat_history_dir = _get_env_str("JARVIS_CHAT_HISTORY_DIR") or None
+
+# ChatGPT temperature
+temperature = _get_env_float("JARVIS_AI_TEMPERATURE") or 0
+
+debug_mode = _get_env_bool("JARVIS_DEBUG_MODE") or False
+log_level = _get_env_log_level("JARVIS_LOG_LEVEL") or logging.INFO
+
+# The main llm model
+llm_model = _get_env_str("JARVIS_LLM_MODEL") or "gpt-3.5-turbo-0301"
+# The model used to handle some simple tasks
+small_llm_model = _get_env_str("JARVIS_SMALL_LLM_MODEL") or "gpt-3.5-turbo-0301"
+token_limit = _get_env_int("JARVIS_TOKEN_LIMIT") or 4000
+
+openai_api_key = _get_env_str("JARVIS_OPENAI_API_KEY", True)
+# If your service is not provided directly by openai,
+# or you just deployed you own AI model with a same API as opeai.
+# Or this configuration is useless
+openai_url_base = _get_env_str("JARVIS_OPENAI_URL_BASE") or None
+
+# Tell Jarvis where to load function modules
+external_function_module_dirs = _get_env_str("JARVIS_EXTERNAL_FUNCTION_MODULE_DIR")
+
+use_azure = False
+def get_azure_deployment_id_for_model(model):
+    assert False
+    # TODO

PoC/agent_jarvis/jarvis/ai_agent/agent_factory.py

@@ -0,0 +1,12 @@
+from jarvis import CFG
+from jarvis.ai_agent.gpt_agent import GptAgent
+from jarvis.ai_agent.webui_agent import WebuiAgent
+from jarvis.ai_agent.base_agent import BaseAgent
+from jarvis.functional_modules.functional_module import CallerContext
+
+
+def create_agent(context: CallerContext) -> BaseAgent:
+    if CFG.use_private_ai:
+        return WebuiAgent(context)
+    else:
+        return GptAgent(context)

PoC/agent_jarvis/jarvis/ai_agent/agent_utils.py

@@ -0,0 +1,72 @@
+import json
+from typing import Dict
+
+from jarvis.functional_modules.functional_module import CallerContext, moduleRegistry
+from jarvis.logger import logger
+
+
+def must_not_be_valid_json(s: str):
+    """
+    Simply check if the string is a JSON.
+    If the string does not contain even 1 pair of '{}',
+    it must not be a JSON, we treat it as a normal string.
+    """
+    if s.count('{') < 1 and s.count("{") < 1:
+        return True
+    return False
+
+def get_thoughts(reply: Dict, assistant_reply_json_valid: dict):
+    assistant_thoughts_reasoning = None
+    assistant_thoughts_speak = None
+
+    assistant_thoughts = assistant_reply_json_valid.get("thoughts", {})
+    assistant_thoughts_text = assistant_thoughts.get("text")
+    if assistant_thoughts:
+        assistant_thoughts_reasoning = assistant_thoughts.get("reasoning")
+        assistant_thoughts_speak = assistant_thoughts.get("speak")
+    reply["thoughts"] = assistant_thoughts_text
+    reply["reasoning"] = assistant_thoughts_reasoning
+    reply["speak"] = assistant_thoughts_speak
+    logger.debug(f" THOUGHTS: {assistant_thoughts_text}")
+    logger.debug(f"REASONING: {assistant_thoughts_reasoning}")
+    logger.debug(f" SPEAKING: {assistant_thoughts_speak}")
+
+
+def get_function(reply: Dict, response_json: Dict):
+    try:
+        if "function" not in response_json:
+            return "Error:", "Missing 'function' object in JSON"
+
+        if not isinstance(response_json, dict):
+            return "Error:", f"'response_json' object is not dictionary {response_json}"
+
+        function = response_json["function"]
+        if not isinstance(function, dict):
+            return "Error:", "'function' object is not a dictionary"
+
+        if "name" not in function:
+            return "Error:", "Missing 'name' field in 'function' object"
+
+        function_name = function["name"]
+
+        # Use an empty dictionary if 'args' field is not present in 'function' object
+        arguments = function.get("args", {})
+
+        reply["function"] = function_name
+        reply["arguments"] = arguments
+
+        return function_name, arguments
+    except json.decoder.JSONDecodeError:
+        return "Error:", "Invalid JSON"
+    except Exception as e:
+        return "Error:", str(e)
+
+
+async def execute_function(
+        context: CallerContext,
+        function_name: str,
+        **arguments,
+):
+    logger.debug(f"Executing function: {function_name}({arguments})")
+    await context.push_notification(f"Executing function: {function_name}({arguments})")
+    return await moduleRegistry.execute_function(context, function_name, **arguments)

PoC/agent_jarvis/jarvis/ai_agent/base_agent.py

@@ -0,0 +1,23 @@
+from jarvis.functional_modules.functional_module import CallerContext
+
+
+class BaseAgent:
+    _caller_context: CallerContext = None
+
+    def __init__(self, context: CallerContext):
+        self._caller_context = context
+
+    async def feed_prompt(self, prompt):
+        raise NotImplementedError("Not implemented")
+
+    def append_history_message(self, role: str, content: str):
+        raise NotImplementedError("Not implemented")
+
+    def clear_history_messages(self):
+        raise NotImplementedError("Not implemented")
+
+    def save_history(self, to_where):
+        raise NotImplementedError("Not implemented")
+
+    def load_history(self, from_where):
+        raise NotImplementedError("Not implemented")

PoC/agent_jarvis/jarvis/ai_agent/gpt_agent.py

@@ -0,0 +1,251 @@
+import asyncio
+import contextlib
+import json
+import time
+from typing import Dict, List
+
+from openai.error import RateLimitError
+
+from jarvis import CFG
+from jarvis.ai_agent.agent_utils import must_not_be_valid_json, get_thoughts, get_function, execute_function
+from jarvis.ai_agent.base_agent import BaseAgent
+from jarvis.functional_modules.functional_module import CallerContext, moduleRegistry
+from jarvis.gpt import token_counter, gpt
+from jarvis.json_utils.json_fix_llm import fix_json_using_multiple_techniques
+from jarvis.json_utils.utilities import validate_json
+from jarvis.logger import logger
+
+
+def _generate_first_prompt():
+    return """I will ask you questions or ask you to do something. You should:
+First, determine if you know the answer of the question or you can accomplish the task directly. 
+If so, response directly. 
+If not, try to complete the task by calling the functions below.
+If you can't accomplish the task by yourself and no function is able to accomplish the task, say "Dear master, sorry, I'm not able to do that."
+
+Your setup:
+```
+{
+    "author": "OpenDAN",
+    "name": "Jarvis",
+}
+```"""
+
+
+class GptAgent(BaseAgent):
+    _system_prompt: str
+    _full_message_history: List[dict] = []
+    _message_tokens: List[int] = []
+
+    def __init__(self, caller_context: CallerContext):
+        super().__init__(caller_context)
+        self._system_prompt = _generate_first_prompt()
+        logger.debug(f"Using GptAgent, system prompt is: {self._system_prompt}")
+        logger.debug(f"{json.dumps(moduleRegistry.to_json_schema())}")
+
+    async def _feed_prompt_to_get_response(self, prompt):
+        reply_type, assistant_reply = await self._chat_with_ai(
+            self._system_prompt,
+            prompt,
+            CFG.token_limit,
+        )
+
+        if reply_type == "content":
+            return {
+                "speak": assistant_reply,
+            }
+        elif reply_type == "function_call":
+            arguments_string = assistant_reply["arguments"]
+            try:
+                arguments = json.loads(arguments_string)
+            except:
+                arguments = await fix_json_using_multiple_techniques()
+            return {
+                "function": assistant_reply["name"],
+                "arguments": arguments
+            }
+
+    async def feed_prompt(self, prompt):
+        # Send message to AI, get response
+        logger.debug(f"Trigger: {prompt}")
+        reply: Dict = None
+        for i in range(3):
+            try:
+                if i == 0:
+                    reply = await self._feed_prompt_to_get_response(prompt)
+                else:
+                    reply = await self._feed_prompt_to_get_response(
+                        prompt + ". Remember to reply using the specified JSON form")
+                break
+            except Exception as e:
+                # TODO: Feed the error to ChatGPT?
+                logger.debug(f"Failed to get reply, try again! {str(e)}")
+                continue
+
+        if reply is None:
+            await self._caller_context.reply_text("Sorry, but I don't understand what you want me to do.")
+            return
+
+        # Execute function
+        function_name: str = reply.get("function")
+        if function_name is None:
+            await self._caller_context.reply_text(reply["speak"])
+        else:
+            arguments: Dict = reply["arguments"]
+
+            function_result = "Failed"
+            try:
+                function_result = await execute_function(self._caller_context, function_name, **arguments)
+            finally:
+                result = f"{function_result}"
+
+                # Check if there's a result from the function append it to the message
+                # history
+                if result is not None:
+                    self.append_history_message_raw({"role": "function", "name": function_name, "content": result})
+                    logger.debug(f"function: {result}")
+                else:
+                    self.append_history_message_raw({"role": "function", "name": function_name, "content": "Unable to execute function"})
+                    logger.debug("function: Unable to execute function")
+
+    def append_history_message(self, role: str, content: str):
+        self._full_message_history.append({'role': role, 'content': content})
+        self._message_tokens.append(-1)
+
+    def append_history_message_raw(self, msg: dict):
+        self._full_message_history.append(msg)
+        self._message_tokens.append(-1)
+
+    def clear_history_messages(self):
+        self._full_message_history.clear()
+        self._message_tokens.clear()
+
+    def save_history(self, to_where):
+        with open(to_where, "w") as f:
+            assert len(self._message_tokens) == len(self._full_message_history)
+            s = json.dumps([
+                self._message_tokens,
+                self._full_message_history,
+            ])
+            f.write(s)
+
+    def load_history(self, from_where):
+        with contextlib.suppress(Exception):
+            with open(from_where, "r") as f:
+                tmp = json.loads(f.read())
+                if isinstance(tmp, list) and len(tmp[0]) == len(tmp[1]):
+                    self._message_tokens = tmp[0]
+                    self._full_message_history = tmp[1]
+
+    async def _chat_with_ai(
+            self, prompt, user_input, token_limit
+    ):
+        """Interact with the OpenAI API, sending the prompt, user input, message history,
+        and permanent memory."""
+        while True:
+            try:
+                model = CFG.llm_model
+                # Reserve 1000 tokens for the response
+
+                send_token_limit = token_limit - 1000
+
+                (
+                    next_message_to_add_index,
+                    current_tokens_used,
+                    insertion_index,
+                    current_context,
+                ) = await self._generate_context(prompt, model)
+
+                current_tokens_used += await token_counter.count_message_tokens(
+                    [{"role": "user", "content": user_input}], model
+                )  # Account for user input (appended later)
+
+                # TODO: OpenAI does not say how to count function tokens, we use this method to roughly get the tokens count
+                #   It's result looks much larger than OpenAI's result
+                current_tokens_used += await token_counter.count_message_tokens(
+                    [{"role": "user", "content": json.dumps(moduleRegistry.to_json_schema())}], model
+                )
+
+                while next_message_to_add_index >= 0:
+                    # print (f"CURRENT TOKENS USED: {current_tokens_used}")
+                    tokens_to_add = await self._get_history_message_tokens(next_message_to_add_index, model)
+                    if current_tokens_used + tokens_to_add > send_token_limit:
+                        break
+
+                    message_to_add = self._full_message_history[next_message_to_add_index]
+                    # Add the most recent message to the start of the current context,
+                    #  after the two system prompts.
+                    current_context.insert(insertion_index, message_to_add)
+
+                    # Count the currently used tokens
+                    current_tokens_used += tokens_to_add
+
+                    # Move to the next most recent message in the full message history
+                    next_message_to_add_index -= 1
+
+                # Append user input, the length of this is accounted for above
+                current_context.extend([{"role": "user", "content": user_input}])
+
+                # Calculate remaining tokens
+                tokens_remaining = token_limit - current_tokens_used
+
+                assert tokens_remaining >= 0
+
+                async def on_single_chat_timeout(will_retry):
+                    await self._caller_context.push_notification(
+                        f'Thinking timeout{", retry" if will_retry else ", give up"}.')
+
+
+                reply_type, assistant_reply = await gpt.acreate_chat_completion(
+                    model=model,
+                    messages=current_context,
+                    temperature=CFG.temperature,
+                    max_tokens=tokens_remaining,
+                    on_single_request_timeout=on_single_chat_timeout,
+                    functions=moduleRegistry.to_json_schema()
+                )
+
+                # Update full message history
+                if reply_type == "content":
+                    self.append_history_message("user", user_input)
+                    self.append_history_message("assistant", assistant_reply)
+                    pass
+                elif reply_type == "function_call":
+                    self.append_history_message("user", user_input)
+                    self.append_history_message_raw({"role": "assistant", "function_call": assistant_reply, "content": None})
+                    pass
+                else:
+                    assert False, "Unexpected reply type"
+
+                return reply_type, assistant_reply
+            except RateLimitError:
+                # TODO: When we switch to langchain, or something else this is built in
+                print("Error: ", "API Rate Limit Reached. Waiting 10 seconds...")
+                await asyncio.sleep(10)
+
+    async def _generate_context(self, prompt, model):
+        # We use the timezone of the session
+        timestamp = time.time() + time.timezone + self._caller_context.get_tz_offset() * 3600
+        time_str = time.strftime('%c', time.localtime(timestamp))
+        current_context = [
+            {"role": "system", "content": prompt},
+            {"role": "system", "content": f"The current time and date is {time_str}"},
+        ]
+
+        # Add messages from the full message history until we reach the token limit
+        next_message_to_add_index = len(self._full_message_history) - 1
+        insertion_index = len(current_context)
+        # Count the currently used tokens
+        current_tokens_used = await token_counter.count_message_tokens(current_context, model)
+        return (
+            next_message_to_add_index,
+            current_tokens_used,
+            insertion_index,
+            current_context,
+        )
+
+    async def _get_history_message_tokens(self, index, model: str = "gpt-3.5-turbo-0301") -> int:
+        if self._message_tokens[index] == -1:
+            # since couting token is relatively slow, we store it here
+            self._message_tokens[index] = await token_counter.count_message_tokens([self._full_message_history[index]], model)
+        return self._message_tokens[index]

PoC/agent_jarvis/jarvis/ai_agent/webui_agent.py

@@ -0,0 +1,208 @@
+import contextlib
+import json
+
+import aiohttp
+
+from jarvis import CFG
+from jarvis.ai_agent.agent_utils import must_not_be_valid_json, get_thoughts, get_function, execute_function
+from jarvis.ai_agent.base_agent import BaseAgent
+from jarvis.functional_modules.functional_module import CallerContext, moduleRegistry
+from jarvis.json_utils.json_fix_llm import fix_json_using_multiple_techniques
+from jarvis.json_utils.utilities import validate_json
+from jarvis.logger import logger
+
+
+def _generate_system_prompt():
+    return """Since now, every your response should satisfy the following JSON format, a 'function' must be chosen:
+```
+{
+    "thoughts": {
+        "text": "<Your thought>",
+        "reasoning": "<Your reasoning, think step by step>",
+        "speak": "<what you want to say to me>"
+    },
+    "function": {
+        "name": "<mandatory, one of listed functions>",
+        "args": {
+            "arg name": "<value>"
+        }
+    }
+}
+```
+I will ask you questions or ask you to do something. You should:
+First, you should determine if you know the answer of the question or you can accomplish the task directly.
+If so, you should response directly.
+If not, you should try to complete the task by calling the functions below.
+If you can't accomplish the task by yourself and no function is able to accomplish the task, say "Dear master, sorry, I'm not able to do that."
+
+```
+Available functions:
+```
+""" + moduleRegistry.to_prompt() + """
+```
+Your setup:
+```
+{
+    "author": "OpenDAN",
+    "name": "Jarvis",
+}
+Example:
+```
+Tom: generate a picture of me.
+Jarvis: {
+    "function": {
+        "name": "stable_diffusion",
+        "args": {
+            "prompt": "me"
+        }
+    }
+}
+```
+"""
+
+
+def _generate_request(prompt: str):
+    return {
+        'prompt': prompt,
+        'max_new_tokens': 1000,
+        'do_sample': True,
+        'temperature': 0.5,
+        'top_p': 0.5,
+        'typical_p': 1,
+        'repetition_penalty': 1.18,
+        'top_k': 40,
+        'min_length': 0,
+        'no_repeat_ngram_size': 0,
+        'num_beams': 1,
+        'penalty_alpha': 0,
+        'length_penalty': 1,
+        'early_stopping': False,
+        'seed': -1,
+        'add_bos_token': True,
+        'truncation_length': 2048,
+        'ban_eos_token': False,
+        'skip_special_tokens': True,
+        'stopping_strings': ["Tom: "]
+    }
+
+
+def _convert_role(role: str):
+    if role == 'user':
+        return 'Tom'
+    if role == 'assistant':
+        return 'Jarvis'
+    return role
+
+
+async def _completion(prompt):
+    async with aiohttp.ClientSession() as session:
+        # body = json.dumps(_generate_request(prompt))
+        async with session.post(CFG.private_ai_address, json=_generate_request(prompt)) as response:
+            if response.status == 200:
+                resp_obj = await response.json()
+                logger.debug(f"Completion result: {json.dumps(resp_obj, indent=2)}")
+                result = resp_obj["results"][0]['text']
+                return result
+
+    return None
+
+
+class WebuiAgent(BaseAgent):
+    _system_prompt: str
+    _history = []
+
+    def __init__(self, context: CallerContext):
+        super().__init__(context)
+        self._system_prompt = _generate_system_prompt()
+
+    async def feed_prompt(self, prompt):
+        prompt = f'Tom: {prompt}'
+        self._history.append(prompt)
+        final_prompt = self._system_prompt + '\n' + '\n'.join(self._history)
+        logger.debug(f"Final prompt: {final_prompt}")
+        reply = await self._feed_prompt_to_get_respones(final_prompt)
+        await self._handle_reply(reply)
+
+    async def _feed_prompt_to_get_respones(self, prompt):
+        assistant_reply = await _completion(prompt)
+
+        reply = {
+            "thoughts": None,
+            "reasoning": None,
+            "speak": None,
+            "function": None,
+            "arguments": None,
+        }
+
+        if must_not_be_valid_json(assistant_reply):
+            raise Exception(f"AI replied an invalid response: {assistant_reply}!")
+        else:
+            assistant_reply_json = await fix_json_using_multiple_techniques(assistant_reply)
+
+        # Print Assistant thoughts
+        if assistant_reply_json != {}:
+            validate_json(assistant_reply_json, "llm_response_format_1")
+            try:
+                get_thoughts(reply, assistant_reply_json)
+                get_function(reply, assistant_reply_json)
+            except Exception as e:
+                logger.error(f"AI replied an invalid response: {assistant_reply}. Error: {str(e)}")
+                raise e
+        else:
+            raise Exception(f"AI replied an invalid response: {assistant_reply}!")
+
+        function_name = reply["function"]
+        if function_name is None or function_name == '':
+            raise Exception(f"Missing a function")
+        arguments = reply["arguments"]
+
+        if not isinstance(arguments, dict):
+            raise Exception(f"Invalid arguments, it MUST be a dict")
+        return reply
+
+    async def _handle_reply(self, reply):
+        # TODO: It's not reliable now, thus do nothing now.
+        return
+        if reply is None:
+            await self._caller_context.reply_text("Sorry, but I don't understand what you want me to do.")
+            return
+
+        # Execute function
+        function_name: str = reply["function"]
+        arguments: dict = reply["arguments"]
+
+        await self._caller_context.reply_text(reply["speak"])
+        execute_error = None
+        try:
+            function_result = await execute_function(self._caller_context, function_name, **arguments)
+        except Exception as e:
+            function_result = "Failed"
+            execute_error = e
+        result = f"Function {function_name} returned: " f"{function_result}"
+
+        if function_name is not None:
+            if result is not None:
+                self.append_history_message("system", result)
+                logger.debug(f"SYSTEM: {result}")
+            else:
+                self.append_history_message("system", "Unable to execute function")
+                logger.debug("SYSTEM: Unable to execute function")
+
+        if execute_error is not None:
+            raise execute_error
+
+    def append_history_message(self, role: str, content: str):
+        self._history.append({'role': role, 'content': content})
+
+    def clear_history_messages(self):
+        self._history.clear()
+
+    def save_history(self, to_where):
+        with open(to_where, "w") as f:
+            s = json.dumps(self._history)
+            f.write(s)
+
+    def load_history(self, from_where):
+        with contextlib.suppress(Exception):
+            with open(from_where, "r") as f:
+                self._history = json.loads(f.read())

PoC/agent_jarvis/jarvis/functional_modules/caller_context.py

@@ -0,0 +1,37 @@
+class CallerContext:
+    __agent: 'BaseAgent' = None
+
+    def __init__(self, agent):
+        self.__agent = agent
+
+    def append_history_message(self, role: str, content: str):
+        self.__agent.append_history_message(role, content)
+
+    def get_tz_offset(self):
+        raise Exception("Function not implemented")
+
+    def get_tz_offset_str(self):
+        of = self.get_tz_offset()
+        if of > 0:
+            return f"+{of}"
+        if of < 0:
+            return f"{of}"
+        return ""
+
+    def get_last_image(self) -> str:
+        raise NotImplementedError("Function not implemented")
+    
+    def set_last_image(self, img: str):
+        raise NotImplementedError("Function not implemented")
+
+    async def reply_text(self, msg):
+        raise NotImplementedError("Function not implemented")
+
+    async def reply_image_base64(self, msg):
+        raise NotImplementedError("Function not implemented")
+
+    async def reply_markdown(self, md):
+        raise NotImplementedError("Function not implemented")
+
+    async def push_notification(self, msg):
+        raise NotImplementedError("Function not implemented")

PoC/agent_jarvis/jarvis/functional_modules/functional_module.py

@@ -0,0 +1,121 @@
+import json
+import traceback
+from typing import Callable, Any, Tuple, Dict, List
+
+from jarvis.functional_modules.caller_context import CallerContext
+from jarvis.logger import logger
+from jarvis.utils import function_error
+from jarvis import CFG
+
+
+class FunctionalModule:
+    name: str
+    description: str
+    method: Callable[..., Any]
+    signature: dict[str, dict]
+
+    def __init__(self, name, description, method, signature):
+        self.name = name
+        self.description = description
+        self.method = method
+        self.signature = signature
+
+
+class FunctionalModuleRegistry:
+    _modules: Dict[str, FunctionalModule] = {}
+
+    def __init__(self):
+        pass
+
+    def register(self, cmd):
+        self._modules.update({cmd.name: cmd})
+
+    def print(self):
+        for cmd in self._modules.values():
+            print(json.dumps({
+                "name": cmd.name,
+                "description": cmd.description,
+                "signature": cmd.signature
+            }))
+
+    @staticmethod
+    def _signature_to_string(signature: dict[str, dict]):
+        return ", ".join([f"{k}: <{v['description']}>" for k, v in signature.items()])
+
+    def to_prompt(self):
+        text = ""
+        i = 1
+        for module in sorted(self._modules.values(), key=lambda cmd: cmd.name):
+            if module.signature is None or len(module.signature) == 0:
+                text += f"{i}. {module.name}: {module.description}, don't need argument\n"
+            else:
+                text += f"{i}. {module.name}: {module.description}, args: {FunctionalModuleRegistry._signature_to_string(module.signature)}\n"
+            i += 1
+        if len(text) > 0:
+            text = text[0:-1]  # Delete the tailing '\n'
+
+        return text
+
+    def to_json_schema(self):
+        return [
+            {
+                "name": module.name,
+                "description": module.description,
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        key: {
+                            k: v for k, v in value.items() if k != "required"
+                        }
+                        for key, value in module.signature.items()
+                    },
+                    "required": [key for key, value in module.signature.items() if value.get("required") != False]
+                }
+
+            }
+            for module in sorted(self._modules.values(), key=lambda cmd: cmd.name)
+        ]
+
+    async def execute_function(self, context: CallerContext, function_name: str, **kwargs):
+        cmd = self._modules.get(function_name)
+        if cmd is not None:
+            return await cmd.method(context, **kwargs)
+        return "(Module Not Found)"
+
+
+moduleRegistry = FunctionalModuleRegistry()
+
+
+def functional_module(name: str,
+                      description: str,
+                      signature: dict[str, dict] = None):
+    if signature is None:
+        signature = {}
+
+    def decorator(func: Callable[..., Any]):
+        async def wrapper(context: CallerContext, *args, **kwargs) -> Any:
+            try:
+                return await func(context, *args, **kwargs)
+            except function_error.FunctionError as e:
+                logger.error(traceback.format_exc())
+                await context.reply_text(f"Sorry, failed to do the job: {e.msg}")
+            except:
+                logger.error(traceback.format_exc())
+                await context.reply_text("Sorry, an unknown error occurred during doing the job")
+            return "Failed"
+
+        cmd = FunctionalModule(
+            name=name,
+            description=description,
+            method=wrapper,
+            signature=signature
+        )
+
+        global moduleRegistry
+        moduleRegistry.register(cmd)
+        if CFG.debug_mode:
+            print("Registering: " + name)
+
+        return wrapper
+
+    return decorator

PoC/agent_jarvis/jarvis/gateway/session.py

@@ -0,0 +1,210 @@
+import asyncio
+import os
+from asyncio import Queue, Task
+import socketio
+from jarvis import CFG
+from jarvis.ai_agent import agent_factory
+from jarvis.ai_agent.base_agent import BaseAgent
+from jarvis.functional_modules.caller_context import CallerContext
+from jarvis.logger import logger
+from jarvis.utils import function_error
+from jarvis.utils.incoming_chat_message_parser import assemble_json_message, IncomingChatMessage
+
+
+class SioConnection:
+    async def emit(self, msg_type: str, msg: str, user_id: str, session_id: str, message_id: str):
+        """
+        msg_type: 'text', 'markdown', 'notification', 'image', 'end'
+        """
+        raise Exception("Not implemented!")
+
+    async def safe_emit(self, msg_type: str, msg: str, user_id: str, session_id: str, msg_id: str):
+        try:
+            await self.emit(msg_type, msg, user_id, session_id, msg_id)
+        except:
+            logger.debug("Failed to safe emit text")
+            pass
+
+
+class SioServerConnection(SioConnection):
+    _sio: socketio.AsyncServer = None
+    _sid: str = None
+
+    def __init__(self, sio: socketio.AsyncServer, sid):
+        self._sio = sio
+        self._sid = sid
+
+    async def emit(self, msg_type: str, msg: str, user_id: str, session_id: str, message_id: str):
+        data = assemble_json_message(msg_type, msg, user_id, session_id, message_id)
+        await self._sio.emit('chat_message', data, self._sid)
+
+
+class SioClientConnection(SioConnection):
+    _sio: socketio.AsyncClient = None
+
+    def __init__(self, sio: socketio.AsyncClient):
+        self._sio = sio
+
+    async def emit(self, msg_type: str, msg: str, user_id: str, session_id: str, message_id: str):
+        data = assemble_json_message(msg_type, msg, user_id, session_id, message_id)
+        await self._sio.emit('chat_message', data)
+
+
+def _get_history_file_dir():
+    if CFG.chat_history_dir is None:
+        return None
+    if CFG.use_private_ai:
+        sub_path = "private"
+    else:
+        sub_path = "gpt"
+    return os.path.join(CFG.chat_history_dir, sub_path)
+
+
+class Session(CallerContext):
+    _agent: BaseAgent = None
+    _sio: SioConnection = None
+    _session_id: str = None
+    _tz_offset: int = 0  # timezone offset, in hours. = local_time - UTC0
+    _last_image: str = None
+
+    _history_dir: str = None
+
+    _message_handle_coro: Task = None
+    _message_queue: Queue = None
+    _is_running: bool = True
+
+    _last_message_id: str = None  # keep last message's ID to avoid handling duplicated messages
+
+    # Tese variables start and end with '_' is temporary, valid only during handling messages
+    _message_user_id_: str = None
+    _message_id_: str = None
+
+    def __init__(self, sio: SioConnection, session_id: str):
+        agent = agent_factory.create_agent(self)
+        super().__init__(agent)
+        self._agent = agent
+        self._sio = sio
+        self._session_id = session_id
+        self._message_queue = Queue()
+
+        self._history_dir = _get_history_file_dir()
+
+        self._is_running = True
+        self._message_handle_coro = asyncio.ensure_future(self._handle_messages())
+        self._load_history()
+
+    def __del__(self):
+        self._save_history()
+
+    async def stop(self):
+        self._is_running = False
+        self._message_queue.put_nowait(None)
+        await self._message_handle_coro
+
+    def set_sio(self, sio: SioConnection):
+        self._sio = sio
+
+    async def _handle_messages(self):
+        try:
+            while self._is_running:
+                msg = await self._message_queue.get()
+                if msg is None:
+                    break
+                logger.debug(f"Got one message from {msg.message_content}, id{msg.message_id}")
+                self._message_user_id_ = msg.user_id
+                self._message_id_ = msg.message_id
+                try:
+                    await self._agent.feed_prompt(msg.message_content)
+                except (InterruptedError, asyncio.CancelledError) as e:
+                    logger.info("_handle_messages coroutine interrupted, exit")
+                    break
+                except BaseException as e:
+                    if isinstance(e, function_error.FunctionError) and e.code == function_error.EC_RESET:
+                        assert self._message_id_ is None
+                    else:
+                        if self._message_id_ is not None:
+                            logger.error(f"Failed to handle request: {str(e)}")
+                            await self._safe_reply_text('Sorry, failed to response your previous request')
+                finally:
+                    if self._message_id_ is not None:
+                        await self._sio.safe_emit('end', '', self._message_user_id_, self._session_id,
+                                                  self._message_id_)
+                self._save_history()
+
+            logger.debug(f"Coro exit: {self._session_id}")
+        except BaseException as e:
+            if isinstance(e, InterruptedError) or isinstance(e, asyncio.CancelledError):
+                return
+            logger.error(f"An unhandled error {str(e)}")
+
+    async def on_chat_message(self, msg: IncomingChatMessage):
+        if self._last_message_id == msg.message_id:
+            logger.warn(f'Duplicated message id, discard: {msg.message_id}')
+            return
+        else:
+            self._last_message_id = msg.message_id
+        self._message_queue.put_nowait(msg)
+
+    async def _safe_reply_text(self, msg: str):
+        try:
+            await self.reply_text(msg)
+        except:
+            logger.debug("Failed to safe reply text")
+
+    def clear_history(self):
+        self._agent.clear_history_messages()
+        while not self._message_queue.empty():
+            self._message_queue.get_nowait()
+        self._message_id_ = None
+        self._save_history()
+
+    def set_tz_offset(self, offset_hours):
+        self._tz_offset = offset_hours
+
+    def get_tz_offset(self):
+        return self._tz_offset
+
+    def get_last_image(self) -> str:
+        return self._last_image
+    
+    def set_last_image(self, img: str):
+        self._last_image = img
+
+    async def reply_text(self, msg):
+        if self._message_id_ is None:
+            raise function_error.FunctionError(function_error.EC_RESET, "Reset")
+        await self._sio.emit('text', msg, self._message_user_id_, self._session_id, self._message_id_)
+
+    async def reply_image_base64(self, msg):
+        if self._message_id_ is None:
+            raise function_error.FunctionError(function_error.EC_RESET, "Reset")
+        await self._sio.emit('image', msg, self._message_user_id_, self._session_id, self._message_id_)
+
+    async def reply_markdown(self, md):
+        if self._message_id_ is None:
+            raise function_error.FunctionError(function_error.EC_RESET, "Reset")
+        await self._sio.emit('markdown', md, self._message_user_id_, self._session_id, self._message_id_)
+
+    async def push_notification(self, msg):
+        if self._message_id_ is None:
+            raise function_error.FunctionError(function_error.EC_RESET, "Reset")
+        await self._sio.emit('notification', msg, self._message_user_id_, self._session_id, self._message_id_)
+
+    def _save_history(self):
+        if self._history_dir is None:
+            return
+        try:
+            os.makedirs(self._history_dir, exist_ok=True)
+            p = os.path.join(self._history_dir, self._session_id) + ".json"
+            self._agent.save_history(p)
+        except:
+            pass
+
+    def _load_history(self):
+        if self._history_dir is None:
+            return
+        try:
+            p = os.path.join(self._history_dir, self._session_id) + ".json"
+            self._agent.load_history(p)
+        except:
+            pass

PoC/agent_jarvis/jarvis/gpt/ai_function.py

@@ -0,0 +1,43 @@
+from typing import List
+
+from jarvis import CFG
+from jarvis.gpt import gpt
+from jarvis.logger import logger
+
+
+async def acall_ai_function(function: str, args: list, description: str, model: str | None = None) -> str:
+    """Call an AI function
+
+    This is a magic function that can do anything with no-code. See
+    https://github.com/Torantulino/AI-Functions for more info.
+
+    Args:
+        function (str): The function to call
+        args (list): The arguments to pass to the function
+        description (str): The description of the function
+        model (str, optional): The model to use. Defaults to None.
+
+    Returns:
+        str: The response from the function
+    """
+    if model is None:
+        model = CFG.small_llm_model
+    # For each arg, if any are None, convert to "None":
+    args = [str(arg) if arg is not None else "None" for arg in args]
+    # parse args to comma separated string
+    args: str = ", ".join(args)
+    messages: List[dict] = [
+        {
+            "role": "system",
+            "content": f"You are now the following python function: ```# {description}"
+                       f"\n{function}```\n\nOnly respond with your `return` value.",
+        },
+        {"role": "user", "content": args},
+    ]
+
+    logger.debug(str(messages))
+
+    msg_type, msg_content = await gpt.acreate_chat_completion(model=model, messages=messages, temperature=0)
+    if msg_type == "content":
+        return msg_content
+    return 'failed'

PoC/agent_jarvis/jarvis/gpt/gpt.py

@@ -0,0 +1,144 @@
+import asyncio
+
+import openai
+from openai.error import RateLimitError, APIError, Timeout
+
+from jarvis import CFG
+from jarvis.logger import logger
+from typing import Callable
+
+openai.api_key = CFG.openai_api_key
+if CFG.openai_url_base is not None:
+    openai.api_base = CFG.openai_url_base
+
+print_total_cost = CFG.debug_mode
+
+
+async def acreate_chat_completion_once(
+        messages: list,  # type: ignore
+        model: str | None = None,
+        temperature: float = CFG.temperature,
+        max_tokens: int | None = None,
+        deployment_id=None,
+        request_timeout=40,
+        **kwargs
+) -> str:
+    """
+    Create a chat completion and update the cost.
+    Args:
+    messages (list): The list of messages to send to the API.
+    model (str): The model to use for the API call.
+    temperature (float): The temperature to use for the API call.
+    max_tokens (int): The maximum number of tokens for the API call.
+    Returns:
+    str: The AI's response.
+    """
+    if deployment_id is not None:
+        response = await openai.ChatCompletion.acreate(
+            deployment_id=deployment_id,
+            model=model,
+            messages=messages,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            request_timeout=request_timeout,
+            **kwargs
+        )
+    else:
+        response = await openai.ChatCompletion.acreate(
+            model=model,
+            messages=messages,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            request_timeout=request_timeout,
+            **kwargs
+        )
+    if CFG.debug_mode:
+        logger.debug(f"Response: {response}")
+    # prompt_tokens = response.usage.prompt_tokens
+    # completion_tokens = response.usage.completion_tokens
+    return response
+
+
+# Overly simple abstraction until we create something better
+# simple retry mechanism when getting a rate error or a bad gateway
+async def acreate_chat_completion(
+        messages: list[dict],
+        model: str = None,
+        temperature: float = CFG.temperature,
+        max_tokens: int = None,
+        request_timeout: int = 40,
+        num_retries=3,
+        on_single_request_timeout: Callable = None,
+        **kwargs
+):
+    """Create a chat completion using the OpenAI API
+
+    Args:
+        messages (List[dict]): The messages to send to the chat completion
+        model (str, optional): The model to use. Defaults to None.
+        temperature (float, optional): The temperature to use. Defaults to 0.9.
+        max_tokens (int, optional): The max tokens to use. Defaults to None.
+        request_timeout (int, optional): The request_timeout of a single openai request.
+        num_retries (int, optional): The max retries.
+        on_single_request_timeout (Callable, optional): This function will be called each time a single openai request
+            timeout, must be an async function, the last timeout will not emit callback.
+
+    Returns:
+        str: The response from the chat completion
+    """
+    if CFG.debug_mode:
+        logger.debug(
+            f"Creating chat completion with model {model}, temperature {temperature}, max_tokens {max_tokens}"
+        )
+
+    response = None
+
+    for attempt in range(num_retries):
+        backoff = min(2 ** (attempt + 2), 8)
+        try:
+            if CFG.use_azure:
+                response = await acreate_chat_completion_once(
+                    deployment_id=CFG.get_azure_deployment_id_for_model(model),
+                    model=model,
+                    messages=messages,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    request_timeout=request_timeout,
+                    **kwargs
+                )
+            else:
+                response = await acreate_chat_completion_once(
+                    model=model,
+                    messages=messages,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    request_timeout=request_timeout,
+                    **kwargs
+                )
+            break
+        except RateLimitError:
+            if CFG.debug_mode:
+                logger.debug(f"Error: Reached rate limit, passing...")
+        except (APIError, Timeout) as e:
+            if isinstance(e, Timeout):
+                if on_single_request_timeout:
+                    await on_single_request_timeout(num_retries < num_retries - 1)
+            if e.http_status != 502:
+                raise
+            if attempt == num_retries - 1:
+                raise
+        if CFG.debug_mode:
+            logger.debug(
+                f"Error: API Bad gateway. Waiting {backoff} seconds..."
+            )
+        await asyncio.sleep(backoff)
+    if response is None:
+        logger.error(f"Failed to get response from GPT after {num_retries} retries")
+        raise RuntimeError(f"Failed to get response after {num_retries} retries")
+
+    choice_message = response.choices[0].message
+    content = choice_message.get("content")
+    if content is None:
+        return "function_call", {k: v for k, v in choice_message["function_call"].items()}
+    else:
+        return "content", content

PoC/agent_jarvis/jarvis/gpt/token_counter.py

@@ -0,0 +1,80 @@
+"""Functions for counting the number of tokens in a message or string."""
+from __future__ import annotations
+
+from typing import List
+import json
+
+import tiktoken_async
+
+
+async def count_message_tokens(
+        messages: List[dict], model: str = "gpt-3.5-turbo-0301"
+) -> int:
+    """
+    Returns the number of tokens used by a list of messages.
+
+    Args:
+        messages (list): A list of messages, each of which is a dictionary
+            containing the role and content of the message.
+        model (str): The name of the model to use for tokenization.
+            Defaults to "gpt-3.5-turbo-0301".
+
+    Returns:
+        int: The number of tokens used by the list of messages.
+    """
+    try:
+        encoding = await tiktoken_async.encoding_for_model(model)
+    except KeyError:
+        print("Warning: model not found. Using cl100k_base encoding.")
+        encoding = await tiktoken_async.get_encoding("cl100k_base")
+    if model == "gpt-3.5-turbo":
+        # !Note: gpt-3.5-turbo may change over time.
+        # Returning num tokens assuming Mgpt-3.5-turbo-0301.")
+        return await count_message_tokens(messages, model="gpt-3.5-turbo-0301")
+    elif model == "gpt-4":
+        # !Note: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.")
+        return await count_message_tokens(messages, model="gpt-4-0314")
+    # TODO: OpenAI has not mention how to count tokens for 0613, thus, we use the former method
+    elif model == "gpt-3.5-turbo-0301" or model == "gpt-3.5-turbo-0613" or model == "gpt-3.5-turbo-16k-0613":
+        tokens_per_message = (
+            4  # every message follows <|start|>{role/name}\n{content}<|end|>\n
+        )
+        tokens_per_name = -1  # if there's a name, the role is omitted
+    elif model == "gpt-4-0314":
+        tokens_per_message = 3
+        tokens_per_name = 1
+    else:
+        raise NotImplementedError(
+            f"num_tokens_from_messages() is not implemented for model {model}.\n"
+            " See https://github.com/openai/openai-python/blob/main/chatml.md for"
+            " information on how messages are converted to tokens."
+        )
+    num_tokens = 0
+    for message in messages:
+        num_tokens += tokens_per_message
+        for key, value in message.items():
+            if not isinstance(value, str):
+                # TODO: Since openai does not mentioned how to count tokens of 'funciton_call',
+                #   and only string is countable, thus, if the value is not a `str` (`function_call`
+                #   field of a message), we convert it into json
+                value = json.dumps(value)
+            num_tokens += len(encoding.encode(value))
+            if key == "name":
+                num_tokens += tokens_per_name
+    num_tokens += 3  # every reply is primed with <|start|>assistant<|message|>
+    return num_tokens
+
+
+async def count_string_tokens(string: str, model_name: str) -> int:
+    """
+    Returns the number of tokens in a text string.
+
+    Args:
+        string (str): The text string.
+        model_name (str): The name of the encoding to use. (e.g., "gpt-3.5-turbo")
+
+    Returns:
+        int: The number of tokens in the text string.
+    """
+    encoding = await tiktoken_async.encoding_for_model(model_name)
+    return len(encoding.encode(string))

PoC/agent_jarvis/jarvis/json_utils/json_fix_general.py

@@ -0,0 +1,122 @@
+"""This module contains functions to fix JSON strings using general programmatic approaches, suitable for addressing
+common JSON formatting issues."""
+from __future__ import annotations
+
+import contextlib
+import json
+import re
+from typing import Optional
+
+from jarvis import CFG
+from jarvis.json_utils.utilities import extract_char_position
+
+
+def fix_invalid_escape(json_to_load: str, error_message: str) -> str:
+    """Fix invalid escape sequences in JSON strings.
+
+    Args:
+        json_to_load (str): The JSON string.
+        error_message (str): The error message from the JSONDecodeError
+          exception.
+
+    Returns:
+        str: The JSON string with invalid escape sequences fixed.
+    """
+    while error_message.startswith("Invalid \\escape"):
+        bad_escape_location = extract_char_position(error_message)
+        json_to_load = (
+            json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1 :]
+        )
+        try:
+            json.loads(json_to_load)
+            return json_to_load
+        except json.JSONDecodeError as e:
+            if CFG.debug_mode:
+                print("json loads error - fix invalid escape", e)
+            error_message = str(e)
+    return json_to_load
+
+
+def balance_braces(json_string: str) -> Optional[str]:
+    """
+    Balance the braces in a JSON string.
+
+    Args:
+        json_string (str): The JSON string.
+
+    Returns:
+        str: The JSON string with braces balanced.
+    """
+
+    open_braces_count = json_string.count("{")
+    close_braces_count = json_string.count("}")
+
+    while open_braces_count > close_braces_count:
+        json_string += "}"
+        close_braces_count += 1
+
+    while close_braces_count > open_braces_count:
+        json_string = json_string.rstrip("}")
+        close_braces_count -= 1
+
+    with contextlib.suppress(json.JSONDecodeError):
+        json.loads(json_string)
+        return json_string
+
+
+def add_quotes_to_property_names(json_string: str) -> str:
+    """
+    Add quotes to property names in a JSON string.
+
+    Args:
+        json_string (str): The JSON string.
+
+    Returns:
+        str: The JSON string with quotes added to property names.
+    """
+
+    def replace_func(match: re.Match) -> str:
+        return f'"{match[1]}":'
+
+    property_name_pattern = re.compile(r"(\w+):")
+    corrected_json_string = property_name_pattern.sub(replace_func, json_string)
+
+    try:
+        json.loads(corrected_json_string)
+        return corrected_json_string
+    except json.JSONDecodeError as e:
+        raise e
+
+
+def correct_json(json_to_load: str) -> str:
+    """
+    Correct common JSON errors.
+    Args:
+        json_to_load (str): The JSON string.
+    """
+
+    try:
+        if CFG.debug_mode:
+            print("json", json_to_load)
+        json.loads(json_to_load)
+        return json_to_load
+    except json.JSONDecodeError as e:
+        if CFG.debug_mode:
+            print("json loads error", e)
+        error_message = str(e)
+        if error_message.startswith("Invalid \\escape"):
+            json_to_load = fix_invalid_escape(json_to_load, error_message)
+        if error_message.startswith(
+            "Expecting property name enclosed in double quotes"
+        ):
+            json_to_load = add_quotes_to_property_names(json_to_load)
+            try:
+                json.loads(json_to_load)
+                return json_to_load
+            except json.JSONDecodeError as e:
+                if CFG.debug_mode:
+                    print("json loads error - add quotes", e)
+                error_message = str(e)
+        if balanced_str := balance_braces(json_to_load):
+            return balanced_str
+    return json_to_load

PoC/agent_jarvis/jarvis/json_utils/json_fix_llm.py

@@ -0,0 +1,201 @@
+"""This module contains functions to fix JSON strings generated by LLM models, such as ChatGPT, using the assistance
+of the ChatGPT API or LLM models."""
+from __future__ import annotations
+
+import contextlib
+import json
+from typing import Any, Dict
+
+from regex import regex
+
+from jarvis import CFG
+from jarvis.gpt.ai_function import acall_ai_function
+from jarvis.json_utils.json_fix_general import correct_json
+from jarvis.logger import logger
+
+JSON_SCHEMA = """
+{
+    "function": {
+        "name": "function name",
+        "args": {
+            "arg name": "value"
+        }
+    },
+    "thoughts":
+    {
+        "text": "thought",
+        "reasoning": "reasoning",
+        "speak": "thoughts summary to say to user"
+    }
+}
+"""
+
+
+async def auto_fix_json(json_string: str, schema: str) -> str:
+    """Fix the given JSON string to make it parseable and fully compliant with
+        the provided schema using GPT-3.
+
+    Args:
+        json_string (str): The JSON string to fix.
+        schema (str): The schema to use to fix the JSON.
+    Returns:
+        str: The fixed JSON string.
+    """
+    # Try to fix the JSON using GPT:
+    function_string = "def fix_json(json_string: str, schema:str=None) -> str:"
+    args = [f"'''{json_string}'''", f"'''{schema}'''"]
+    description_string = (
+        "This function takes a JSON string (try to make it valid if it's not a valid JSON string) and ensures that it"
+        " is parseable and fully compliant with the provided schema. If an object"
+        " or field specified in the schema isn't contained within the correct JSON,"
+        " it is omitted. The function also escapes any double quotes within JSON"
+        " string values to ensure that they are valid. If the JSON string contains"
+        " any None or NaN values, they are replaced with null before being parsed."
+    )
+
+    # If it doesn't already start with a "`", add one:
+    if not json_string.startswith("`"):
+        json_string = "```json\n" + json_string + "\n```"
+    result_string = await acall_ai_function(
+        function_string, args, description_string, model=CFG.small_llm_model
+    )
+    print("------------ JSON FIX ATTEMPT ---------------")
+    print(f"Original JSON: {json_string}")
+    print("-----------")
+    print(f"Fixed JSON: {result_string}")
+    print("----------- END OF FIX ATTEMPT ----------------")
+
+    try:
+        json.loads(result_string)  # just check the validity
+        return result_string
+    except json.JSONDecodeError:  # noqa: E722
+        # Get the call stack:
+        # import traceback
+        # call_stack = traceback.format_exc()
+        # print(f"Failed to fix JSON: '{json_string}' "+call_stack)
+        return "failed"
+
+
+async def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]:
+    """Fix the given JSON string to make it parseable and fully compliant with two techniques.
+
+    Args:
+        json_string (str): The JSON string to fix.
+
+    Returns:
+        str: The fixed JSON string.
+    """
+
+    # Parse and print Assistant response
+    assistant_reply_json = await fix_and_parse_json(assistant_reply)
+    if assistant_reply_json == {}:
+        assistant_reply_json = await attempt_to_fix_json_by_finding_outermost_brackets(
+            assistant_reply
+        )
+
+    if assistant_reply_json != {}:
+        return assistant_reply_json
+
+    logger.debug(
+        "warn: The following AI output couldn't be converted to a JSON:\n",
+        assistant_reply,
+    )
+    # if CFG.speak_mode:
+    #     say_text("I have received an invalid JSON response from the OpenAI API.")
+
+    return {}
+
+
+async def fix_and_parse_json(
+        json_to_load: str, try_to_fix_with_gpt: bool = True
+) -> Dict[Any, Any]:
+    """Fix and parse JSON string
+
+    Args:
+        json_to_load (str): The JSON string.
+        try_to_fix_with_gpt (bool, optional): Try to fix the JSON with GPT.
+            Defaults to True.
+
+    Returns:
+        str or dict[Any, Any]: The parsed JSON.
+    """
+
+    with contextlib.suppress(json.JSONDecodeError):
+        json_to_load = json_to_load.replace("\t", "")
+        return json.loads(json_to_load)
+
+    with contextlib.suppress(json.JSONDecodeError):
+        json_to_load = correct_json(json_to_load)
+        return json.loads(json_to_load)
+    # Let's do something manually:
+    # sometimes GPT responds with something BEFORE the braces:
+    # "I'm sorry, I don't understand. Please try again."
+    # {"text": "I'm sorry, I don't understand. Please try again.",
+    #  "confidence": 0.0}
+    # So let's try to find the first brace and then parse the rest
+    #  of the string
+    try:
+        brace_index = json_to_load.index("{")
+        maybe_fixed_json = json_to_load[brace_index:]
+        last_brace_index = maybe_fixed_json.rindex("}")
+        maybe_fixed_json = maybe_fixed_json[: last_brace_index + 1]
+        return json.loads(maybe_fixed_json)
+    except (json.JSONDecodeError, ValueError) as e:
+        return await try_ai_fix(try_to_fix_with_gpt, e, json_to_load)
+
+
+async def try_ai_fix(
+        try_to_fix_with_gpt: bool, exception: Exception, json_to_load: str
+) -> Dict[Any, Any]:
+    """Try to fix the JSON with the AI
+
+    Args:
+        try_to_fix_with_gpt (bool): Whether to try to fix the JSON with the AI.
+        exception (Exception): The exception that was raised.
+        json_to_load (str): The JSON string to load.
+
+    Raises:
+        exception: If try_to_fix_with_gpt is False.
+
+    Returns:
+        str or dict[Any, Any]: The JSON string or dictionary.
+    """
+    if not try_to_fix_with_gpt:
+        raise exception
+    if CFG.debug_mode:
+        logger.debug(
+            "Warning: Failed to parse AI output, attempting to fix."
+            "\n If you see this warning frequently, it's likely that"
+            " your prompt is confusing the AI. Try changing it up"
+            " slightly."
+        )
+    # Now try to fix this up using the ai_functions
+    ai_fixed_json = await auto_fix_json(json_to_load, JSON_SCHEMA)
+
+    if ai_fixed_json != "failed":
+        return json.loads(ai_fixed_json)
+    # This allows the AI to react to the error message,
+    #   which usually results in it correcting its ways.
+    # logger.error("Failed to fix AI output, telling the AI.")
+    return {}
+
+
+async def attempt_to_fix_json_by_finding_outermost_brackets(json_string: str):
+    try:
+        json_pattern = regex.compile(r"\{(?:[^{}]|(?R))*\}")
+        json_match = json_pattern.search(json_string)
+
+        if json_match:
+            # Extract the valid JSON object from the string
+            json_string = json_match.group(0)
+            logger.debug("Apparently json was fixed.")
+        else:
+            return {}
+
+    except (json.JSONDecodeError, ValueError):
+        if CFG.debug_mode:
+            logger.debug(f"Error: Invalid JSON: {json_string}\n")
+        logger.error("Error: Invalid JSON, setting it to empty JSON now.\n")
+        json_string = {}
+
+    return await fix_and_parse_json(json_string)

PoC/agent_jarvis/jarvis/json_utils/llm_response_format_1.json

@@ -0,0 +1,31 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "properties": {
+        "thoughts": {
+            "type": "object",
+            "properties": {
+                "text": {"type": "string"},
+                "reasoning": {"type": "string"},
+                "plan": {"type": "string"},
+                "criticism": {"type": "string"},
+                "speak": {"type": "string"}
+            },
+            "required": [],
+            "additionalProperties": false
+        },
+        "function": {
+            "type": "object",
+            "properties": {
+                "name": {"type": "string"},
+                "args": {
+                    "type": "object"
+                }
+            },
+            "required": ["name"],
+            "additionalProperties": false
+        }
+    },
+    "required": ["thoughts", "function"],
+    "additionalProperties": false
+}

PoC/agent_jarvis/jarvis/json_utils/utilities.py

@@ -0,0 +1,51 @@
+"""Utilities for the json_fixes package."""
+import json
+import re
+
+from jsonschema import Draft7Validator
+
+from jarvis import CFG
+from jarvis.logger import logger
+
+
+def extract_char_position(error_message: str) -> int:
+    """Extract the character position from the JSONDecodeError message.
+
+    Args:
+        error_message (str): The error message from the JSONDecodeError
+          exception.
+
+    Returns:
+        int: The character position.
+    """
+
+    char_pattern = re.compile(r"\(char (\d+)\)")
+    if match := char_pattern.search(error_message):
+        return int(match[1])
+    else:
+        raise ValueError("Character position not found in the error message.")
+
+
+def validate_json(json_object: object, schema_name: object) -> object:
+    """
+    :type schema_name: object
+    :param schema_name:
+    :type json_object: object
+    """
+    with open(f"jarvis/json_utils/{schema_name}.json", "r") as f:
+        schema = json.load(f)
+    validator = Draft7Validator(schema)
+
+    if errors := sorted(validator.iter_errors(json_object), key=lambda e: e.path):
+        logger.debug("The JSON object is invalid.")
+        if CFG.debug_mode:
+            # Replace 'json_object' with the variable containing the JSON data
+            logger.debug(json.dumps(json_object, indent=4))
+            logger.debug("The following issues were found:")
+
+            for error in errors:
+                logger.debug(f"Error: {error.message}")
+    elif CFG.debug_mode:
+        logger.debug("The JSON object is valid.")
+
+    return json_object

PoC/agent_jarvis/jarvis/logger.py

@@ -0,0 +1,23 @@
+import logging
+
+from jarvis import CFG
+
+
+def _init_logger():
+    pass
+
+
+_init_logger()
+
+logger = logging.getLogger("main_logger")
+logger.setLevel(CFG.log_level)
+
+file_handler = logging.FileHandler('log.txt')
+console_handler = logging.StreamHandler()
+
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+logger.addHandler(file_handler)
+logger.addHandler(console_handler)

PoC/agent_jarvis/jarvis/main.py

@@ -0,0 +1,199 @@
+import asyncio
+import os
+import sys
+import time
+
+from jarvis import CFG
+from jarvis.logger import logger
+from pathlib import Path
+
+from aiohttp import web
+import socketio
+import socketio.exceptions
+
+import importlib.util
+from importlib.machinery import SourceFileLoader
+from jarvis.gateway.session import Session, SioServerConnection, SioClientConnection
+from jarvis.utils.incoming_chat_message_parser import parse_incoming_chat_message
+
+
+def _import_external_functions():
+    def import_recursive(path: str):
+        files = os.listdir(path)
+        no_subdir = False
+        for file in files:
+            if file.endswith(".module.py"):
+                # If a module file exists, then it's the only module we are going to load
+                full_path = os.path.join(path, file)
+                # Add the module path
+                sys.path.append(os.path.dirname(full_path))
+                SourceFileLoader(full_path, full_path).load_module()
+                no_subdir = True
+
+        if not no_subdir:
+            # This is not the root of a module, let's dig in
+            for file in files:
+                full_path = os.path.join(path, file)
+                if os.path.isdir(full_path):
+                    import_recursive(full_path)
+
+    import_recursive(CFG.external_function_module_dirs)
+
+
+def _import_functions():
+    py_files = []
+    dir_path = os.path.join(Path(__file__).parent, "functional_modules")
+    for file in os.listdir(dir_path):
+        if file.endswith(".py"):
+            py_files.append(file)
+
+    for file in py_files:
+        if file == "functional_module.py" or file == "caller_context.py":
+            continue
+        SourceFileLoader(file, os.path.join("jarvis/functional_modules", file)).load_module()
+
+    _import_external_functions()
+
+
+logger.info("Registering functions...")
+_import_functions()
+
+
+def run_server_mode():
+    logger.info("Starting server...")
+
+    async def index(request):
+        """Serve the client-side application."""
+        with open('./TestPage/index.html') as f:
+            return web.Response(text=f.read(), content_type='text/html')
+
+    app = web.Application()
+    session_map = {}
+
+    sio: socketio.AsyncServer = socketio.AsyncServer(
+        max_http_buffer_size=50000000,  # 50M
+    )
+    sio.attach(app)
+
+    @sio.event
+    def connect(sid, environ):
+        logger.debug(f"connect {sid}")
+        session_map.update({sid: Session(SioServerConnection(sio, sid), sid)})
+
+    @sio.event
+    async def disconnect(sid):
+        logger.debug(f'disconnect {sid}')
+        session: Session = session_map[sid]
+        session_map.update({sid: None})
+        await session.stop()
+
+    @sio.on('chat_message')
+    async def chat_message(sid, data):
+        logger.debug(f"message {data}")
+        msg = parse_incoming_chat_message(data)
+        if msg is None:
+            return
+
+        session = session_map[sid]
+        if session is None:
+            logger.debug(f"Error: session {sid} not found!")
+            return
+
+        if msg.message_type == 'clear':
+            session.clear_history()
+        elif msg.message_type == 'set_ts_offset':
+            offset = int(msg.message_content)
+            if offset > 12 or offset < -12:
+                logger.error(f"Invalid tz offset: {msg.message_content}")
+                return
+            session.set_tz_offset(offset)
+        elif msg.message_type == 'text':
+            await session.on_chat_message(msg)
+        elif msg.message_type == 'image':
+            session.set_last_image(msg.message_content)
+
+    app.router.add_static('/js', './TestPage/js')
+    app.router.add_static('/css', './TestPage/css')
+    app.router.add_get('/', index)
+    web.run_app(app, host='0.0.0.0', port=CFG.server_mode_port)
+
+
+async def run_client_mode(session_map: dict[str, Session]):
+    sio = socketio.AsyncClient()
+    # The connection is re-established, thus re-set sio of all sessions.
+    for s in session_map.values():
+        s.set_sio(SioClientConnection(sio))
+
+    # @sio.event
+    @sio.on('connect')
+    def connect():
+        logger.debug(f"connected")
+
+    @sio.event
+    def disconnect():
+        logger.debug(f'disconnected')
+        # Do nothing, sessions will not be proactively destoryed in this mode.
+
+    @sio.on('chat_message')
+    async def chat_message(data):
+        logger.debug(f"message {data}")
+        msg = parse_incoming_chat_message(data)
+        if msg is None:
+            return
+        sid = msg.chat_id
+        if sid in session_map.keys():
+            session = session_map[sid]
+            assert session is not None
+        else:
+            session = Session(SioClientConnection(sio), sid)
+            session_map.update({sid: session})
+
+        if msg.message_type == 'clear':
+            session.clear_history()
+        elif msg.message_type == 'set_ts_offset':
+            offset = int(msg.message_content)
+            if offset > 12 or offset < -12:
+                logger.error(f"Invalid tz offset: {msg.message_content}")
+                return
+            session.set_tz_offset(offset)
+        elif msg.message_type == 'text':
+            await session.on_chat_message(msg)
+        elif msg.message_type == 'image':
+            session.set_last_image(msg.message_content)
+
+    await sio.connect(CFG.bot_server_url)
+    try:
+        await sio.wait()
+    except:
+        # I don't known why, but if we don't catch here, the logger.debug below will 
+        # die when the program is interrupted by SIGINT
+        raise
+    finally:
+        del sio
+        logger.debug("Client mode end")
+
+
+async def run_client_mode_async(session_map: dict[str, Session]):
+    while True:
+        try:
+            await run_client_mode(session_map)
+        except (InterruptedError, asyncio.CancelledError):
+            logger.info(f"Interrupted, exit...")
+            break
+        except BaseException as e:
+            logger.error(f"Failed to run in client mode, try again 1 seconds later: {str(e)}")
+            await asyncio.sleep(1)
+
+
+def main():
+    if CFG.is_server_mode:
+        run_server_mode()
+    else:
+        session_map = {}
+        asyncio.run(run_client_mode_async(session_map))
+
+
+if __name__ == '__main__':
+    main()
+
+logger.debug("End jarvis")

PoC/agent_jarvis/jarvis/utils/asynchttp.py

@@ -0,0 +1,29 @@
+import asyncio
+import json
+import time
+from typing import List, Dict
+import aiohttp
+
+
+async def do_post(url, body, params=None) -> Dict | List:
+    if not isinstance(body, str):
+        body = json.dumps(body)
+    headers = {
+        'accept': 'application/json',
+        'Content-Type': 'application/json',
+    }
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, headers=headers, data=body, params=params) as response:
+            return await response.json()
+
+
+async def do_get(url, params=None) -> Dict | List:
+    headers = {
+        'accept': 'application/json',
+        'Content-Type': 'application/json',
+    }
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, headers=headers, params=params) as response:
+            return await response.json()

PoC/agent_jarvis/jarvis/utils/function_error.py

@@ -0,0 +1,13 @@
+EC_SUCCESS = 0
+
+EC_UNKNOWN_ERROR = -1
+
+EC_RESET = 1
+
+EC_DECODE_JSON_ERROR = 100
+
+
+class FunctionError(Exception):
+    def __init__(self, code, msg):
+        self.code = code
+        self.msg = msg

PoC/agent_jarvis/jarvis/utils/incoming_chat_message_parser.py

@@ -0,0 +1,65 @@
+import json
+
+from jarvis.logger import logger
+
+
+class IncomingChatMessage:
+    user_id: str = None
+    chat_id: str = None
+    message_type: str = None
+    message_content: str = None
+    message_id: str = None
+
+    def __init__(self):
+        pass
+
+
+def parse_incoming_chat_message(data: str | dict):
+    """
+    The expected format of data is
+    {
+      user: {
+        id: string
+      }
+      chat: {
+        id: string
+      }
+      message: {
+        type: 'text' | 'voice' | ...
+        content: string
+        id: string
+      }
+    }
+    """
+    result = IncomingChatMessage()
+    try:
+        if isinstance(data, dict):
+            obj = data
+        else:
+            obj = json.loads(data)
+        result.user_id = obj["user"]["id"]
+        result.chat_id = obj["chat"]["id"]
+        result.message_type = obj["message"]["type"]
+        result.message_id = obj["message"]["id"]
+        result.message_content = obj["message"]["content"]
+        # TODO: Check if they are str
+        return result
+    except Exception as e:
+        logger.debug(f"An invalid message from session: {data}")
+        return None
+
+
+def assemble_json_message(msg_type: str, msg: str, user_id: str, session_id: str, message_id: str):
+    return {
+        "user": {
+            "id": user_id
+        },
+        "chat": {
+            "id": session_id
+        },
+        "message": {
+            "type": msg_type,
+            "content": msg,
+            "id": message_id
+        }
+    }