Refactor the code directory structure to better suit the current complexity

src/aios/environment/asr_function.py

@@ -0,0 +1,55 @@
+import logging
+from typing import Dict
+
+from ..frame.compute_kernel import ComputeKernel
+from ..agent.ai_function import AIFunction
+
+logger = logging.getLogger(__name__)
+
+
+class AsrFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "speech_to_text"
+        self.description = "语音识别，将语音转换为文字"
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "audio_file": {"type": "string", "description": "音频文件路径"},
+                "model": {"type": "string", "description": "识别模型", "enum": ["openai-whisper"]},
+                "prompt": {"type": "string", "description": "提示语句,可以为None"},
+                "response_format": {"type": "string", "description": "返回格式", "enum": ["text", "json", "srt", "verbose_json", "vtt"]},
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute asr function: {kwargs}")
+
+        audio_file = kwargs.get("audio_file")
+        model = kwargs.get("model")
+        prompt = kwargs.get("prompt")
+        response_format = kwargs.get("response_format")
+        if response_format is None:
+            response_format = "text"
+
+        result = await ComputeKernel.get_instance().do_speech_to_text(audio_file, model, prompt, response_format)
+        if result is not None:
+            return f"exec speech_to_text Ok. {response_format} is\n```\n{result.result_str}\n```"
+        else:
+            return "exec speech_to_text failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False

src/aios/environment/code_interpreter.py

@@ -0,0 +1,426 @@
+import logging
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+import re
+import time
+import ast
+from concurrent.futures import ThreadPoolExecutor
+from hashlib import md5
+from typing import Optional, Union, List, Tuple
+from generic_escape import GenericEscape
+
+from ..storage.storage import AIStorage
+
+try:
+    import docker
+except ImportError:
+    docker = None
+
+CODE_BLOCK_PATTERN = r"```[ \t]*(\w+)?[ \t]*\r?\n(.*?)\r?\n[ \t]*```"
+UNKNOWN = "unknown"
+TIMEOUT_MSG = "Timeout"
+DEFAULT_TIMEOUT = 600
+WIN32 = sys.platform == "win32"
+PATH_SEPARATOR = WIN32 and "\\" or "/"
+
+logger = logging.getLogger(__name__)
+
+
+BUILT_IN_MODULES = set(
+    [
+        "sys",
+        "os",
+        "math",
+        "random",
+        "datetime",
+        "json",
+        "re",
+        "subprocess",
+        "time",
+        "threading",
+        "logging",
+        "collections",
+        "itertools",
+        "functools",
+        "operator",
+        "pathlib",
+        "shutil",
+        "tempfile",
+        "pickle",
+        "io",
+        "argparse",
+        "typing",
+        "unittest",
+        "contextlib",
+        "abc",
+        "heapq",
+        "bisect",
+        "copy",
+        "decimal",
+        "fractions",
+        "hashlib",
+        "secrets",
+        "statistics",
+        "difflib",
+        "doctest",
+        "enum",
+        "inspect",
+        "traceback",
+        "weakref",
+        "gc",
+        "mmap",
+        "msvcrt",
+        "winreg",
+        "array",
+        "audioop",
+        "binascii",
+        "cProfile",
+        "concurrent.futures",
+        "configparser",
+        "csv",
+        "ctypes",
+        "dateutil",
+        "dis",
+        "fnmatch",
+        "getopt",
+        "glob",
+        "gzip",
+        "pdb",
+        "pprint",
+        "profile",
+        "pstats",
+        "queue",
+        "socket",
+        "sqlite3",
+        "ssl",
+        "struct",
+        "tarfile",
+        "telnetlib",
+        "timeit",
+        "tokenize",
+        "uuid",
+        "xml",
+        "zipfile",
+        "zlib",
+    ]
+)
+
+
+def get_imports(code: str) -> List[str]:
+    root = ast.parse(code)
+
+    imports = []
+    for node in ast.iter_child_nodes(root):
+        if isinstance(node, ast.Import):
+            module_names = [alias.name for alias in node.names]
+        elif isinstance(node, ast.ImportFrom):
+            module_names = [node.module]
+        else:
+            continue
+
+        for name in module_names:
+            # Exclude built-in modules
+            if name not in BUILT_IN_MODULES:
+                imports.append(name)
+
+    return imports
+
+
+def write_requirements(code: str, requirements_filepath: str):
+    imports = get_imports(code)
+
+    with open(requirements_filepath, "w") as file:
+        for module in imports:
+            file.write(module + "\n")
+
+
+def _cmd(lang):
+    if lang.startswith("python") or lang in ["bash", "sh", "powershell"]:
+        return lang
+    if lang in ["shell"]:
+        return "sh"
+    if lang in ["ps1"]:
+        return "powershell"
+    raise NotImplementedError(f"{lang} not recognized in code execution")
+
+
+def create_runner(code: str, timeout: int = 30) -> str:
+    """
+    Create a Python script that runs the code and prints the output
+    """
+    code = GenericEscape().escape(code)
+    # Create a runner script
+    runner = f"""
+import os
+import subprocess
+
+my_env = os.environ.copy()
+my_env["PYTHONIOENCODING"] = "utf-8"
+
+process = subprocess.Popen(
+    f"python -i -q -u".split(),
+    stdin=subprocess.PIPE,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+    text=True,
+    bufsize=0,
+    universal_newlines=True,
+    env=my_env
+)
+
+process.stdin.write("{code}" + "\\n")
+process.stdin.write("exit()\\n")
+process.stdin.flush()
+
+try:
+    process.wait({timeout})
+except Exception as e:
+    process.terminate()
+
+for line in iter(process.stdout.readline, ""):
+    print(line)
+    
+for line in iter(process.stderr.readline, ""):
+    if line.startswith(">>>"):
+        continue
+    print(line)
+"""
+    return runner
+
+
+def _run_cmd(cmd: [str], work_dir: str, timeout: int) -> str:
+    if WIN32:
+        logger.warning("SIGALRM is not supported on Windows. No timeout will be enforced.")
+        result = subprocess.run(
+            cmd,
+            cwd=work_dir,
+            capture_output=True,
+            text=True,
+        )
+    else:
+        with ThreadPoolExecutor(max_workers=1) as executor:
+            future = executor.submit(
+                subprocess.run,
+                cmd,
+                cwd=work_dir,
+                capture_output=True,
+                text=True,
+            )
+            result = future.result(timeout=timeout)
+    return result
+
+
+def execute_code(
+        code: Optional[str] = None,
+        timeout: Optional[int] = None,
+        filename: Optional[str] = None,
+        work_dir: Optional[str] = None,
+        use_docker: Optional[Union[List[str], str, bool]] = None,
+        lang: Optional[str] = "python",
+) -> Tuple[int, str]:
+    """Execute code in a docker container.
+    This function is not tested on MacOS.
+
+    Args:
+        code (Optional, str): The code to execute.
+            If None, the code from the file specified by filename will be executed.
+            Either code or filename must be provided.
+        timeout (Optional, int): The maximum execution time in seconds.
+            If None, a default timeout will be used. The default timeout is 600 seconds. On Windows, the timeout is not enforced when use_docker=False.
+        filename (Optional, str): The file name to save the code or where the code is stored when `code` is None.
+            If None, a file with a randomly generated name will be created.
+            The randomly generated file will be deleted after execution.
+            The file name must be a relative path. Relative paths are relative to the working directory.
+        work_dir (Optional, str): The working directory for the code execution.
+            If None, a default working directory will be used.
+            The default working directory is the "extensions" directory under
+            "path_to_autogen".
+        use_docker (Optional, list, str or bool): The docker image to use for code execution.
+            If a list or a str of image name(s) is provided, the code will be executed in a docker container
+            with the first image successfully pulled.
+            If None, False or empty, the code will be executed in the current environment.
+            Default is None, which will be converted into an empty list when docker package is available.
+            Expected behaviour:
+                - If `use_docker` is explicitly set to True and the docker package is available, the code will run in a Docker container.
+                - If `use_docker` is explicitly set to True but the Docker package is missing, an error will be raised.
+                - If `use_docker` is not set (i.e., left default to None) and the Docker package is not available, a warning will be displayed, but the code will run natively.
+            If the code is executed in the current environment,
+            the code must be trusted.
+        lang (Optional, str): The language of the code. Default is "python".
+
+    Returns:
+        int: 0 if the code executes successfully.
+        str: The error message if the code fails to execute; the stdout otherwise.
+    """
+    if all((code is None, filename is None)):
+        error_msg = f"Either {code=} or {filename=} must be provided."
+        logger.error(error_msg)
+        raise AssertionError(error_msg)
+
+    # Warn if use_docker was unspecified (or None), and cannot be provided (the default).
+    # In this case the current behavior is to fall back to run natively, but this behavior
+    # is subject to change.
+    if use_docker is None:
+        if docker is None:
+            use_docker = False
+            logger.warning(
+                "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change"
+            )
+        else:
+            # Default to true
+            use_docker = True
+
+    timeout = timeout or DEFAULT_TIMEOUT
+    original_filename = filename
+    if WIN32 and lang in ["sh", "shell"] and (not use_docker):
+        lang = "ps1"
+    if filename is None:
+        code_hash = md5(code.encode()).hexdigest()
+        # create a file with a automatically generated name
+        filename = f"tmp_code_{code_hash}.{'py' if lang.startswith('python') else lang}"
+    if work_dir is None:
+        WORKING_DIR = os.path.join(AIStorage.get_instance().get_myai_dir(), "tmp_code")
+        pathlib.Path(WORKING_DIR).mkdir(exist_ok=True)
+        work_dir = os.path.join(WORKING_DIR, code_hash)
+        pathlib.Path(work_dir).mkdir(exist_ok=True)
+    filepath = os.path.join(work_dir, filename)
+    file_dir = os.path.dirname(filepath)
+    os.makedirs(file_dir, exist_ok=True)
+    if code is not None:
+        write_requirements(code, os.path.join(file_dir, "requirements.txt"))
+        code = create_runner(code, 30)
+        with open(filepath, "w", encoding="utf-8") as fout:
+            fout.write(code)
+
+
+    # check if already running in a docker container
+    in_docker_container = os.path.exists("/.dockerenv")
+    if not use_docker or in_docker_container:
+        try:
+            env_cmd = ["python", "-m", "venv", os.path.join(file_dir, "venv")]
+            _run_cmd(env_cmd, file_dir, timeout)
+            if WIN32:
+                venv_path = os.path.join(file_dir, "venv", "Scripts")
+            else:
+                venv_path = os.path.join(file_dir, "venv", "bin")
+            pip_cmd = [os.path.join(venv_path, "python"), "-m", "pip", "install", "-r", "requirements.txt"]
+            _run_cmd(pip_cmd, file_dir, timeout)
+            # already running in a docker container
+            cmd = [
+                os.path.join(venv_path, "python"),
+                f".\\{filename}" if WIN32 else filename,
+            ]
+            result = _run_cmd(cmd, file_dir, timeout)
+        except TimeoutError:
+            if original_filename is None:
+                shutil.rmtree(os.path.join(file_dir, "venv"))
+                os.remove(filepath)
+                os.remove(os.path.join(file_dir, "requirements.txt"))
+                try:
+                    os.removedirs(file_dir)
+                except Exception:
+                    pass
+            return 1, TIMEOUT_MSG
+        if original_filename is None:
+            shutil.rmtree(os.path.join(file_dir, "venv"))
+            os.remove(filepath)
+            os.remove(os.path.join(file_dir, "requirements.txt"))
+            try:
+                os.removedirs(file_dir)
+            except Exception:
+                pass
+        if result.returncode:
+            logs = result.stderr
+            if original_filename is None:
+                abs_path = str(pathlib.Path(filepath).absolute())
+                logs = logs.replace(str(abs_path), "").replace(filename, "")
+            else:
+                abs_path = str(pathlib.Path(work_dir).absolute()) + PATH_SEPARATOR
+                logs = logs.replace(str(abs_path), "")
+        else:
+            logs = result.stdout
+        return result.returncode, logs
+
+    # create a docker client
+    client = docker.from_env()
+    image_list = (
+        ["python:3-alpine", "python:3", "python:3-windowsservercore"]
+        if use_docker is True
+        else [use_docker]
+        if isinstance(use_docker, str)
+        else use_docker
+    )
+    for image in image_list:
+        # check if the image exists
+        try:
+            client.images.get(image)
+            break
+        except docker.errors.ImageNotFound:
+            # pull the image
+            logger.info("Pulling image", image)
+            try:
+                client.images.pull(image, stream=True, decode=True)
+                break
+            except docker.errors.DockerException as e:
+                logger.error("Failed to pull image", image)
+                logger.exception(e)
+    # get a randomized str based on current time to wrap the exit code
+    exit_code_str = f"exitcode{time.time()}"
+    start_str = f'start{time.time()}'
+    abs_path = pathlib.Path(work_dir).absolute()
+    cmd = [
+        "sh",
+        "-c",
+        f"pip install --quiet -r requirements.txt; echo -n {start_str}; {_cmd(lang)} {filename}; exit_code=$?; echo -n {exit_code_str}; echo -n $exit_code; echo {exit_code_str};",
+    ]
+    # create a docker container
+    container = client.containers.run(
+        image,
+        command=cmd,
+        working_dir="/workspace",
+        detach=True,
+        # get absolute path to the working directory
+        volumes={abs_path: {"bind": "/workspace", "mode": "rw"}},
+    )
+    start_time = time.time()
+    while container.status != "exited" and time.time() - start_time < timeout:
+        # Reload the container object
+        container.reload()
+    if container.status != "exited":
+        container.stop()
+        container.remove()
+        if original_filename is None:
+            os.remove(filepath)
+        return 1, TIMEOUT_MSG, image
+    # get the container logs
+    logs: str = container.logs().decode("utf-8").rstrip()
+    start_pos = logs.find(start_str)
+    if start_pos != -1:
+        logs = logs[start_pos + len(start_str):]
+    # # commit the image
+    # tag = filename.replace("/", "")
+    # container.commit(repository="python", tag=tag)
+    # remove the container
+    container.remove()
+    # check if the code executed successfully
+    exit_code = container.attrs["State"]["ExitCode"]
+    if exit_code == 0:
+        # extract the exit code from the logs
+        pattern = re.compile(f"{exit_code_str}(\\d+){exit_code_str}")
+        match = pattern.search(logs)
+        exit_code = 1 if match is None else int(match.group(1))
+        # remove the exit code from the logs
+        logs = logs if match is None else pattern.sub("", logs)
+
+    if original_filename is None:
+        os.remove(filepath)
+        os.remove(os.path.join(file_dir, "requirements.txt"))
+        os.removedirs(file_dir)
+    if exit_code:
+        logs = logs.replace(f"/workspace/{filename if original_filename is None else ''}", "")
+    # return the exit code, logs and image
+    return exit_code, logs
+

src/aios/environment/code_interpreter_function.py

@@ -0,0 +1,41 @@
+from typing import Dict
+
+from ..agent.ai_function import AIFunction
+from .code_interpreter import execute_code
+
+
+class CodeInterpreterFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "code_interpreter"
+        self.description = "execute python code"
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "code": {"type": "string", "description": "python code"}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        code = kwargs.get("code")
+        ret_code, result = execute_code(code=code)
+        if ret_code == 0:
+            return result.strip()
+        else:
+            return result.strip()
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False

src/aios/environment/duckduckgo_text_search_function.py

@@ -0,0 +1,52 @@
+import json
+from typing import Dict
+
+from ..agent.ai_function import AIFunction
+from duckduckgo_search import AsyncDDGS
+
+
+class DuckDuckGoTextSearchFunction(AIFunction):
+    def __init__(self):
+        self.name = "duckduckgo_text_search"
+        self.description = "Search text from duckduckgo.com"
+        self.region = "wt-wt"
+        self.safesearch = "moderate"
+        self.time = "y"
+        self.max_results = 5
+
+    def get_name(self) -> str:
+        return self.name
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {"type": "object",
+                "properties": {
+                    "query": {"type": "string", "description": "The query to search for."}
+                }
+                }
+
+    async def execute(self, **kwargs) -> str:
+        query = kwargs.get("query")
+
+        async with AsyncDDGS() as ddgs:
+            results = [r async for r in ddgs.text(
+                query,
+                region=self.region,
+                safesearch=self.safesearch,
+                timelimit=self.time,
+                backend="api",
+                max_results=self.max_results
+            )]
+
+            return json.dumps(results)
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False

src/aios/environment/environment.py

@@ -0,0 +1,148 @@
+# basic environment class
+# we have some built-in environment: Calender(include timer),Home(connect to IoT device in your home), ,KnwoledgeBase,FileSystem,
+
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Optional,Dict,Awaitable,List
+import logging
+
+from ..agent.ai_function import AIFunction
+
+logger = logging.getLogger(__name__)
+
+class EnvironmentEvent(ABC):
+    @abstractmethod
+    def display(self) -> str:
+        pass
+
+EnvironmentEventHandler = Callable[[str,EnvironmentEvent],Awaitable[Any]]
+
+class Environment:
+    _all_env = {}
+    @classmethod
+    def get_env_by_id(cls,env_id:str):
+        return cls._all_env.get(env_id)
+
+    @classmethod
+    def set_env_by_id(cls,id,env):
+        assert id == env.get_id()
+        cls._all_env[env.get_id()] = env
+
+    def __init__(self,env_id:str) -> None:
+        self.env_id = env_id
+        self.values:Dict[str,str] = {}
+        self.get_handlers:Dict[str,Callable] = {}
+        self.owner_env:Dict[str,Environment] = {}
+        # self.valid_keys:Dict[str,bool] = None
+        self.event_handlers:Dict[str,List[EnvironmentEventHandler]]= {}
+
+        self.functions : Dict[str,AIFunction] = {}
+
+    def get_id(self) -> str:
+        return self.env_id
+
+    def add_owner_env(self,env) -> None:
+        self.owner_env[env.get_id()] = env
+
+    #@abstractmethod
+    #TODO: how to use env? different env has different prompt
+    def get_env_prompt(self) -> str:
+       pass
+
+    def add_ai_function(self,func:AIFunction) -> None:
+        if self.functions.get(func.get_name()) is not None:
+            logger.warn(f"add ai_function {func.get_name()} in env {self.env_id}:function already exist")
+
+        self.functions[func.get_name()] = func
+
+    def get_ai_function(self,func_name:str) -> AIFunction:
+        func = self.functions.get(func_name)
+        if func is not None:
+            return func
+
+        for owner_env in self.owner_env.values():
+            func = owner_env.get_ai_function(func_name)
+            if func is not None:
+                return func
+
+        return None
+
+    #def enable_ai_function(self,func_name:str) -> None:
+    #    pass
+
+    #def disable_ai_function(self,func_name:str) -> None:
+    #    pass
+
+    def get_all_ai_functions(self) -> List[AIFunction]:
+        func_list = []
+        func_list.extend(self.functions.values())
+        for owner_env in self.owner_env.values():
+            func_list.extend(owner_env.get_all_ai_functions())
+        return func_list
+
+    @abstractmethod
+    def _do_get_value(self,key:str) -> Optional[str]:
+        pass
+
+    def register_get_handler(self,key:str,handler:Callable) -> None:
+        h = self.get_handlers.get(key)
+        if h is not None:
+            logger.warn(f"register get_handler {key} in env {self.env_id}:handler already exist")
+
+        self.get_handlers[key] = handler
+
+
+    def attach_event_handler(self,event_id:str,handler:Callable) -> None:
+        handler_list = self.event_handlers.get(event_id)
+        if handler_list is None:
+            handler_list = []
+            self.event_handlers[event_id] = handler_list
+
+        handler_list.append(handler)
+
+    def remove_event_handler(self,event_id:str,handler:Callable) -> None:
+        handler_list = self.event_handlers.get(event_id)
+        if handler is not None:
+            handler_list.remove(handler)
+            return
+
+        logger.warn(f"remove event_handler {event_id} in env {self.env_id}:handler not found")
+
+    async def fire_event(self,event_id:str,event:EnvironmentEvent) -> None:
+        handler_list = self.event_handlers.get(event_id)
+        if handler_list is not None:
+            for handler in handler_list:
+                await handler(self.env_id,event)
+        else:
+            logger.debug(f"fire event {event_id} in env {self.env_id}:handler not found")
+        return
+
+    def __getitem__(self, key):
+        return self.get_value(key)
+
+    def get_value(self,key:str) -> Optional[str]:
+        handler = self.get_handlers.get(key)
+        if handler is not None:
+            return handler()
+
+        s = self.values.get(key)
+        if isinstance(s,str):
+            return s
+        else:
+            logger.warn(f"get value {key} in env {self.env_id} failed!,type is not str")
+
+        s = self._do_get_value(key)
+        if s is not None:
+            return s
+        if self.owner_env is not None:
+            for env in self.owner_env.values():
+                s = env.get_value(key)
+                if s is not None:
+                    return s
+
+        logger.warn(f"get value {key} in env {self.env_id} failed!,not found")
+        return None
+
+    def set_value(self, key: str, str_value: str,is_storage:bool = True):
+        logger.info(f"set value {key} in env {self.env_id} to {str_value}")
+        self.values[key] = str_value
+

src/aios/environment/image_2_text_function.py

@@ -0,0 +1,46 @@
+import logging
+from typing import Dict
+
+from ..frame.compute_kernel import ComputeKernel
+from ..agent.ai_function import AIFunction
+
+logger = logging.getLogger(__name__)
+
+class Image2TextFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "image_2_text"
+        self.description = "According to the input image file address, return the description of the image content"
+        logger.info(f"init Image2TextFunction")
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute image_2_text function: {kwargs}")
+        image_path = kwargs.get("image_path")
+        data = await ComputeKernel.get_instance().do_image_2_text(image_path, '')
+        try:
+            result = data['message']['choices'][0]['message']['content']
+        except (KeyError, TypeError, IndexError):
+            logger.error(f"image_2_text error: {data}")
+            result = ""
+        return result
+
+
+    def is_local(self) -> bool:
+        return False
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+

src/aios/environment/script_to_speech_function.py

@@ -0,0 +1,109 @@
+import io
+import logging
+import os
+import random
+from pathlib import Path
+from typing import Dict
+
+from ..agent.ai_function import AIFunction
+from ..frame.compute_kernel import ComputeKernel
+from ..storage.storage import AIStorage
+
+
+from pydub import AudioSegment
+
+logger = logging.getLogger(__name__)
+
+
+class ScriptToSpeechFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "script_to_speech"
+        self.description = "根据输入的剧本生成音频文件，成功时会返回音频文件路径"
+        self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts")
+        Path(self.speech_path).mkdir(exist_ok=True)
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
+                "model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]},
+                "roles": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "gender": {"type": "string", "description": "角色性别", "enum": ["man", "female"]},
+                        "age": {"type": "string", "description": "年龄", "enum": ["child", "adult"]},
+                    }}},
+                "lines": {"type": "array", "items": {
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "description": "角色名字"},
+                        "tone": {"type": "string", "description": "演播情感",
+                                 "enum": ["happy", "sad", "angry", "fear", "disgust", "surprise", "neutral"]},
+                        "text": {"type": "string", "description": "台词"},
+                    }
+                }}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute text_to_speech function: {kwargs}")
+
+        language = kwargs.get("language")
+        if language is None:
+            language = "zh"
+        model = kwargs.get("model")
+        roles = kwargs.get("roles")
+        lines = kwargs.get("lines")
+
+        audio = None
+        for line in lines:
+            name = line.get("name")
+            tone = line.get("tone")
+            text = line.get("text")
+            gender = None
+            age = None
+            for role in roles:
+                role_name = role.get("name")
+                if role_name == name:
+                    gender = role.get("gender")
+                    age = role.get("age")
+                    break
+            i = 0
+            while i < 3:
+                try:
+                    data = await ComputeKernel.get_instance().do_text_to_speech(text, language, gender, age, name, tone, model_name=model)
+                    if audio is None:
+                        audio = AudioSegment.from_mp3(io.BytesIO(data))
+                    else:
+                        audio = audio + AudioSegment.from_mp3(io.BytesIO(data))
+                    break
+                except Exception as e:
+                    logger.error(f"do_text_to_speech failed: {e}")
+                    i += 1
+                    continue
+
+        if audio is not None:
+            path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
+            audio.export(path, format="mp3")
+            return "exec text_to_speech OK，speech file store at ```{}```".format(path)
+        else:
+            return "exec text_to_speech failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+

src/aios/environment/simple_kb_db.py

@@ -0,0 +1,228 @@
+
+import sqlite3
+import json
+import threading
+import logging
+from datetime import datetime
+
+from typing import Optional, List
+
+logger = logging.getLogger(__name__)
+
+class SimpleKnowledgeDB:
+    def __init__(self,db_path:str):
+        self.db_path = db_path
+        self._get_conn()
+        
+    def _get_conn(self):
+        """ get db connection """
+        local = threading.local()
+        if not hasattr(local, 'conn'):
+            local.conn = self._create_connection(self.db_path)
+        return local.conn
+
+
+    def _create_connection(self, db_file):
+        """ create a database connection to a SQLite database """
+        conn = None
+        try:
+            conn = sqlite3.connect(db_file)
+        except Exception as e:
+            logger.error("Error occurred while connecting to database: %s", e)
+            return None
+
+        if conn:
+            self._create_tables(conn)
+
+        return conn
+    
+    def _create_tables(self,conn):
+        cursor = conn.cursor()
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS documents (
+                doc_path TEXT PRIMARY KEY,
+                length INTEGER,
+                last_modify TEXT,
+                doc_hash TEXT,
+                create_time TEXT
+            )
+        ''')
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS knowledge (
+                doc_hash TEXT PRIMARY KEY,
+                title TEXT,
+                summary TEXT,
+                content TEXT,
+                catalogs TEXT,
+                tags TEXT,
+                llm_title TEXT,
+                llm_summary TEXT,
+                create_time TEXT
+            )
+        ''')
+            
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_documents_doc_hash
+            ON documents (doc_hash)
+        ''')
+
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_knowledge_tags
+            ON knowledge (tags)
+        ''')
+
+        conn.commit()
+
+    def add_doc(self, doc_path: str, length: int, last_modify: str, doc_hash: Optional[str] = None):
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        cursor.execute('''
+            INSERT INTO documents (doc_path, length, last_modify, doc_hash,create_time) 
+            VALUES (?, ?, ?, ?,?)
+        ''', (doc_path, length, last_modify, doc_hash,create_time))
+        conn.commit()
+
+    def is_doc_exist(self, doc_path: str) -> bool:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT doc_path
+            FROM documents
+            WHERE doc_path = ?
+        ''', (doc_path,))
+        return len(cursor.fetchall()) > 0
+
+    def set_doc_hash(self, doc_path: str, doc_hash: str):
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            UPDATE documents
+            SET doc_hash = ?
+            WHERE doc_path = ?
+        ''', (doc_hash, doc_path))
+        conn.commit()
+    
+    def get_docs_without_hash(self,limit:int=1024) -> List[str]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT doc_path
+            FROM documents
+            WHERE doc_hash IS NULL OR doc_hash = ''
+            ORDER BY create_time DESC
+            LIMIT ?
+        ''',(limit,))
+        return [row[0] for row in cursor.fetchall()]
+
+    #metadata["summary"]
+    #metadata["catelogs"]
+    #metadata["tags"]
+    def add_knowledge(self, doc_hash: str, title: str, metadata: dict,content:str = None,):
+        conn = self._get_conn()
+        cursor = conn.cursor()
+
+        create_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        summary = metadata.get("summary", "")
+        catalogs = metadata.get("catalogs","")
+        tags = ','.join(metadata.get("tags", []))
+
+        cursor.execute('''
+            INSERT INTO knowledge (doc_hash, title , summary , catalogs , tags,create_time) 
+            VALUES (?, ?, ?, ?, ?,?)
+        ''', (doc_hash, title, summary, catalogs, tags,create_time))
+        conn.commit()
+
+    #llm_result["summary"]
+    #llm_result["tags"]
+    #llm_result["catelog"]
+    def set_knowledge_llm_result(self, doc_hash: str, llm_result: dict):
+        conn = self._get_conn()
+        cursor = conn.cursor()
+
+        title = llm_result.get("title", "")
+        summary = llm_result.get("summary", "")
+        catalogs = json.dumps(llm_result.get("catalogs", {}))
+        tags = ','.join(llm_result.get("tags", []))
+
+        cursor.execute('''
+            UPDATE knowledge
+            SET llm_title = ?,llm_summary = ?, catalogs = ?, tags = ?
+            WHERE doc_hash = ?
+        ''', (title,summary, catalogs, tags, doc_hash))
+        conn.commit()
+
+    def get_hash_by_doc_path(self, doc_path: str) -> Optional[str]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT doc_hash
+            FROM documents
+            WHERE doc_path = ?
+        ''', (doc_path,))
+        row = cursor.fetchone()
+        if row is None:
+            return None
+        return row[0]
+
+    def get_knowledge(self, doc_hash: str) -> Optional[dict]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT title, summary, catalogs, tags, llm_title, llm_summary
+            FROM knowledge
+            WHERE doc_hash = ?
+        ''', (doc_hash,))
+        row = cursor.fetchone()
+        if row is None:
+            return None
+        
+        # get doc path
+        cursor.execute('''
+            SELECT doc_path
+            FROM documents
+            WHERE doc_hash = ?
+        ''', (doc_hash,))
+        row2 = cursor.fetchone()
+        if row2 is None:
+            return None
+        doc_path = row2[0]
+    
+
+        return {
+            "full_path": doc_path,
+            "title": row[0],
+            "summary": row[1],
+            "catalogs": row[2],
+            "tags": row[3],
+            "llm_title" : row[4],
+            "llm_summary" : row[5],
+        }
+
+    def get_knowledge_without_llm_title(self,limit:int=16) -> List[str]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT doc_hash
+            FROM knowledge
+            WHERE llm_title IS NULL OR llm_title = ''
+            ORDER BY create_time DESC
+            LIMIT ?
+        ''',(limit,))
+        return [row[0] for row in cursor.fetchall()]
+
+    def query_docs_by_tag(self, tag: str) -> List[str]:
+        conn = self._get_conn()
+        cursor = conn.cursor()
+        tag_json = json.dumps(tag)  # 将标签转换为 JSON 字符串
+        cursor.execute('''
+            SELECT documents.doc_path
+            FROM documents
+            JOIN knowledge ON documents.doc_hash = knowledge.doc_hash
+            WHERE json_extract(knowledge.tags, '$') LIKE ?
+        ''', (tag))
+        return [row[0] for row in cursor.fetchall()]
+    
+    def query(self,sql:str):
+        pass
+        #cursor = self.conn.cursor()

src/aios/environment/sql_database.py

@@ -0,0 +1,493 @@
+"""
+Taken from: langchain
+SQLAlchemy wrapper around a database.
+"""
+from __future__ import annotations
+import os
+
+import warnings
+from typing import Any, Dict, Iterable, List, Literal, Optional, Sequence, Union
+
+import sqlalchemy
+from sqlalchemy import MetaData, Table, create_engine, inspect, select, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import ProgrammingError, SQLAlchemyError
+from sqlalchemy.schema import CreateTable
+from sqlalchemy.types import NullType
+
+
+def get_from_env(key: str, env_key: str, default: Optional[str] = None) -> str:
+    """Get a value from a dictionary or an environment variable."""
+    if env_key in os.environ and os.environ[env_key]:
+        return os.environ[env_key]
+    elif default is not None:
+        return default
+    else:
+        raise ValueError(
+            f"Did not find {key}, please add an environment variable"
+            f" `{env_key}` which contains it, or pass"
+            f"  `{key}` as a named parameter."
+        )
+
+
+def _format_index(index: sqlalchemy.engine.interfaces.ReflectedIndex) -> str:
+    return (
+        f'Name: {index["name"]}, Unique: {index["unique"]},'
+        f' Columns: {str(index["column_names"])}'
+    )
+
+
+def truncate_word(content: Any, *, length: int, suffix: str = "...") -> str:
+    """
+    Truncate a string to a certain number of words, based on the max string
+    length.
+    """
+
+    if not isinstance(content, str) or length <= 0:
+        return content
+
+    if len(content) <= length:
+        return content
+
+    return content[: length - len(suffix)].rsplit(" ", 1)[0] + suffix
+
+
+class SQLDatabase:
+    """SQLAlchemy wrapper around a database."""
+
+    def __init__(
+            self,
+            engine: Engine,
+            schema: Optional[str] = None,
+            metadata: Optional[MetaData] = None,
+            ignore_tables: Optional[List[str]] = None,
+            include_tables: Optional[List[str]] = None,
+            sample_rows_in_table_info: int = 3,
+            indexes_in_table_info: bool = False,
+            custom_table_info: Optional[dict] = None,
+            view_support: bool = False,
+            max_string_length: int = 300,
+    ):
+        """Create engine from database URI."""
+        self._engine = engine
+        self._schema = schema
+        if include_tables and ignore_tables:
+            raise ValueError("Cannot specify both include_tables and ignore_tables")
+
+        self._inspector = inspect(self._engine)
+
+        # including view support by adding the views as well as tables to the all
+        # tables list if view_support is True
+        self._all_tables = set(
+            self._inspector.get_table_names(schema=schema)
+            + (self._inspector.get_view_names(schema=schema) if view_support else [])
+        )
+
+        self._include_tables = set(include_tables) if include_tables else set()
+        if self._include_tables:
+            missing_tables = self._include_tables - self._all_tables
+            if missing_tables:
+                raise ValueError(
+                    f"include_tables {missing_tables} not found in database"
+                )
+        self._ignore_tables = set(ignore_tables) if ignore_tables else set()
+        if self._ignore_tables:
+            missing_tables = self._ignore_tables - self._all_tables
+            if missing_tables:
+                raise ValueError(
+                    f"ignore_tables {missing_tables} not found in database"
+                )
+        usable_tables = self.get_usable_table_names()
+        self._usable_tables = set(usable_tables) if usable_tables else self._all_tables
+
+        if not isinstance(sample_rows_in_table_info, int):
+            raise TypeError("sample_rows_in_table_info must be an integer")
+
+        self._sample_rows_in_table_info = sample_rows_in_table_info
+        self._indexes_in_table_info = indexes_in_table_info
+
+        self._custom_table_info = custom_table_info
+        if self._custom_table_info:
+            if not isinstance(self._custom_table_info, dict):
+                raise TypeError(
+                    "table_info must be a dictionary with table names as keys and the "
+                    "desired table info as values"
+                )
+            # only keep the tables that are also present in the database
+            intersection = set(self._custom_table_info).intersection(self._all_tables)
+            self._custom_table_info = dict(
+                (table, self._custom_table_info[table])
+                for table in self._custom_table_info
+                if table in intersection
+            )
+
+        self._max_string_length = max_string_length
+
+        self._metadata = metadata or MetaData()
+        # including view support if view_support = true
+        self._metadata.reflect(
+            views=view_support,
+            bind=self._engine,
+            only=list(self._usable_tables),
+            schema=self._schema,
+        )
+
+    @classmethod
+    def from_uri(
+            cls, database_uri: str, engine_args: Optional[dict] = None, **kwargs: Any
+    ) -> SQLDatabase:
+        """Construct a SQLAlchemy engine from URI."""
+        _engine_args = engine_args or {}
+        return cls(create_engine(database_uri, **_engine_args), **kwargs)
+
+    @classmethod
+    def from_databricks(
+            cls,
+            catalog: str,
+            schema: str,
+            host: Optional[str] = None,
+            api_token: Optional[str] = None,
+            warehouse_id: Optional[str] = None,
+            cluster_id: Optional[str] = None,
+            engine_args: Optional[dict] = None,
+            **kwargs: Any,
+    ) -> SQLDatabase:
+        """
+        Class method to create an SQLDatabase instance from a Databricks connection.
+        This method requires the 'databricks-sql-connector' package. If not installed,
+        it can be added using `pip install databricks-sql-connector`.
+
+        Args:
+            catalog (str): The catalog name in the Databricks database.
+            schema (str): The schema name in the catalog.
+            host (Optional[str]): The Databricks workspace hostname, excluding
+                'https://' part. If not provided, it attempts to fetch from the
+                environment variable 'DATABRICKS_HOST'. If still unavailable and if
+                running in a Databricks notebook, it defaults to the current workspace
+                hostname. Defaults to None.
+            api_token (Optional[str]): The Databricks personal access token for
+                accessing the Databricks SQL warehouse or the cluster. If not provided,
+                it attempts to fetch from 'DATABRICKS_TOKEN'. If still unavailable
+                and running in a Databricks notebook, a temporary token for the current
+                user is generated. Defaults to None.
+            warehouse_id (Optional[str]): The warehouse ID in the Databricks SQL. If
+                provided, the method configures the connection to use this warehouse.
+                Cannot be used with 'cluster_id'. Defaults to None.
+            cluster_id (Optional[str]): The cluster ID in the Databricks Runtime. If
+                provided, the method configures the connection to use this cluster.
+                Cannot be used with 'warehouse_id'. If running in a Databricks notebook
+                and both 'warehouse_id' and 'cluster_id' are None, it uses the ID of the
+                cluster the notebook is attached to. Defaults to None.
+            engine_args (Optional[dict]): The arguments to be used when connecting
+                Databricks. Defaults to None.
+            **kwargs (Any): Additional keyword arguments for the `from_uri` method.
+
+        Returns:
+            SQLDatabase: An instance of SQLDatabase configured with the provided
+                Databricks connection details.
+
+        Raises:
+            ValueError: If 'databricks-sql-connector' is not found, or if both
+                'warehouse_id' and 'cluster_id' are provided, or if neither
+                'warehouse_id' nor 'cluster_id' are provided and it's not executing
+                inside a Databricks notebook.
+        """
+        try:
+            from databricks import sql  # noqa: F401
+        except ImportError:
+            raise ValueError(
+                "databricks-sql-connector package not found, please install with"
+                " `pip install databricks-sql-connector`"
+            )
+        context = None
+        try:
+            from dbruntime.databricks_repl_context import get_context
+
+            context = get_context()
+        except ImportError:
+            pass
+
+        default_host = context.browserHostName if context else None
+        if host is None:
+            host = get_from_env("host", "DATABRICKS_HOST", default_host)
+
+        default_api_token = context.apiToken if context else None
+        if api_token is None:
+            api_token = get_from_env("api_token", "DATABRICKS_TOKEN", default_api_token)
+
+        if warehouse_id is None and cluster_id is None:
+            if context:
+                cluster_id = context.clusterId
+            else:
+                raise ValueError(
+                    "Need to provide either 'warehouse_id' or 'cluster_id'."
+                )
+
+        if warehouse_id and cluster_id:
+            raise ValueError("Can't have both 'warehouse_id' or 'cluster_id'.")
+
+        if warehouse_id:
+            http_path = f"/sql/1.0/warehouses/{warehouse_id}"
+        else:
+            http_path = f"/sql/protocolv1/o/0/{cluster_id}"
+
+        uri = (
+            f"databricks://token:{api_token}@{host}?"
+            f"http_path={http_path}&catalog={catalog}&schema={schema}"
+        )
+        return cls.from_uri(database_uri=uri, engine_args=engine_args, **kwargs)
+
+    @classmethod
+    def from_cnosdb(
+            cls,
+            url: str = "127.0.0.1:8902",
+            user: str = "root",
+            password: str = "",
+            tenant: str = "cnosdb",
+            database: str = "public",
+    ) -> SQLDatabase:
+        """
+        Class method to create an SQLDatabase instance from a CnosDB connection.
+        This method requires the 'cnos-connector' package. If not installed, it
+        can be added using `pip install cnos-connector`.
+
+        Args:
+            url (str): The HTTP connection host name and port number of the CnosDB
+                service, excluding "http://" or "https://", with a default value
+                of "127.0.0.1:8902".
+            user (str): The username used to connect to the CnosDB service, with a
+                default value of "root".
+            password (str): The password of the user connecting to the CnosDB service,
+                with a default value of "".
+            tenant (str): The name of the tenant used to connect to the CnosDB service,
+                with a default value of "cnosdb".
+            database (str): The name of the database in the CnosDB tenant.
+
+        Returns:
+            SQLDatabase: An instance of SQLDatabase configured with the provided
+            CnosDB connection details.
+        """
+        try:
+            from cnosdb_connector import make_cnosdb_langchain_uri
+
+            uri = make_cnosdb_langchain_uri(url, user, password, tenant, database)
+            return cls.from_uri(database_uri=uri)
+        except ImportError:
+            raise ValueError(
+                "cnos-connector package not found, please install with"
+                " `pip install cnos-connector`"
+            )
+
+    @property
+    def dialect(self) -> str:
+        """Return string representation of dialect to use."""
+        return self._engine.dialect.name
+
+    def get_usable_table_names(self) -> Iterable[str]:
+        """Get names of tables available."""
+        if self._include_tables:
+            return sorted(self._include_tables)
+        return sorted(self._all_tables - self._ignore_tables)
+
+    def get_table_names(self) -> Iterable[str]:
+        """Get names of tables available."""
+        warnings.warn(
+            "This method is deprecated - please use `get_usable_table_names`."
+        )
+        return self.get_usable_table_names()
+
+    @property
+    def table_info(self) -> str:
+        """Information about all tables in the database."""
+        return self.get_table_info()
+
+    def get_table_info(self, table_names: Optional[List[str]] = None) -> str:
+        """Get information about specified tables.
+
+        Follows best practices as specified in: Rajkumar et al, 2022
+        (https://arxiv.org/abs/2204.00498)
+
+        If `sample_rows_in_table_info`, the specified number of sample rows will be
+        appended to each table description. This can increase performance as
+        demonstrated in the paper.
+        """
+        all_table_names = self.get_usable_table_names()
+        if table_names is not None:
+            missing_tables = set(table_names).difference(all_table_names)
+            if missing_tables:
+                raise ValueError(f"table_names {missing_tables} not found in database")
+            all_table_names = table_names
+
+        meta_tables = [
+            tbl
+            for tbl in self._metadata.sorted_tables
+            if tbl.name in set(all_table_names)
+               and not (self.dialect == "sqlite" and tbl.name.startswith("sqlite_"))
+        ]
+
+        tables = []
+        for table in meta_tables:
+            if self._custom_table_info and table.name in self._custom_table_info:
+                tables.append(self._custom_table_info[table.name])
+                continue
+
+            # Ignore JSON datatyped columns
+            for k, v in table.columns.items():
+                if type(v.type) is NullType:
+                    table._columns.remove(v)
+
+            # add create table command
+            create_table = str(CreateTable(table).compile(self._engine))
+            table_info = f"{create_table.rstrip()}"
+            has_extra_info = (
+                    self._indexes_in_table_info or self._sample_rows_in_table_info
+            )
+            if has_extra_info:
+                table_info += "\n\n/*"
+            if self._indexes_in_table_info:
+                table_info += f"\n{self._get_table_indexes(table)}\n"
+            if self._sample_rows_in_table_info:
+                table_info += f"\n{self._get_sample_rows(table)}\n"
+            if has_extra_info:
+                table_info += "*/"
+            tables.append(table_info)
+        tables.sort()
+        final_str = "\n\n".join(tables)
+        return final_str
+
+    def _get_table_indexes(self, table: Table) -> str:
+        indexes = self._inspector.get_indexes(table.name)
+        indexes_formatted = "\n".join(map(_format_index, indexes))
+        return f"Table Indexes:\n{indexes_formatted}"
+
+    def _get_sample_rows(self, table: Table) -> str:
+        # build the select command
+        command = select(table).limit(self._sample_rows_in_table_info)
+
+        # save the columns in string format
+        columns_str = "\t".join([col.name for col in table.columns])
+
+        try:
+            # get the sample rows
+            with self._engine.connect() as connection:
+                sample_rows_result = connection.execute(command)  # type: ignore
+                # shorten values in the sample rows
+                sample_rows = list(
+                    map(lambda ls: [str(i)[:100] for i in ls], sample_rows_result)
+                )
+
+            # save the sample rows in string format
+            sample_rows_str = "\n".join(["\t".join(row) for row in sample_rows])
+
+        # in some dialects when there are no rows in the table a
+        # 'ProgrammingError' is returned
+        except ProgrammingError:
+            sample_rows_str = ""
+
+        return (
+            f"{self._sample_rows_in_table_info} rows from {table.name} table:\n"
+            f"{columns_str}\n"
+            f"{sample_rows_str}"
+        )
+
+    def _execute(
+            self,
+            command: str,
+            fetch: Union[Literal["all"], Literal["one"]] = "all",
+    ) -> Sequence[Dict[str, Any]]:
+        """
+        Executes SQL command through underlying engine.
+
+        If the statement returns no rows, an empty list is returned.
+        """
+        with self._engine.begin() as connection:
+            if self._schema is not None:
+                if self.dialect == "snowflake":
+                    connection.exec_driver_sql(
+                        "ALTER SESSION SET search_path = %s", (self._schema,)
+                    )
+                elif self.dialect == "bigquery":
+                    connection.exec_driver_sql("SET @@dataset_id=?", (self._schema,))
+                elif self.dialect == "mssql":
+                    pass
+                elif self.dialect == "trino":
+                    connection.exec_driver_sql("USE ?", (self._schema,))
+                elif self.dialect == "duckdb":
+                    # Unclear which parameterized argument syntax duckdb supports.
+                    # The docs for the duckdb client say they support multiple,
+                    # but `duckdb_engine` seemed to struggle with all of them:
+                    # https://github.com/Mause/duckdb_engine/issues/796
+                    connection.exec_driver_sql(f"SET search_path TO {self._schema}")
+                elif self.dialect == "oracle":
+                    connection.exec_driver_sql(
+                        f"ALTER SESSION SET CURRENT_SCHEMA = {self._schema}"
+                    )
+                else:  # postgresql and other compatible dialects
+                    connection.exec_driver_sql("SET search_path TO %s", (self._schema,))
+            cursor = connection.execute(text(command))
+            if cursor.returns_rows:
+                if fetch == "all":
+                    result = [x._asdict() for x in cursor.fetchall()]
+                elif fetch == "one":
+                    first_result = cursor.fetchone()
+                    result = [] if first_result is None else [first_result._asdict()]
+                else:
+                    raise ValueError("Fetch parameter must be either 'one' or 'all'")
+                return result
+        return []
+
+    def run(
+            self,
+            command: str,
+            fetch: Union[Literal["all"], Literal["one"]] = "all",
+    ) -> str:
+        """Execute a SQL command and return a string representing the results.
+
+        If the statement returns rows, a string of the results is returned.
+        If the statement returns no rows, an empty string is returned.
+        """
+        result = self._execute(command, fetch)
+        # Convert columns values to string to avoid issues with sqlalchemy
+        # truncating text
+        res = [
+            tuple(truncate_word(c, length=self._max_string_length) for c in r.values())
+            for r in result
+        ]
+        if not res:
+            return ""
+        else:
+            return str(res)
+
+    def get_table_info_no_throw(self, table_names: Optional[List[str]] = None) -> str:
+        """Get information about specified tables.
+
+        Follows best practices as specified in: Rajkumar et al, 2022
+        (https://arxiv.org/abs/2204.00498)
+
+        If `sample_rows_in_table_info`, the specified number of sample rows will be
+        appended to each table description. This can increase performance as
+        demonstrated in the paper.
+        """
+        try:
+            return self.get_table_info(table_names)
+        except ValueError as e:
+            """Format the error message"""
+            return f"Error: {e}"
+
+    def run_no_throw(
+            self,
+            command: str,
+            fetch: Union[Literal["all"], Literal["one"]] = "all",
+    ) -> str:
+        """Execute a SQL command and return a string representing the results.
+
+        If the statement returns rows, a string of the results is returned.
+        If the statement returns no rows, an empty string is returned.
+
+        If the statement throws an error, the error message is returned.
+        """
+        try:
+            return self.run(command, fetch)
+        except SQLAlchemyError as e:
+            """Format the error message"""
+            return f"Error: {e}"

src/aios/environment/sql_database_function.py

@@ -0,0 +1,112 @@
+from datetime import timedelta, datetime
+from typing import Dict
+
+from cachetools import TLRUCache, cached
+
+from ..agent.ai_function import AIFunction
+from .sql_database import SQLDatabase, get_from_env
+
+
+def _my_ttu(_key, _value, now):
+    return now + timedelta(seconds=600)
+
+
+database_cache = TLRUCache(ttu=_my_ttu, maxsize=10000, timer=datetime.now)
+
+
+@cached(cache=database_cache)
+def get_database(uri: str) -> SQLDatabase:
+    return SQLDatabase.from_uri(uri)
+
+
+class GetTableInfosFunction(AIFunction):
+    def __init__(self):
+        super().__init__()
+        self.name = "get_table_infos"
+        self.description = "Get table informations in the database"
+
+    def get_name(self) -> str:
+        return self.name
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "database_url": {"type": "string", "description": "Database URL,Can be set to None"},
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        database_url: str = kwargs.get("database_url")
+        if (database_url is None
+                or database_url.strip() == ""
+                or database_url.strip().lower() == "none"
+                or database_url.strip().lower() == "null"):
+            database_url = get_from_env(key="database url", env_key="DATABASE_URL")
+            if database_url is None:
+                return "error: database_url is None"
+        database = get_database(database_url)
+        tables = database.get_usable_table_names()
+        table_infos = database.get_table_info(tables)
+        return table_infos
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+
+class ExecuteSqlFunction(AIFunction):
+    def __init__(self):
+        super().__init__()
+        self.name = "execute_sql"
+        self.description = """
+        Input to this function is a detailed and correct SQL query, output is a result from the database.
+        If the query is not correct, an error message will be returned.
+        If an error is returned, rewrite the query, check the query, and try again.
+        """
+
+    def get_name(self) -> str:
+        return self.name
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "database_url": {"type": "string", "description": "Database URL,Can be set to None"},
+                "sql": {"type": "string", "description": "SQL to execute"}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        database_url = kwargs.get("database_url")
+        if (database_url is None
+                or database_url.strip() == ""
+                or database_url.strip().lower() == "none"
+                or database_url.strip().lower() == "null"):
+            database_url = get_from_env(key="database url", env_key="DATABASE_URL")
+            if database_url is None:
+                return "error: database_url is None"
+        sql = kwargs.get("sql")
+
+        database = get_database(database_url)
+        return database.run_no_throw(sql)
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False

src/aios/environment/text_to_speech_function.py

@@ -0,0 +1,79 @@
+import io
+import logging
+import os
+import random
+from pathlib import Path
+from typing import Dict
+
+from ..agent.ai_function import AIFunction
+from ..frame.compute_kernel import ComputeKernel
+from ..storage.storage import AIStorage
+
+
+from pydub import AudioSegment
+
+logger = logging.getLogger(__name__)
+
+
+class TextToSpeechFunction(AIFunction):
+    def __init__(self):
+        self.func_id = "text_to_speech"
+        self.description = "根据输入的文本生成音频文件，成功时会返回音频文件路径"
+        self.speech_path = os.path.join(AIStorage.get_instance().get_myai_dir(), "tts")
+        Path(self.speech_path).mkdir(exist_ok=True)
+
+    def get_name(self) -> str:
+        return self.func_id
+
+    def get_description(self) -> str:
+        return self.description
+
+    def get_parameters(self) -> Dict:
+        return {
+            "type": "object",
+            "properties": {
+                "language": {"type": "string", "description": "演播语言", "enum": ["zh", "en"]},
+                "model": {"type": "string", "description": "演播模型", "enum": ["tts-1", "tts-1-hd"]},
+                "text": {"type": "string", "description": "文本内容"}
+            }
+        }
+
+    async def execute(self, **kwargs) -> str:
+        logger.info(f"execute text_to_speech function: {kwargs}")
+
+        language = kwargs.get("language")
+        if language is None:
+            language = "en"
+        model = kwargs.get("model")
+        text = kwargs.get("text")
+
+        i = 0
+        while i < 3:
+            try:
+                data = await ComputeKernel.get_instance().do_text_to_speech(text, language, None, None, None, None,
+                                                                            model_name=model)
+                if data is not None:
+                    audio = AudioSegment.from_mp3(io.BytesIO(data))
+                break
+            except Exception as e:
+                logger.error(f"do_text_to_speech failed: {e}")
+                i += 1
+                continue
+
+        if audio is not None:
+            path = os.path.join(self.speech_path, "{}.mp3".format(''.join(random.sample('zyxwvutsrqponmlkjihgfedcba', 10))))
+            audio.export(path, format="mp3")
+            return "exec text_to_speech OK，speech file store at ```{}```".format(path)
+        else:
+            return "exec text_to_speech failed"
+
+    def is_local(self) -> bool:
+        return True
+
+    def is_in_zone(self) -> bool:
+        return True
+
+    def is_ready_only(self) -> bool:
+        return False
+
+

src/aios/environment/workflow_env.py

@@ -0,0 +1,421 @@
+
+from datetime import datetime
+import asyncio
+import json
+import sqlite3 # Because sqlite3 IO operation is small, so we can use sqlite3 directly.(so we don't need to use async sqlite3 now)
+from sqlite3 import Error
+import threading
+import logging
+from typing import Optional
+import aiosqlite
+
+from ..proto.compute_task import *
+from ..agent.ai_function import SimpleAIFunction
+from ..frame.compute_kernel import ComputeKernel
+from ..frame.contact_manager import ContactManager,Contact,FamilyMember
+from ..storage.storage import AIStorage
+
+from .environment import Environment,EnvironmentEvent
+from .script_to_speech_function import ScriptToSpeechFunction
+from .image_2_text_function import Image2TextFunction
+
+logger = logging.getLogger(__name__)
+
+class CalenderEvent(EnvironmentEvent):
+    def __init__(self,data) -> None:
+        super().__init__()
+        self.event_name = "timer"
+        self.data = data
+
+    def display(self) -> str:
+        return f"#event timer:{self.data}"
+
+# AI Calender GOAL: Let user use "create notify after 2 days" to create a timer event
+class CalenderEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+        self.db_file = AIStorage.get_instance().get_myai_dir() / "calender.db"
+        self.is_run = False
+
+        self.add_ai_function(SimpleAIFunction("get_time",
+                                        "get current time",
+                                        self._get_now))
+        get_param = {
+            "start_time": "start time (UTC) of event",
+            "end_time": "end time (UTC) of event"
+        }
+        self.add_ai_function(SimpleAIFunction("get_events",
+                                              "get events in calender by time range",
+                                              self._get_events_by_time_range,get_param))
+
+        add_param = {
+            "title": "title of event",
+            "start_time": "start time (UTC) of event",
+            "end_time": "end time (UTC) of event",
+            "participants": "participants of event",
+            "location": "location of event",
+            "details": "details of event"
+        }
+        self.add_ai_function(SimpleAIFunction("add_event",
+                                        "add event to calender",
+                                        self._add_event,add_param))
+
+        delete_param = {
+            "event_id": "id of event"
+        }
+        self.add_ai_function(SimpleAIFunction("delete_event",
+                                        "delete event from calender",
+                                        self._delete_event,delete_param))
+
+        update_param = {
+            "event_id": "id of event",
+            "new_title": "new title of event",
+            "new_participants": "new participants of event",
+            "new_location": "new location of event",
+            "new_details": "new details of event",
+            "start_time": "new start time (UTC) of event",
+            "end_time": "new end time (UTC) of event"
+        }
+        self.add_ai_function(SimpleAIFunction("update_event",
+                                        "update event in calender",
+                                        self._update_event,update_param))
+
+
+        #maybe this function should be in other env?
+        paint_param = {
+            "prompt": "A description of the content of the painting",
+            "model_name": "Which model to use to draw the picture, can be None"
+        }
+        self.add_ai_function(SimpleAIFunction("paint",
+                                        "Draw a picture according to the description",
+                                        self._paint,paint_param))
+
+        self.add_ai_function(SimpleAIFunction("get_contact",
+                                        "get contact info",
+                                        self._get_contact,{"name":"name of contact"}))
+
+        self.add_ai_function(SimpleAIFunction("set_contact",
+                                        "set contact info",
+                                        self._set_contact,{"name":"name of contact","contact_info":"A json to descrpit contact"}))
+
+
+
+
+        #self.add_ai_function(SimpleAIFunction("user_confirm",
+        #                                      "user confirm",
+        #                                      self._user_confirm))
+
+    async def init_db(self):
+        async with aiosqlite.connect(self.db_file) as db:
+            await db.execute("""
+                CREATE TABLE IF NOT EXISTS events (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    title TEXT,
+                    start_time DATETIME,
+                    end_time DATETIME,
+                    participants TEXT,
+                    location TEXT,
+                    details TEXT
+                );
+            """)
+            await db.commit()
+
+    async def _add_event(self,title, start_time, end_time, participants=None, location=None, details=None):
+        async with aiosqlite.connect(self.db_file) as db:
+            await db.execute("""
+                INSERT INTO events (title, start_time, end_time, participants, location, details)
+                VALUES (?, ?, ?, ?, ?, ?);
+            """, (title, start_time, end_time, participants, location, details))
+            await db.commit()
+            return f"execute add_event OK,event '{title}' already add to calender!"
+
+    async def _search_events(self,query):
+        async with aiosqlite.connect(self.db_file) as db:
+            cursor = await db.execute("""
+                SELECT id,title, start_time, end_time, participants, location, details FROM events
+                WHERE title LIKE ? OR participants LIKE ? OR location LIKE ? OR details LIKE ?;
+            """, (f"%{query}%", f"%{query}%", f"%{query}%", f"%{query}%"))
+            rows = await cursor.fetchall()
+
+            result = {}
+            for row in rows:
+                _event = {}
+                _event["title"] = row[1]
+                _event["start_time"] = row[2]
+                _event["end_time"] = row[3]
+                _event["participants"] = row[4]
+                _event["location"] = row[5]
+                _event["details"] = row[6]
+                result[row[0]] = _event
+            return json.dumps(result, indent=4, sort_keys=True)
+
+    async def _get_events_by_time_range(self,start_time, end_time):
+        async with aiosqlite.connect(self.db_file) as db:
+            cursor = await db.execute("""
+                SELECT id,title, start_time, end_time, participants, location, details FROM events
+                WHERE start_time >= ? AND end_time <= ?;
+            """, (start_time, end_time))
+            rows = await cursor.fetchall()
+
+            result = {}
+            have_result = False
+            for row in rows:
+                have_result = True
+                _event = {}
+                _event["title"] = row[1]
+                _event["start_time"] = row[2]
+                _event["end_time"] = row[3]
+                _event["participants"] = row[4]
+                _event["location"] = row[5]
+                _event["details"] = row[6]
+                result[row[0]] = _event
+
+            if not have_result:
+                return "No event."
+
+            return json.dumps(result, indent=4, sort_keys=True)
+
+    async def _update_event(self,event_id, new_title=None, new_participants=None, new_location=None, new_details=None ,start_time=None, end_time=None):
+        fields_to_update = []
+        values = []
+
+        if new_title is not None:
+            fields_to_update.append("title = ?")
+            values.append(new_title)
+
+        if new_participants is not None:
+            fields_to_update.append("participants = ?")
+            values.append(new_participants)
+
+        if new_location is not None:
+            fields_to_update.append("location = ?")
+            values.append(new_location)
+
+        if new_details is not None:
+            fields_to_update.append("details = ?")
+            values.append(new_details)
+
+        if start_time is not None:
+            fields_to_update.append("start_time = ?")
+            values.append(start_time)
+
+        if end_time is not None:
+            fields_to_update.append("end_time = ?")
+            values.append(end_time)
+
+        if not fields_to_update:
+            return "No fields to update."
+
+        sql_update_query = f"""
+            UPDATE events
+            SET {', '.join(fields_to_update)}
+            WHERE id = ?;
+        """
+
+        values.append(event_id)
+
+        async with aiosqlite.connect(self.db_file) as db:
+            await db.execute(sql_update_query, values)
+            await db.commit()
+            return "update ok"
+
+    async def _delete_event(self,event_id):
+        async with aiosqlite.connect(self.db_file) as db:
+            await db.execute("""
+                DELETE FROM events
+                WHERE id = ?;
+            """, (event_id,))
+            await db.commit()
+            return "Delete event ok"
+
+    def _do_get_value(self,key:str) -> Optional[str]:
+        return None
+
+    async def _get_contact(self,name:str) -> str:
+        cm = ContactManager.get_instance()
+        contact : Contact = cm.find_contact_by_name(name)
+        if contact:
+            s = json.dumps(contact.to_dict())
+            return f"Execute get_contact OK , contact {name} is {s}"
+        else:
+            return f"Execute get_contact OK , contact {name} not found!"
+
+    async def _set_contact(self,name:str,contact_info:str) -> str:
+        cm = ContactManager.get_instance()
+        contact = cm.find_contact_by_name(name)
+        contact_info = json.loads(contact_info)
+        if contact is None:
+            contact = Contact(name)
+            contact.email = contact_info.get("email")
+            contact.telegram = contact_info.get("telegram")
+            contact.notes = contact_info.get("notes")
+            contact.added_by = self.env_id
+
+            cm.add_contact(name,contact)
+
+            return f"Execute set_contact OK , new contact {name} added!"
+        else:
+            if contact_info.get("email") is not None:
+                contact.email = contact_info.get("email")
+            if contact_info.get("telegram") is not None:
+                contact.telegram = contact_info.get("telegram")
+            if contact_info.get("notes") is not None:
+                contact.notes = contact_info.get("notes")
+
+            contact.added_by = self.env_id
+            cm.set_contact(name,contact)
+
+            return f"Execute set_contact OK , contact {name} updated!"
+
+    async def start(self) -> None:
+        if self.is_run:
+            return
+        self.is_run = True
+        await self.init_db()
+
+        self.register_get_handler("now",self.get_now)
+        async def timer_loop():
+            while True:
+                if self.is_run == False:
+                    break
+
+                await asyncio.sleep(1.0)
+                now = datetime.now()
+                formatted_time = now.strftime('%Y-%m-%d %H:%M:%S')
+                env_event:CalenderEvent = CalenderEvent(formatted_time)
+                await self.fire_event("timer",env_event)
+
+            return
+
+        asyncio.create_task(timer_loop())
+
+    def stop(self):
+        self.is_run = False
+
+    def get_now(self)->str:
+        now = datetime.now()
+        formatted_time = now.strftime('%Y-%m-%d %H:%M:%S')
+        return formatted_time
+
+    async def _get_now(self) -> str:
+        now = datetime.now()
+        formatted_time = now.strftime('%Y-%m-%d %H:%M:%S')
+        return formatted_time
+
+
+    async def _paint(self, prompt, model_name = None) -> str:
+        result = await ComputeKernel.get_instance().do_text_2_image(prompt, model_name)
+        if result.result_code == ComputeTaskResultCode.ERROR:
+            return f"exec paint failed. err:{result.error_str}"
+        else:
+            return f'exec paint OK, saved as a local file, path is: {result.result["file"]}'
+
+
+class PaintEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+        self.is_run = False
+
+        paint_param = {
+            "prompt": "Keywords of the content of the painting",
+            "model_name": "Which model to use to draw the picture, can be None",
+            "negative_prompt": "Keywords that describe what is not to be drawn, can be None"
+        }
+        self.add_ai_function(SimpleAIFunction("paint",
+                                        "Draw a picture according to the keywords",
+                                        self._paint,paint_param))
+
+    def _do_get_value(self,key:str) -> Optional[str]:
+        return None
+
+
+    async def _paint(self, prompt, model_name = None, negative_prompt = None) -> str:
+        err, result = await ComputeKernel.get_instance().do_text_2_image(prompt, model_name, negative_prompt)
+        if err is not None:
+            return f"exec paint failed. err:{err}"
+        else:
+            return f'exec paint OK, saved as a local file, path is: {result.result["file"]}'
+
+
+# Default Workflow Environment(Context)
+class WorkflowEnvironment(Environment):
+    def __init__(self, env_id: str,db_file:str) -> None:
+        super().__init__(env_id)
+        self.db_file = db_file
+        self.local = threading.local()
+        self.table_name = "WorkflowEnv_" + env_id
+        self.add_ai_function(ScriptToSpeechFunction())
+        self.add_ai_function(Image2TextFunction())
+
+
+    def _get_conn(self):
+        """ get db connection """
+        if not hasattr(self.local, 'conn'):
+            self.local.conn = self._create_connection()
+        return self.local.conn
+
+    def _create_connection(self):
+        """ create a database connection to a SQLite database """
+        conn = None
+        try:
+            conn = sqlite3.connect(self.db_file)
+        except Error as e:
+            logging.error("Error occurred while connecting to database: %s", e)
+            return None
+
+        if conn:
+            self._create_table(conn)
+
+        return conn
+
+    def close(self):
+        if not hasattr(self.local, 'conn'):
+            return
+        self.local.conn.close()
+
+    def _create_table(self, conn):
+        """ create table """
+        try:
+            # create sessions table
+            conn.execute(f"""
+                CREATE TABLE IF NOT EXISTS """ + self.table_name + """ (
+                    EnvKey TEXT PRIMARY KEY,
+                    EnvValue TEXT,
+                    UpdateTime TEXT
+                );
+            """)
+            conn.commit()
+        except Error as e:
+            logging.error("Error occurred while creating tables: %s", e)
+
+    def _do_get_value(self, key: str) -> str | None:
+        try:
+            conn = self._get_conn()
+            c = conn.cursor()
+            c.execute("SELECT EnvValue FROM " + self.table_name +" WHERE EnvKey = ?", (key,))
+            value = c.fetchone()
+            if value is None:
+                return None
+            return value[0]
+        except Error as e:
+            logging.error(f"Error occurred while _do_get_value{key}: {e}")
+            return None
+
+    def set_value(self, key: str, str_value: str, is_storage:bool=True):
+        super().set_value(key,str_value)
+        if is_storage is False:
+            return
+
+        try:
+            conn = self._get_conn()
+            conn.execute("""
+                INSERT OR REPLACE INTO """ + self.table_name+ """ (EnvKey, EnvValue, UpdateTime)
+                VALUES (?, ?, ?) 
+            """, (key, str_value, datetime.now()))
+            conn.commit()
+            return 0  # return 0 if successful
+        except Error as e:
+            logging.error(f"Error occurred while update env{self.env_id}.{key} ,error:{e}")
+
+    def get_functions(self):
+        pass

src/aios/environment/workspace_env.py

@@ -0,0 +1,789 @@
+# this env is designed for workflow owner filesystem, support file/directory operations
+
+import hashlib
+import json
+import subprocess
+import logging
+import tempfile
+import threading
+import traceback
+import time
+import ast
+import sys
+import os
+import re
+import asyncio
+import aiofiles
+from typing import Any,List
+import os
+import chardet
+from markdown import Markdown
+import PyPDF2
+
+from ..proto.agent_msg import *
+from ..agent.agent_base import AgentTodo,AgentPrompt,AgentTodoResult
+from ..agent.ai_function import AIFunction,SimpleAIFunction
+from ..storage.storage import AIStorage,ResourceLocation
+from .simple_kb_db import SimpleKnowledgeDB
+from .environment import Environment,EnvironmentEvent
+
+logger = logging.getLogger(__name__)
+
+class WorkspaceEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+        myai_path = AIStorage.get_instance().get_myai_dir() 
+        self.root_path = f"{myai_path}/workspace/{env_id}"
+        if not os.path.exists(self.root_path):
+            os.makedirs(self.root_path+"/todos")
+
+        self.known_todo = {}
+        self.kb_db = SimpleKnowledgeDB(f"{self.root_path}/kb.db")
+        self.doc_dirs = {}
+        self._scan_thread = None 
+        self._scan_dirthread = None
+        
+
+    def set_root_path(self,path:str):
+        self.root_path = path
+
+    def get_prompt(self) -> AgentMsg:
+        return None
+    
+    def get_role_prompt(self,role_id:str) -> AgentPrompt:
+        return None
+
+    def get_knowledge_base(self,root_dir=None,indent=0) -> str:
+        pass
+
+
+    def get_do_prompt(self,todo:AgentTodo=None)->AgentPrompt:
+        return None
+
+    # result mean: list[op_error_str],have_error
+    async def exec_op_list(self,oplist:List,agent_id:str)->tuple[List[str],bool]:
+        result_str = "op list is none"
+        if oplist is None:
+            return None,False
+        
+        result_str = []
+        have_error = False
+        for op in oplist:
+            if op["op"] == "create":
+                await self.create(op["path"],op["content"])
+            elif op["op"] == "write_file":
+                is_append = op.get("is_append")
+                if is_append is None:
+                    is_append = False
+                error_str = await self.write(op["path"],op["content"],is_append)
+            elif op["op"] == "delete":
+                error_str = await self.delete(op["path"])
+            elif op["op"] == "rename":
+                error_str = await self.rename(op["path"],op["new_name"])
+            elif op["op"] == "mkdir":
+                error_str = await self.mkdir(op["path"])
+            elif op["op"] == "create_todo":
+                todoObj = AgentTodo.from_dict(op["todo"])
+                todoObj.worker = agent_id
+                todoObj.createor = agent_id
+                parent_id = op.get("parent")
+                error_str = await self.create_todo(parent_id,todoObj)
+            elif op["op"] == "update_todo":
+                todo_id = op["id"]
+                new_stat = op["state"]
+                error_str = await self.update_todo(todo_id,new_stat)
+            else:
+                logger.error(f"execute op list failed: unknown op:{op['op']}")
+                error_str = f"execute op list failed: unknown op:{op['op']}"
+            
+            if error_str:
+                have_error = True
+                result_str.append(error_str)
+            else:
+                result_str.append(f"execute success!")  
+    
+        
+        return result_str,have_error
+    
+    # file system operation: list,read,write,delete,move,stat
+    # inner_function
+    async def list(self,path:str,only_dir:bool=False) -> str:
+        directory_path = self.root_path + path
+        items = []
+
+        with await aiofiles.os.scandir(directory_path) as entries:
+            async for entry in entries:
+                is_dir = entry.is_dir()
+                if only_dir and not is_dir:
+                    continue
+                item_type = "directory" if is_dir else "file"
+                items.append({"name": entry.name, "type": item_type})
+
+        return json.dumps(items)
+    
+    # inner_function
+    async def read(self,path:str) -> str:
+        file_path = self.root_path + path
+        cur_encode = "utf-8"
+        async with aiofiles.open(file_path,'rb') as f:
+            cur_encode = chardet.detect(await f.read())['encoding']
+
+        async with aiofiles.open(file_path, mode='r', encoding=cur_encode) as f:
+            content = await f.read(2048)
+        return content
+    
+
+    # operation or inner_function (MOST IMPORTANT FUNCTION)
+    async def write(self,path:str,content:str,is_append:bool=False) -> str:
+        file_path = self.root_path + path
+        try:
+            if is_append:
+                async with aiofiles.open(file_path, mode='a', encoding="utf-8") as f:
+                    await f.write(content)
+            else:
+                if content is None:
+                    # create dir
+                    dir_path = self.root_path + path
+                    os.makedirs(dir_path)
+                    return True
+                else:
+                    file_path = self.root_path + path
+                    os.makedirs(os.path.dirname(file_path),exist_ok=True)
+                    async with aiofiles.open(file_path, mode='w', encoding="utf-8") as f:
+                        await f.write(content)
+                    return True
+        
+        except Exception as e:
+            return str(e)
+        return None
+    
+        
+    # operation or inner_function
+    async def delete(self,path:str) -> str:
+        try:
+            file_path = self.root_path + path
+            os.remove(file_path)
+        except Exception as e:
+            return str(e)
+        
+        return None
+    
+    # operation or inner_function
+    async def move(self,path:str,new_path:str) -> str:
+        try:
+            file_path = self.root_path + path
+            new_path = self.root_path + new_path
+            os.rename(file_path,new_path)
+        except Exception as e:
+            return str(e)
+        
+        return None
+    
+    # inner_function
+    async def stat(self,path:str) -> str:
+        try:
+            file_path = self.root_path + path
+            stat = os.stat(file_path)
+            return json.dumps(stat)
+        except Exception as e:
+            return str(e)
+
+    # operation or inner_function   
+    async def symlink(self,path:str,target:str) -> str:
+        try:
+            #file_path = self.root_path + path
+            target_path = self.root_path + target
+            dir_path = os.path.dirname(target_path)
+            os.makedirs(dir_path,exist_ok=True)
+            os.symlink(path,target_path)
+        except Exception as e:
+            logger.error("symlink failed:%s",e)
+            return str(e)
+        
+        return None
+        
+    # TODO use diff to update large file content
+    async def update_by_diff(self,path:str,diff):
+
+        pass 
+    
+    # doc system （read_only,agent cann't modify doc）
+
+    # inner_function
+    async def list_db(self) -> str:
+        pass
+    # inner_function
+    async def get_db_desc(self,db_name:str) -> str:
+        pass  
+    # inner_function
+    async def query(self,db_name:str,sql:str) -> str:
+        pass
+
+    # search (web)
+    # inner_function
+    async def google_search(self,keyword:str,opt=None) -> str:
+        pass
+
+    # inner_function
+    async def local_search(self,keyword:str,root_path=None ,opt=None) -> str:
+        pass
+    
+    # inner_function, might be return a image is better
+    async def web_get(self,url:str) -> str:
+        pass
+    
+    # inner_function
+    async def blockchain_get(self,chainid:str,query:dict) -> str:
+        pass
+
+    # code interpreter
+    # inner_function or operation
+    async def eval_code(self,pycode:str) -> str:
+        pass
+    
+    # operation or inner_function
+    async def improve_code(self,path:str):
+        pass
+    
+    # operation or inner_function
+    async def run(self,file_path:str)->str:
+        pass
+
+    # operation or inner_function
+    async def pub_service(self,project_path:str):
+        pass
+
+    # operation or inner_function
+    async def exec_tx(self,chain_id:str,tx:dict) -> str:
+        pass
+    
+    # social ability
+    # operation or inner_function
+    async def post_message(self,target:str,msg:AgentMsg,wait_time) -> AgentMsg:
+        pass
+    
+    # operation or inner_function
+    async def add_contact(self,name:str,contact_info) -> str:
+        pass
+
+    # inner_function , include contact realtime info
+    async def get_contact(self,name_list:List[str],opt:dict) -> List:
+        pass
+
+
+    # Task/todo system , create,update,delete,query
+    async def get_todo_tree(self,path:str = None,deep:int = 4):
+        if path:
+            directory_path = self.root_path + "/todos/" + path
+        else:
+            directory_path = self.root_path + "/todos"
+
+        
+        str_result:str = "/todos\n"
+        todo_count:int = 0 
+
+        async def scan_dir(directory_path:str,deep:int):
+            nonlocal str_result
+            nonlocal todo_count
+            if deep <= 0:
+                return
+            
+            if os.path.exists(directory_path) is False:
+                return 
+            
+            for entry in os.scandir(directory_path):
+                is_dir = entry.is_dir()
+                if not is_dir:
+                    continue
+
+                if entry.name.startswith("."):
+                    continue
+                
+                todo_count = todo_count +  1
+                str_result = str_result + f"{'  '*(4-deep)}{entry.name}\n"
+                await scan_dir(entry.path,deep-1)
+
+        await scan_dir(directory_path,deep)
+        return str_result,todo_count
+
+    async def get_todo_list(self,agent_id:str,path:str = None)->List[AgentTodo]:
+        logger.info("get_todo_list:%s,%s",agent_id,path)
+        if path:
+            directory_path = self.root_path + "/todos/" + path
+        else:
+            directory_path = self.root_path + "/todos"
+
+        result_list:List[AgentTodo] = []
+
+        async def scan_dir(directory_path:str,deep:int,parent:AgentTodo=None):
+            nonlocal result_list
+            if os.path.exists(directory_path) is False:
+                return 
+
+            for entry in os.scandir(directory_path):
+                is_dir = entry.is_dir()
+                if not is_dir:
+                    continue
+
+                if entry.name.startswith("."):
+                    continue
+                
+                todo = await self.get_todo_by_fullpath(entry.path)
+                if todo:
+                    if todo.worker:
+                        if todo.worker != agent_id:
+                            continue
+                        
+                    if parent:
+                        parent.sub_todos[todo.todo_id] = todo
+                    
+                    result_list.append(todo)
+                    todo.rank = int(todo.create_time)>>deep
+                    await scan_dir(entry.path,deep + 1,todo)
+            
+            return 
+
+        await scan_dir(directory_path,0) 
+        #sort by rank
+        result_list.sort(key=lambda x:(x.rank,x.title))
+        logger.info("get_todo_list return,todolist.length() is %d",len(result_list))
+        return result_list
+
+    async def get_todo_by_fullpath(self,path:str) -> AgentTodo:
+        logger.info("get_todo_by_fullpath:%s",path)
+
+        detail_path = path + "/detail"
+        try:
+            async with aiofiles.open(detail_path, mode='r', encoding="utf-8") as f:
+                content = await f.read(4096)
+                logger.debug("get_todo_by_fullpath:%s,content:%s",path,content)
+                todo_dict = json.loads(content)
+                result_todo =  AgentTodo.from_dict(todo_dict)
+                if result_todo:
+                    relative_path = os.path.relpath(path, self.root_path + "/todos/")
+                    if not relative_path.startswith('/'):
+                        relative_path = '/' + relative_path
+                    result_todo.todo_path = relative_path
+                    self.known_todo[result_todo.todo_id] = result_todo
+                else:
+                    logger.error("get_todo_by_path:%s,parse failed!",path)
+                
+                return result_todo
+        except Exception as e:
+            logger.error("get_todo_by_path:%s,failed:%s",path,e)
+            return None
+        
+    async def get_todo(self,id:str) -> AgentTodo:
+        return self.known_todo.get(id)
+
+    async def create_todo(self,parent_id:str,todo:AgentTodo) -> str:
+        try:
+            if parent_id:
+                if parent_id not in self.known_todo:
+                    logger.error("create_todo failed: parent_id not found!")
+                    return False
+                
+                parent_path = self.known_todo.get(parent_id).todo_path
+                todo_path = f"{parent_path}/{todo.title}"
+            else:
+                todo_path = todo.title
+
+            dir_path = f"{self.root_path}/todos/{todo_path}"
+    
+            os.makedirs(dir_path)
+            detail_path = f"{dir_path}/detail"
+            if todo.todo_path is None:
+                todo.todo_path = todo_path
+            logger.info("create_todo %s",detail_path)
+            async with aiofiles.open(detail_path, mode='w', encoding="utf-8") as f:
+                await f.write(json.dumps(todo.to_dict()))
+                self.known_todo[todo.todo_id] = todo
+        except Exception as e:
+            logger.error("create_todo failed:%s",e)
+            return str(e)
+        
+        return None
+
+    async def update_todo(self,todo_id:str,new_stat:str)->str:
+        try:
+            todo : AgentTodo = self.known_todo.get(todo_id)
+            if todo:
+                todo.state = new_stat
+                detail_path =  f"{self.root_path}/todos/{todo.todo_path}/detail"
+                async with aiofiles.open(detail_path, mode='w', encoding="utf-8") as f:
+                    await f.write(json.dumps(todo.to_dict()))
+                    return None
+            else:
+                return "todo not found."
+        except Exception as e:
+            return str(e)
+    
+    async def append_worklog(self,todo:AgentTodo,result:AgentTodoResult):
+        worklog = f"{self.root_path}/todos/{todo.todo_path}/.worklog"
+
+        async with aiofiles.open(worklog, mode='w+', encoding="utf-8") as f:
+            content = await f.read()
+            if len(content) > 0:
+                json_obj = json.loads(content)
+            else:
+                json_obj = {}
+            logs = json_obj.get("logs")
+            if logs is None:
+                logs = []
+            logs.append(result.to_dict())
+            json_obj["logs"] = logs
+            await f.write(json.dumps(json_obj))
+
+    async def set_wakeup_timer(self,todo_id:str,timestamp:int) -> str:
+        pass
+
+    # knowledge base system
+    def get_knowledge_base_ai_functions(self):
+        all_inner_function = []
+
+        all_inner_function.append(SimpleAIFunction("get_knowledge_catalog","get knowledge catalog in tree format",
+                                                                self.get_knowledege_catalog,
+                                                                {"path":f"catalog path,none is /","depth":"max depth of catalog tree,default is 4"}))
+        all_inner_function.append(SimpleAIFunction("get_knowledge","get knowledge metadata",
+                                                                self.get_knowledge,
+                                                                {"path":f"knowledge path"}))
+        all_inner_function.append(SimpleAIFunction("load_knowledge_content","load knowledge content",
+                                                                 self.load_knowledge_content,
+                                                                  {"path":f"knowledge path","pos":"start position of content","length":"length of content"}))
+        result_func = []
+        result_len = 0
+        for inner_func in all_inner_function:
+            func_name = inner_func.get_name()
+
+            this_func = {}
+            this_func["name"] = func_name
+            this_func["description"] = inner_func.get_description()
+            this_func["parameters"] = inner_func.get_parameters()
+            result_len += len(json.dumps(this_func)) / 4
+            result_func.append(this_func)
+
+        return result_func,result_len                                                        
+
+    async def get_knowledege_catalog(self,path:str=None,only_dir =True,max_depth:int=5)->str:
+        if path:
+            full_path = f"{self.root_path}/knowledge/{path}"
+        else:
+            full_path = f"{self.root_path}/knowledge"
+        
+        catlogs,file_count = await self.get_directory_structure(full_path,max_depth,only_dir)
+        return catlogs
+ 
+    async def get_directory_structure(self,root_dir, max_depth:int=4, only_dir=True, indent=1):
+        file_count = 0
+        structure_str = ''
+        if os.path.isdir(root_dir):
+            sub_files = []
+            with os.scandir(root_dir) as it:
+                for entry in it:
+                    if entry.is_dir():
+                        sub_structure, sub_count = await self.get_directory_structure(entry.path, max_depth, only_dir, indent + 1)
+                        if sub_structure:
+                            structure_str += sub_structure
+                        file_count += sub_count
+                    else:
+                        file_count += 1
+                        sub_files.append(entry.name)
+
+                if only_dir is False:
+                    for file_name in sub_files:
+                        structure_str = structure_str + '  ' * (indent+1) + file_name + '\n' 
+
+            dir_name = os.path.basename(root_dir)
+            dir_info = f"{dir_name} <count: {file_count}>"
+        
+
+            structure_str = '  ' * indent + dir_info + '\n' + structure_str
+
+        if indent - 1 >= max_depth:
+            return None, file_count
+        else:
+            return structure_str, file_count
+
+    # inner_function    
+    async def get_knowledge(self,path:str) -> str:
+        full_path = f"{self.root_path}/knowledge/{path}"
+        if os.islink(full_path):
+            org_path = os.readlink(full_path)
+            hash = self.kb_db.get_hash_by_doc_path(org_path)
+            if hash:
+                return self.kb_db.get_knowledge(org_path)
+        
+        return "not found"
+
+    async def load_knowledge_content(self,path:str,pos:int=0,length:int=None) -> str:
+        if path.endswith("pdf"):
+            logger.info("load_knowledge_content:pdf")
+            dir_path = os.path.dirname(path)
+            base_name = os.path.basename(path)
+            text_content_path = f"{dir_path}/.{base_name}.txt"
+            if os.path.exists(text_content_path) is False:
+                return None
+            async with aiofiles.open(path, mode='r', encoding=cur_encode) as f:
+                await f.seek(pos)
+                content = await f.read(length)
+                return content
+        else:
+            async with aiofiles.open(path,'rb') as f:
+                cur_encode = chardet.detect(await f.read())['encoding']
+
+            async with aiofiles.open(path, mode='r', encoding=cur_encode) as f:
+                await f.seek(pos)
+                content = await f.read(length)
+                return content
+
+        return "load content failed."
+
+    def _add_document_dir(self,path:str):
+        self.doc_dirs[path] = 0
+
+    def _start_scan_document(self):
+        if self._scan_thread is None:
+            self._scan_thread = threading.Thread(target=self._scan_document)
+            self._scan_thread.start()
+        if self._scan_dirthread is None:
+            self._scan_dirthread = threading.Thread(target=self._scan_dir)
+            self._scan_dirthread.start()
+
+    def _parse_pdf_bookmarks(self,bookmarks, parent:list):
+
+        for item in bookmarks:
+            if isinstance(item,list):
+                self._parse_pdf_bookmarks(item,parent)
+            else:
+                if item.title:
+                    new_item = {}
+                    new_item["page"] = item.page.idnum
+                    new_item["title"] = item.title 
+                    my_childs = []
+                    if item.childs:
+                        if len(item.childs) > 0:
+                            self._parse_pdf_bookmarks(item.childs, my_childs)
+                            new_item["childs"] = my_childs
+                    parent.append(new_item)
+                else:
+                    logger.warning("parse pdf bookmarks failed: item.title is None!")
+
+        return
+
+    def _parse_pdf(self,doc_path:str):
+        metadata = {}
+        with open(doc_path, 'rb') as file:
+            reader = PyPDF2.PdfReader(file)
+            try:
+                doc_info = reader.metadata
+                if doc_info:
+                    if doc_info.title:
+                        metadata["title"] = doc_info.title
+                    if doc_info.author:
+                        metadata["authors"] = doc_info.author
+            except Exception as e:
+                logger.warn("parse pdf metadata failed:%s",e)
+
+            dir_path = os.path.dirname(doc_path)
+            base_name = os.path.basename(doc_path)
+            text_content_path = f"{dir_path}/.{base_name}.txt"
+            full_text = ""
+
+            for page in reader.pages:
+                text = page.extract_text()
+                full_text += text
+            with open(text_content_path, 'w', encoding='utf-8') as f:
+                f.write(full_text)
+
+            try:
+                bookmarks = reader.outline
+                if bookmarks:
+                    catalogs = []
+                    self._parse_pdf_bookmarks(bookmarks,catalogs)
+                    metadata["catalogs"] = json.dumps(catalogs)
+            except Exception as e:
+                logger.warn("parse pdf bookmarks failed:%s",e)
+
+        return metadata
+
+    def _parse_txt(self,doc_path:str):
+        return {}
+
+    def _parse_md(self,doc_path:str):
+        metadata = {} 
+        cur_encode = "utf-8"
+        with open(doc_path,'rb') as f:
+            cur_encode = chardet.detect(f.read(1024))['encoding']
+
+        with open(doc_path, mode='r', encoding=cur_encode) as f:
+            content = f.read()
+            match = re.search(r'^# (.*)', content, re.MULTILINE)
+            if match:
+                metadata['title'] = match.group(1).strip()
+            md = Markdown(extensions=['toc'])
+            html_str = md.convert(content)
+            toc = md.toc
+            if toc:
+                metadata['catalogs'] = toc
+            
+        return metadata
+
+    def _parse_document(self,doc_path:str):
+        hash_result = None
+        title = os.path.basename(doc_path)
+        meta_data = {}
+
+        with open(doc_path, "rb") as f:
+            hash_md5 = hashlib.md5()
+            for chunk in iter(lambda: f.read(1024*1024), b""):
+                hash_md5.update(chunk)
+            hash_result = hash_md5.hexdigest()
+        try:
+            if doc_path.endswith(".md"):
+                meta_data = self._parse_md(doc_path)
+            elif doc_path.endswith(".pdf"):
+                meta_data = self._parse_pdf(doc_path)
+        except Exception as e:
+            logger.error("parse document %s failed:%s",doc_path,e)
+            traceback.print_exc()
+
+        if meta_data.get("title"):
+            title = meta_data["title"]
+        logger.info("parse document %s!",doc_path)
+        return hash_result,title,meta_data
+
+
+    def _support_file(self,file_name:str) -> bool:
+        if file_name.startswith("."):
+            return False
+        
+        if file_name.endswith(".pdf"):
+            return True
+        if file_name.endswith(".md"):
+            return True
+        if file_name.endswith(".txt"):
+            return True
+        return False
+
+    def _scan_dir(self):
+        while True:
+            time.sleep(10)
+            for directory in self.doc_dirs.keys():
+                now = time.time()
+                if now - self.doc_dirs[directory] > 60*15:
+                    self.doc_dirs[directory] = time.time()
+                else:
+                    continue
+
+                for root, dirs, files in os.walk(directory):
+                    for file in files:
+                        if self._support_file(file):
+                            full_path = os.path.join(root, file)
+                            full_path = os.path.normpath(full_path)
+                            if self.kb_db.is_doc_exist(full_path):
+                                continue
+                            
+                            file_stat = os.stat(full_path)
+                            if file_stat.st_size < 1:
+                                continue
+
+                            if file_stat.st_size < 1024*1024*8:
+                                #parse and insert
+                                hash,title,meta_data = self._parse_document(full_path)
+                                self.kb_db.add_doc(full_path,file_stat.st_size,file_stat.st_mtime,hash)
+                                self.kb_db.add_knowledge(hash,title,meta_data)
+                                
+                            else:
+                                self.kb_db.add_doc(full_path,file_stat.st_size,file_stat.st_mtime)
+
+    def _scan_document(self):
+       while True:
+        time.sleep(10)
+        parse_queue = self.kb_db.get_docs_without_hash()
+        for doc_path in parse_queue:
+                hash,title,meta_data = self._parse_document(doc_path)
+                self.kb_db.set_doc_hash(doc_path,hash)
+                self.kb_db.add_knowledge(hash,title,meta_data)
+        
+    
+
+
+# merge to standard workspace env, **ABANDON this!**
+class KnowledgeBaseFileSystemEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+        self.root_path = "."
+
+        operator_param = {
+            "path": "full path of target directory",
+        }
+        self.add_ai_function(SimpleAIFunction("list",
+                                        "list the files and sub directory in target directory,result is a json array",
+                                        self.list,operator_param))
+        
+        operator_param = {
+            "path": "full path of target file",
+        }
+        self.add_ai_function(SimpleAIFunction("cat",
+                                        "cat the file content in target path,result is a string",
+                                        self.cat,operator_param))
+    
+    def set_root_path(self,path:str):
+        self.root_path = path
+
+    
+    async def list(self,path:str) -> str:
+        directory_path = self.root_path + path
+        items = []
+
+        with await aiofiles.os.scandir(directory_path) as entries:
+            async for entry in entries:
+                item_type = "directory" if entry.is_dir() else "file"
+                items.append({"name": entry.name, "type": item_type})
+
+        return json.dumps(items)
+
+    async def cat(self,path:str) -> str:
+        file_path = self.root_path + path
+        cur_encode = "utf-8"
+        async with aiofiles.open(file_path,'rb') as f:
+            cur_encode = chardet.detect(await f.read())['encoding']
+
+        async with aiofiles.open(file_path, mode='r', encoding=cur_encode) as f:
+            content = await f.read(2048)
+        return content
+
+
+class ShellEnvironment(Environment):
+    def __init__(self, env_id: str) -> None:
+        super().__init__(env_id)
+
+        operator_param = {
+            "command": "command will execute",
+        }
+        self.add_ai_function(SimpleAIFunction("shell_exec",
+                                        "execute shell command in linux bash",
+                                        self.shell_exec,operator_param))
+        
+        #run_code_param = {
+        #    "pycode": "python code will execute",
+        #}
+        #self.add_ai_function(SimpleAIFunction("run_code",
+        #                                "execute python code",
+        #                                self.run_code,run_code_param))
+        
+
+    async def shell_exec(self,command:str) -> str:
+        import asyncio.subprocess
+        process = await asyncio.create_subprocess_shell(
+            command,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await process.communicate()
+        returncode = process.returncode
+        if returncode == 0:
+            return f"Execute success! stdout is:\n{stdout}\n"
+        else:
+            return f"Execute failed! stderr is:\n{stderr}\n"
+