0) Complete Docker build scriptes. 1) Refactor storage dir struct for release. 2) Add a common 'user_config' model. 3)More friendly CLI experience

Dockerfile

@@ -1,8 +1,13 @@
 FROM python:3.11
 WORKDIR /opt/aios
 COPY ./src /opt/aios
-COPY ./rootfs /var/aios
+COPY ./rootfs /opt/aios/app
-RUN pip install --no-cache-dir -r /opt/opendan/requirements.txt
+
+RUN mkdir -p /root/myai/app
+RUN mkdir -p /root/myai/data
+RUN mkdir -p /root/myai/etc
+
+RUN pip install --no-cache-dir -r /opt/aios/requirements.txt
 
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1

build_all_in_one.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/bash
+
+pipreqs ./src --force
+# Build the docker image
+docker build -t aios .

rootfs/tunnels.cfg.toml

@@ -1,19 +1,19 @@
-#[[tunnels]]
+[[tunnels]]
-#tunnel_id = "MyRoobot"
+tunnel_id = "MyRoobot"
-#type="TelegramTunnel"
+type="TelegramTunnel"
-#target="agent_1"
+target="agent_1"
+token="your_token"
 
-#token=""
 
-#[[tunnels]]
+[[tunnels]]
-#tunnel_id="MyEmailRobot"
+tunnel_id="MyEmailRobot"
-#type="EmailTunnel"
+type="EmailTunnel"
-#target="agent_1"
+target="agent_1"
 
-#email="your_email@msn.com"
+email="youremail@msn.com"
-#imap="outlook.office365.com:993"
+imap="outlook.office365.com:993"
-#smtp="outlook.office365.com:587"
+smtp="outlook.office365.com:587"
-#user=""
+user=""
-#password=""
+password=""
-#folder="inbox"
+folder="inbox"
-#interval=10
+interval=10

src/aios_kernel/__init__.py

@@ -14,3 +14,6 @@ from .google_text_to_speech_node import GoogleTextToSpeechNode
 from .tunnel import AgentTunnel
 from .tg_tunnel import TelegramTunnel
 from .email_tunnel import EmailTunnel
+from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
+
+AIOS_Version = "0.5.1, build 2023-9-17"

src/aios_kernel/open_ai_node.py

@@ -7,6 +7,7 @@ import logging
 
 from .compute_task import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType
 from .compute_node import ComputeNode
+from .storage import AIStorage,UserConfig
 
 logger = logging.getLogger(__name__)
 
@@ -19,23 +20,35 @@ class OpenAI_ComputeNode(ComputeNode):
             cls._instance = OpenAI_ComputeNode()
         return cls._instance
     
+    @classmethod
+    def declare_user_config(cls):
+        if os.getenv("OPENAI_API_KEY_") is None:
+            user_config = AIStorage.get_instance().get_user_config()
+            user_config.add_user_config("openai_api_key","openai api key",False,None)
 
     def __init__(self) -> None:
         super().__init__()
 
         self.is_start = False
         # openai.organization = "org-AoKrOtF2myemvfiFfnsSU8rF" #buckycloud
-        self.openai_api_key = ""
+        self.openai_api_key = None
         self.node_id = "openai_node"
-
         self.task_queue = Queue()
 
-        if os.getenv("OPENAI_API_KEY") is not None:
-            openai.api_key = os.getenv("OPENAI_API_KEY")
-        else:
-            openai.api_key = self.openai_api_key
 
+    async def initial(self):
+        if os.getenv("OPENAI_API_KEY") is not None:
+            self.openai_api_key = os.getenv("OPENAI_API_KEY")
+        else:
+            self.openai_api_key = AIStorage.get_instance().get_user_config().get_user_config("openai_api_key")
+
+        if self.openai_api_key is None:
+            logger.error("openai_api_key is None!")
+            return False
+        
+        openai.api_key = self.openai_api_key
         self.start()
+        return True
 
     async def push_task(self, task: ComputeTask, proiority: int = 0):
         logger.info(f"openai_node push task: {task.display()}")

src/aios_kernel/stability_node.py

@@ -10,6 +10,7 @@ import stability_sdk.interfaces.gooseai.generation.generation_pb2 as generation
 
 from .compute_task import ComputeTask, ComputeTaskResult, ComputeTaskState, ComputeTaskType
 from .compute_node import ComputeNode
+from .storage import AIStorage,UserConfig
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +23,10 @@ class Stability_ComputeNode(ComputeNode):
             cls._instance = Stability_ComputeNode()
         return cls._instance
     
+    @classmethod
+    def declare_user_config(cls):
+        user_config = AIStorage.get_instance().get_user_config()
+        user_config.add_user_config("stability_api_key",False,None,"STABILITY_API_KEY")
 
     def __init__(self) -> None:
         super().__init__()

src/aios_kernel/storage.py

@@ -0,0 +1,171 @@
+from typing import Any
+from pathlib import Path
+import os
+import logging
+import toml
+import aiofiles
+
+logger = logging.getLogger(__name__)
+
+_file_dir = os.path.dirname(__file__)
+
+class ResourceLocation:
+    def __init__(self) -> None:
+        pass
+
+class UserConfigItem:
+    def __init__(self) -> None:
+        self.default_value = None 
+        self.is_optional = False
+        self.item_type = "str"
+        self.desc = None
+        self.value = None
+        self.user_set = False
+        
+
+class UserConfig:
+    def __init__(self) -> None:
+        self.config_table = {}
+        self.user_config_path:str = None
+
+
+    def add_user_config(self,key:str,desc:str,is_optional:bool,default_value:Any=None,item_type="str") -> None:
+        if self.config_table.get(key) is not None:
+            logger.warning("user config key %s already exist, will be overrided",key)
+    
+        new_config_item = UserConfigItem()
+        new_config_item.default_value = default_value
+        new_config_item.is_optional = is_optional
+        new_config_item.desc = desc
+        new_config_item.item_type = item_type
+        self.config_table[key] = new_config_item
+        
+    async def load_value_from_file(self,file_path:str,is_user_config = False) -> None:
+        try: 
+            all_config = toml.load(file_path)
+            if all_config is not None:
+                for key,value in all_config.items():
+                    config_item = self.config_table.get(key)
+                    if config_item is None:
+                        logger.warning("user config key %s not exist",key)
+                        continue
+                    config_item.value = value
+                    config_item.user_set = is_user_config
+
+        except Exception as e:
+            logger.warn(f"load user config from {file_path} failed!")
+
+    async def save_value_to_user_config(self) -> None:
+        will_save_config = {}
+        for key,value in self.config_table.items():
+            if value.user_set:
+                will_save_config[key] = value.value
+        
+        if len(will_save_config) > 0:
+            try:
+                directory = os.path.dirname(self.user_config_path)
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+
+                async with aiofiles.open(self.user_config_path,"w") as f:
+                    toml_str = toml.dumps(will_save_config)
+                    await f.write(toml_str)
+            except Exception as e:
+                logger.error(f"save user config to {self.user_config_path} failed!")
+                return False
+        
+        return True
+
+    def get_user_config(self,key:str) -> Any:
+        config_item = self.config_table.get(key)
+        if config_item is None:
+            raise Exception("user config key %s not exist",key)
+        
+        if config_item.value is None:
+            return config_item.default_value
+        
+        return config_item.value
+
+    def set_user_config(self,key:str,value:Any) -> None:
+        config_item = self.config_table.get(key)
+        if config_item is None:
+            logger.warning("user config key %s not exist",key)
+            return 
+        
+        config_item.value = value
+        config_item.user_set = True
+        #TODO: save to file?
+
+        
+    def check_user_config(self) -> None:
+        check_result = {}
+        for key,config_item in self.config_table.items():
+            if config_item.value is None and not config_item.is_optional:
+                check_result[key] = config_item
+        
+        if len(check_result) > 0:
+            return check_result
+        else:
+            return None
+
+# storage sytem for current user
+class AIStorage:
+    _instance = None
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = AIStorage()
+        return cls._instance
+
+    def __init__(self) -> None:
+        self.is_dev_mode = False
+        self.user_config = UserConfig()
+
+    async def initial(self)->bool:
+        self.user_config.user_config_path = str(self.get_myai_dir() / "etc/system.cfg.toml")
+        await self.user_config.load_value_from_file(self.get_system_dir() + "/system.cfg.toml")
+        await self.user_config.load_value_from_file(self.user_config.user_config_path,True)
+
+    def get_user_config(self) -> UserConfig:
+        return self.user_config
+
+    def get_system_dir(self) -> str:
+        """ 
+        system dir is dir for aios system
+        /opt/aios
+        """
+        if self.is_dev_mode:
+            return os.path.abspath(_file_dir + "/../")
+        else:
+            return "/opt/aios/"
+        
+    
+    def get_system_app_dir(self)->str:
+        """
+        system app dir is the dir for aios build-in app 
+        /opt/aios/app
+        """
+        if self.is_dev_mode:
+            return os.path.abspath(_file_dir + "/../../rootfs/")
+        else:
+            return "/opt/aios/app/"
+    
+    def get_myai_dir(self) -> str:
+        """
+        my ai dir is the dir for user to store their ai app and data
+        ~/myai/
+        """
+        return Path.home() / "myai"
+        
+    def get_db(self,app_name:str)->ResourceLocation:
+        pass
+
+    def open_file(self,file_path:str,options:dict):
+        pass
+
+    def get_named_object(self,name:str) -> Any:
+        pass
+
+    def put_named_object(self,name:str,obj:Any) -> None:
+        pass
+

src/component/agent_manager/agent_manager.py

@@ -2,7 +2,7 @@
 import logging
 import toml
 
-from aios_kernel import AIAgent,AIAgentTemplete
+from aios_kernel import AIAgent,AIAgentTemplete,AIStorage
 from package_manager import PackageEnv,PackageEnvManager,PackageMediaInfo,PackageInstallTask
 
 logger = logging.getLogger(__name__)
@@ -12,15 +12,18 @@ class AgentManager:
     _instance = None
    
     @classmethod
-    def get_instance(cls):
+    def get_instance(cls)->'AgentManager':
         if cls._instance is None:
             cls._instance = AgentManager()
         return cls._instance
     
-    def initial(self,root_dir:str) -> None:
+    def initial(self) -> None:
-        self.agent_templete_env : PackageEnv = PackageEnvManager().get_env(f"{root_dir}/templetes/templetes.cfg")
+        system_app_dir = AIStorage.get_instance().get_system_app_dir()
-        self.agent_env : PackageEnv = PackageEnvManager().get_env(f"{root_dir}/agents/agents.cfg")
+        user_data_dir = AIStorage.get_instance().get_myai_dir()
-        self.db_path = f"{root_dir}/agents_chat.db"
+
+        self.agent_templete_env : PackageEnv = PackageEnvManager().get_env(f"{system_app_dir}/templates/templetes.cfg")
+        self.agent_env : PackageEnv = PackageEnvManager().get_env(f"{system_app_dir}/agents/agents.cfg")
+        self.db_path = f"{user_data_dir}/messages.db"
         self.loaded_agent_instance = {}
         if self.agent_templete_env is None:
             raise Exception("agent_manager initial failed")

src/component/workflow_manager/workflow_manager.py

@@ -1,10 +1,11 @@
 import logging
 import toml
-from aios_kernel import Workflow
+import os
+
+from aios_kernel import Workflow,AIStorage
 from package_manager import PackageEnv,PackageEnvManager,PackageMediaInfo,PackageInstallTask
 from agent_manager import AgentManager
 logger = logging.getLogger(__name__)
-import os
 
 class WorkflowManager:
     _instance = None
@@ -16,10 +17,13 @@ class WorkflowManager:
         return cls._instance
 
 
-    def initial(self,root_dir:str) -> None:
+    def initial(self) -> None:
         self.loaded_workflow = {}
-        self.workflow_env = PackageEnvManager().get_env(f"{root_dir}/workflows.cfg")
+        system_app_dir = AIStorage.get_instance().get_system_app_dir()
-        self.db_file = os.path.abspath(f"{root_dir}/workflows.db")
+        user_data_dir = AIStorage.get_instance().get_myai_dir()
+
+        self.workflow_env = PackageEnvManager().get_env(f"{system_app_dir}/workflows.cfg")
+        self.db_file = os.path.abspath(f"{user_data_dir}/messages.db")
         if self.workflow_env is None:
             raise Exception("WorkflowManager initial failed")
         

src/requirements.txt

@@ -6,6 +6,7 @@ beautifulsoup4==4.12.2
 mail_parser==3.15.0
 openai==0.27.10
 Pillow==10.0.1
+Pillow==10.0.1
 prompt_toolkit==3.0.39
 python-telegram-bot==20.5
 Requests==2.31.0

src/service/aios_shell/aios_shell.py

@@ -18,18 +18,17 @@ from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
 from prompt_toolkit.completion import WordCompleter
 from prompt_toolkit.styles import Style
 
-
-
-
 directory = os.path.dirname(__file__)
 sys.path.append(directory + '/../../')
-from aios_kernel import Workflow,AIAgent,AgentMsg,AgentMsgStatus,ComputeKernel,OpenAI_ComputeNode,AIBus,AIChatSession,AgentTunnel,TelegramTunnel,CalenderEnvironment,Environment,EmailTunnel
+from aios_kernel import AIOS_Version,UserConfigItem,AIStorage,Workflow,AIAgent,AgentMsg,AgentMsgStatus,ComputeKernel,OpenAI_ComputeNode,AIBus,AIChatSession,AgentTunnel,TelegramTunnel,CalenderEnvironment,Environment,EmailTunnel
 
 sys.path.append(directory + '/../../component/')
 from agent_manager import AgentManager
 from workflow_manager import WorkflowManager
 
 
+logger = logging.getLogger(__name__)
+
 shell_style = Style.from_dict({
     'title': '#87d7ff bold', #RGB
     'content': '#007f00 bold',
@@ -42,6 +41,15 @@ class AIOS_Shell:
         self.username = username
         self.current_target = "_"
         self.current_topic = "default"
+        self.is_working = True
+
+    def declare_all_user_config(self):
+        user_config = AIStorage.get_instance().get_user_config()
+        user_config.add_user_config("username","username is your full name when using AIOS",False,None,)
+
+        openai_node = OpenAI_ComputeNode.get_instance()
+        openai_node.declare_user_config()
+
 
     async def _handle_no_target_msg(self,bus:AIBus,msg:AgentMsg) -> bool:
         target_id = msg.target.split(".")[0]
@@ -70,10 +78,14 @@ class AIOS_Shell:
         cal_env.start()
         Environment.set_env_by_id("calender",cal_env)
         
-        AgentManager.get_instance().initial(os.path.abspath(directory + "/../../../rootfs/"))
+        AgentManager.get_instance().initial()
-        WorkflowManager.get_instance().initial(os.path.abspath(directory + "/../../../rootfs/workflows/"))
+        WorkflowManager.get_instance().initial()
+
         open_ai_node = OpenAI_ComputeNode.get_instance()
-        open_ai_node.start()
+        if await open_ai_node.initial() is not True:
+            logger.error("openai node initial failed!")
+            return False
+        
         ComputeKernel.get_instance().add_compute_node(open_ai_node)
         AIBus().get_default_bus().register_unhandle_message_handler(self._handle_no_target_msg)
         AIBus().get_default_bus().register_message_handler(self.username,self._user_process_msg)
@@ -81,15 +93,23 @@ class AIOS_Shell:
         TelegramTunnel.register_to_loader()
         EmailTunnel.register_to_loader()
 
-        tunnels_config_path = os.path.abspath(directory + "/../../../rootfs/tunnels.cfg.toml")
+        user_data_dir = AIStorage.get_instance().get_myai_dir()
-        tunnel_config = toml.load(tunnels_config_path)
+        tunnels_config_path = os.path.abspath(f"{user_data_dir}/tunnels.cfg.toml")
-        await AgentTunnel.load_all_tunnels_from_config(tunnel_config["tunnels"])
+        tunnel_config = None
+        try: 
+            tunnel_config = toml.load(tunnels_config_path)
+            if tunnel_config is not None:
+                await AgentTunnel.load_all_tunnels_from_config(tunnel_config["tunnels"])
+        except Exception as e:
+            logger.warning(f"load tunnels config from {tunnels_config_path} failed!")
+            
+
 
         return True 
         
 
     def get_version(self) -> str:
-        return "0.0.1"
+        return "0.5.1"
 
     async def send_msg(self,msg:str,target_id:str,topic:str,sender:str = None) -> str:
         agent_msg = AgentMsg()
@@ -101,9 +121,6 @@ class AIOS_Shell:
         else:
             return "error!"
 
-    async def install_workflow(self,workflow_id:Workflow) -> None:
-        pass
-    
     async def _user_process_msg(self,msg:AgentMsg) -> AgentMsg:
         pass
 
@@ -161,8 +178,8 @@ class AIOS_Shell:
                 return FormattedText([("class:title", f"help~~~")])
 
 
-#######################################################################################    
+##########################################################################################################################    
-history = FileHistory('history.txt')
+history = FileHistory('aios_shell_history.txt')
 session = PromptSession(history=history) 
 
 def parse_function_call(func_string):
@@ -179,21 +196,102 @@ def parse_function_call(func_string):
 
     return func_name, params
 
+async def get_user_config_from_input(check_result:dict) -> bool:
+    for key,item in check_result.items():
+        user_input = await session.prompt_async(f"{key} ({item.desc}) not define! \nPlease input:",style=shell_style)
+        if len(user_input) > 0:
+            AIStorage.get_instance().get_user_config().set_user_config(key,user_input)
+
+    await AIStorage.get_instance().get_user_config().save_value_to_user_config()
+    return True
+
+async def main_daemon_loop(shell:AIOS_Shell):
+    while shell.is_working:
+        await asyncio.sleep(1)
+
+    return 0
+
+def print_welcome_screen():
+    print("\033[1;31m")  
+    logo = """
+\t_______                    ____________________   __
+\t__  __ \______________________  __ \__    |__  | / /
+\t_  / / /__  __ \  _ \_  __ \_  / / /_  /| |_   |/ / 
+\t/ /_/ /__  /_/ /  __/  / / /  /_/ /_  ___ |  /|  /  
+\t\____/ _  .___/\___//_/ /_//_____/ /_/  |_/_/ |_/   
+\t        /_/                                          
+
+    """
+    print(logo)
+    print("\033[0m")  
+
+    print("\033[1;32m \t\tWelcome to OpenDAN - Your Personal AI OS\033[0m\n")  
+    introduce = """
+\tThe core goal of version 0.5.1 is to turn the concept of AIOS into code and get it up and running as quickly as possible. 
+\tAfter three weeks of development, our plans have undergone some changes based on the actual progress of the system. 
+\tUnder the guidance of this goal, some components do not need to be fully implemented. Furthermore, 
+\tbased on the actual development experience from several demo Intelligent Applications, 
+\twe intend to strengthen some components. This document will explain these changes and provide an update 
+\ton the current development progress of MVP(0.5.1,0.5.2)
+
+"""
+    print(introduce)
+
+    print(f"\033[1;34m \t\tVersion: {AIOS_Version}\n\033")  
+    print("\033[1;33m \tOpenDAN is an open-source project, let's define the future of Humans and AI together.\033[0m")  
+    print("\033[1;33m \tGithub\t: https://github.com/fiatrete/OpenDAN-Personal-AI-OS\033[0m")  
+    print("\033[1;33m \tWebsite\t: https://www.opendan.ai\033[0m")
+    print("\n\n")
+
 
 async def main():
-    print("aios shell prepareing...")
+    print_welcome_screen()
+    print("OpenDAN is booting...")
     logging.basicConfig(filename="aios_shell.log",filemode="w",encoding='utf-8',force=True,
                         level=logging.INFO,
                         format='[%(asctime)s]%(name)s[%(levelname)s]: %(message)s')
+    
+    if os.path.isdir(f"{directory}/../../../rootfs"):
+        AIStorage.get_instance().is_dev_mode = True
+    else:
+        AIStorage.get_instance().is_dev_mode = False    
+
+    is_daemon = False
+    if os.name != 'nt':
+        if os.getppid() == 1:
+            is_daemon = True
+
     shell = AIOS_Shell("user")   
-    await shell.initial()
+    shell.declare_all_user_config() 
+    await AIStorage.get_instance().initial()
+    check_result = AIStorage.get_instance().get_user_config().check_user_config()
+    if check_result is not None:
+        if is_daemon:
+            logger.error(check_result)
+            return 1
+        else:
+            #Remind users to enter necessary configurations.
+            if await get_user_config_from_input(check_result) is False:
+                return 1
+    
+    init_result = await shell.initial()
+    if init_result is False:
+        if is_daemon:
+            logger.error("aios shell initial failed!")
+            return 1
+        else:
+            print("aios shell initial failed!")
 
     print(f"aios shell {shell.get_version()} ready.")
+    if is_daemon:
+        return await main_daemon_loop(shell)
 
+    #TODO: read last input config
     completer = WordCompleter(['send($target,$msg,$topic)', 
                                'open($target,$topic)', 
                                'history($num,$offset)',
-                               'login($username)'
+                               'login($username)',
+                               'connect($target)',
                                'show()',
                                'exit()', 
                                'help()'], ignore_case=True)
@@ -217,8 +315,6 @@ async def main():
         print_formatted_text(show_text,style=shell_style)
         #print_formatted_text(f"{shell.username}<->{shell.current_topic}@{shell.current_target} >>> {resp}",style=shell_style)
 
-    
-
 if __name__ == "__main__":    
     asyncio.run(main())
 

test/daemon_test.py

@@ -0,0 +1,17 @@
+import daemon
+from time import sleep
+import logging
+
+logger = logging.getLogger(__name__)
+
+logging.basicConfig(filename="daemon_test.log",filemode="w",encoding='utf-8',force=True,
+                    level=logging.INFO,
+                    format='[%(asctime)s]%(name)s[%(levelname)s]: %(message)s')
+
+def main_program():
+    while True:
+        logger.info("hello world")
+        sleep(1)
+
+with daemon.DaemonContext():
+    main_program()