add knowledge pipeline

aios_shell_history.txt

@@ -0,0 +1,6 @@
+
+# 2023-09-19 16:04:55.011656
++tsukasa
+
+# 2023-09-19 16:05:27.714815
++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

src/aios_kernel/__init__.py

@@ -6,16 +6,17 @@ from .compute_kernel import ComputeKernel,ComputeTask
 from .compute_node import ComputeNode,LocalComputeNode
 from .open_ai_node import OpenAI_ComputeNode
 from .knowledge_base import KnowledgeBase 
+from .knowledge_pipeline import EmailSpider
 from .role import AIRole,AIRoleGroup
 from .workflow import Workflow
 from .bus import AIBus
 from .workflow_env import WorkflowEnvironment,CalenderEnvironment,CalenderEvent
 from .local_llama_compute_node import LocalLlama_ComputeNode
 from .whisper_node import WhisperComputeNode
-# from .google_text_to_speech_node import GoogleTextToSpeechNode
+from .google_text_to_speech_node import GoogleTextToSpeechNode
 from .tunnel import AgentTunnel
-# from .tg_tunnel import TelegramTunnel
+from .tg_tunnel import TelegramTunnel
-# from .email_tunnel import EmailTunnel
+from .email_tunnel import EmailTunnel
 from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem
 
 AIOS_Version = "0.5.1, build 2023-9-17"

src/aios_kernel/knowledge_base.py

@@ -1,7 +1,7 @@
 # define a knowledge base class
 import json
 import logging
-from . import AgentPrompt, ComputeKernel
+from . import AgentPrompt, ComputeKernel, AIStorage
 from knowledge import *
 
 
@@ -239,9 +239,6 @@ class KnowledgeBase:
         prompt.messages.append({"role": "knowledge", "content": content})    
 
         return prompt
-                    
+
-                        
-                    
-                
 
     

src/aios_kernel/knowledge_pipeline.py

@@ -0,0 +1,192 @@
+"""
+Capture your email locally, and parse out the pictures in the email body and the pictures, videos and other files in the attachment. Subsequently, it supports vectorized analysis of your personal data and serves as a knowledge base to enable large language model answers. Better results.
+
+An example of a local file is as follows:
+├── data
+│ └── alex0072@gmail.com
+│   └── 5de3e52f3a6b90cabe6cbdd4ae3a5c5b
+│     ├── email.txt
+│     ├── meta.json
+│     ├── image
+│     │   ├── 0648B869@99C03070.DB94B354.jpg
+│     └── body_image
+│         ├── 11044884873.jpg
+│         ├── 282985198265470.gif
+│         └── dd-login-service-min.png
+
+"""
+
+import imaplib
+import os
+import toml
+import logging
+import mailparser
+import hashlib
+import json
+import base64
+from bs4 import BeautifulSoup
+import requests
+
+class EmailSpider:
+    def __init__(self):
+        # logger config
+        self.logger = logging.getLogger('email spider')
+        self.logger.setLevel(logging.DEBUG)
+        ch = logging.StreamHandler()
+        formatter = logging.Formatter('%(asctime)s [%(name)s] [%(levelname)s] %(message)s')
+        ch.setFormatter(formatter)
+        self.logger.addHandler(ch)
+
+        # read config from toml file
+        # and read from config config.local.toml if exists (config.local.toml is ignored by git)
+        self.config = toml.load('./rootfs/email/config.toml')
+        if os.path.exists('./rootfs/email/config.local.toml'):
+            self.config = toml.load('./rootfs/email/config.local.toml')
+
+        self.client = self.email_client()
+
+    def email_client(self) -> imaplib.IMAP4_SSL:
+        self.logger.info(f"read email config from {self.config.get('EMAIL_IMAP_SERVER')}")
+        client = imaplib.IMAP4_SSL(
+            host=self.config.get('EMAIL_IMAP_SERVER'), 
+            port=self.config.get('EMAIL_IMAP_PORT')
+        )
+        client.login(self.config.get('EMAIL_ADDRESS'), self.config.get('EMAIL_PASSWORD'))
+        return client
+
+    def list_box(self):
+        _, mailbox_list = self.client.list()
+        for mailbox in mailbox_list:
+            print(mailbox.decode())
+
+    def read_emails(self, folder: str = 'INBOX', imap_keyword: str = "UNSEEN"):
+        self.client.select(folder)
+        _, data = self.client.uid('search', None, imap_keyword)
+        
+        # get email uid list
+        email_list = data[0].split()
+        self.logger.info(f"got {len(email_list)} emails")
+        email_list.reverse()
+        for uid in email_list:
+            if self.check_email_saved(uid):
+                self.logger.info(f"email uid {uid} already saved")
+            else:
+                self.read_and_save_email(uid)
+                self.logger.info(f"email uid {uid} saved")
+
+    def read_and_save_email(self, uid: str):
+        message_parts = "(BODY.PEEK[])"
+        _, email_data = self.client.uid('fetch', uid, message_parts)
+        mail = mailparser.parse_from_bytes(email_data[0][1])
+        self.logger.info(f"got email subject [{mail.subject}]")
+        self.save_email(mail)
+
+    def get_local_dir_name(self, mail: mailparser.MailParser) -> str:
+        dir =  f"{self.config.get('LOCAL_DIR')}/{self.config.get('EMAIL_ADDRESS')}"
+        name = f"{mail.subject}__{mail.date}"
+        name = hashlib.md5(name.encode('utf-8')).hexdigest()
+        return f"{dir}/{name}"
+
+    def check_email_saved(self, uid: str):
+        message_parts = "(BODY[HEADER])"
+        _, email_data = self.client.uid('fetch', uid, message_parts)
+        mail = mailparser.parse_from_bytes(email_data[0][1])
+        self.logger.info(f"[{uid}]check email subject [{mail.subject}]")
+        dir = self.get_local_dir_name(mail)
+        self.logger.info(f"check email saved {dir}")
+        file = f"{dir}/email.txt"
+        if os.path.exists(file):
+            return False
+        return False
+
+    # save email attachment(images)
+    def save_email_attachment(self, mail: mailparser.MailParser, email_dir: str):
+        for attachment in mail.attachments:
+            if attachment['mail_content_type'] in ['image/png', 'image/jpeg', 'image/gif']:
+                print('current mail have image attachment')
+                img_dir = f"{email_dir}/image"
+                if not os.path.exists(img_dir):
+                    os.makedirs(img_dir)
+                filename = attachment['filename']
+                filefullname = f"{img_dir}/{filename}"
+                image_data = attachment['payload']
+                try:
+                    image_data = base64.b64decode(image_data)
+                except base64.binascii.Error:
+                    image_data = image_data.encode()
+                with open(filefullname, 'wb') as f:
+                    f.write(image_data)
+                    self.logger.info(f"save email image {filename} success")
+
+    # save email body images(html content)
+    def save_body_images(self, html_content: str, email_dir: str):
+        # get all image urls
+        soup = BeautifulSoup(html_content, 'html.parser')
+        img_tags = soup.find_all('img')
+        img_urls = [img['src'] for img in img_tags if 'src' in img.attrs]
+        self.logger.info(f'Found {len(img_urls)} images in email body')
+
+        if not os.path.exists(email_dir):
+            os.makedirs(email_dir)
+
+        for img_url in img_urls:
+            # keep the original image filename(last of url)
+            img_filename = os.path.join(email_dir, img_url.split('/')[-1])
+            # download image 
+            response = requests.get(img_url, stream=True)
+            if response.status_code == 200:
+                with open(img_filename, 'wb') as img_file:
+                    for chunk in response.iter_content(1024):
+                        img_file.write(chunk)
+                self.logger.info(f'Downloaded {img_url} to {img_filename}')
+            else:
+                self.logger.info(f'Failed to download {img_url}')
+
+    # save email content to local dir
+    def save_email(self, mail: mailparser.MailParser):
+        dir = f"{self.config.get('LOCAL_DIR')}/{self.config.get('EMAIL_ADDRESS')}"
+        if not os.path.exists(dir):
+            os.makedirs(dir)
+        email_dir = self.get_local_dir_name(mail)
+        self.logger.info(f"save email to {email_dir}")
+        if not os.path.exists(email_dir):
+            os.makedirs(email_dir)
+        with open(f"{email_dir}/email.txt", "w") as f:
+            f.write(mail.body)
+        with open(f"{email_dir}/meta.json", "w", encoding='utf-8') as f:
+            mail_dict = json.loads(mail.mail_json)
+            if 'body' in mail_dict:
+                del mail_dict['body']
+            json.dump(mail_dict, f, ensure_ascii=False, indent=4)
+            self.logger.info(f"save email meta info {f.name}")
+        
+        self.save_email_attachment(mail, email_dir)
+        self.save_body_images(mail.body, f"{email_dir}/body_image")
+
+
+
+from . import AIStorage, KnowledgeBase
+
+# define singleton class knowledge pipline
+class KnowledgePipline:
+    _instance = None
+
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = KnowledgePipline()
+            cls._instance.__singleton_init__()
+
+        return cls._instance
+
+    def __singleton_init__(self) -> None:
+        self.knowledge_base = KnowledgeBase()
+
+    @classmethod
+    def declare_user_config(cls):
+        user_config = AIStorage.get_instance().get_user_config()
+        user_config.add_user_config("email_spiders","email addresses to build knowledge base",True,None,"list")
+        user_config.add_user_config("personal_dirs", "personal directories to build knowledge base", True, None, "list")
+
+    def add_email_address(self, ) -> None:
+        pass

src/requirements.txt

@@ -1,25 +1,83 @@
-chromadb==0.4
-moviepy==1.0
-base58==2.1
-base36==0.1
 aiofiles==23.2.1
-aiohttp==3.7.0
+aiohttp==3.8.5
 aioimaplib==1.0.1
+aiosignal==1.3.1
 aiosmtplib==2.0.2
+anyio==4.0.0
+async-timeout==4.0.3
+attrs==23.1.0
+backoff==2.2.1
+base36==0.1.1
+base58==2.1.1
 beautifulsoup4==4.12.2
-mail_parser==3.15.0
+cachetools==5.3.1
-prompt_toolkit==3.0.39
+certifi==2023.7.22
-pydantic==1.10.11
+charset-normalizer==3.2.0
-python-telegram-bot==20.5
+chroma-hnswlib==0.7.1
-Requests==2.31.0
+chromadb==0.4.0
-protobuf
+click==8.1.7
-stability_sdk
+colorama==0.4.6
-toml
+coloredlogs==15.0.1
-base58
+decorator==4.4.2
-google-cloud-texttospeech
+fastapi==0.99.1
-openai
+filelock==3.12.3
-Pillow
+flatbuffers==23.5.26
-aiosqlite
+frozenlist==1.4.0
-PySocks==1.7.1
+fsspec==2023.9.0
-toml==0.10.2
+google==3.0.0
+google-api-core==2.11.1
+google-auth==2.23.0
+google-cloud==0.34.0
 google-cloud-texttospeech==2.14.1
+googleapis-common-protos==1.60.0
+grpcio==1.58.0
+grpcio-status==1.58.0
+h11==0.14.0
+httpcore==0.17.3
+httptools==0.6.0
+httpx==0.24.1
+huggingface-hub==0.16.4
+humanfriendly==10.0
+idna==3.4
+imageio==2.31.3
+imageio-ffmpeg==0.4.8
+importlib-resources==6.0.1
+mail-parser==3.15.0
+monotonic==1.6
+moviepy==1.0.0
+mpmath==1.3.0
+multidict==6.0.4
+numpy==1.25.2
+onnxruntime==1.15.1
+openai==0.28.0
+overrides==7.4.0
+packaging==23.1
+pandas==2.1.0
+Pillow==10.0.0
+posthog==3.0.2
+proglog==0.1.10
+prompt-toolkit==3.0.39
+proto-plus==1.22.3
+protobuf==4.24.3
+pulsar-client==3.3.0
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pydantic==1.10.12
+PyPika==0.48.9
+pyreadline3==3.4.1
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-telegram-bot==20.5
+pytz==2023.3.post1
+PyYAML==6.0.1
+requests==2.31.0
+rsa==4.9
+simplejson==3.19.1
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.5
+starlette==0.27.0
+sympy==1.12
+telegram==0.0.1
+tokenizers==0.14.0
+toml==0.10.0