a issue parser of email

src/component/mail_environment/1/mail.json

@@ -0,0 +1,6 @@
+{
+    "subject": "开发dmc开源客户端", 
+    "from_addr": "sichangjun@buckyos.com",
+    "to_addr": ["liuzhicong@buckyos.com"],
+    "date": "2023-4-10 21:00"
+}

src/component/mail_environment/1/mail.txt

@@ -0,0 +1,51 @@
+最小功能集：4.15准备好oktc合约和客户端工具；4.21之前完成一些矿场节点部署；
+Oktc合约和rust接口 （秋总）
+实现以DMC结算的bill 和 order （done）
+实现链上挑战—— merkle联通证明；用户提交低深度半路径；矿工提供叶子原文和高深度半路径；（doing）
+实现按照价格和质押率索引的spv节点 —— 支持用户按参数匹配下单；（doing）
+实现对eth发起的http请求 auth 头认证 —— 支持https实现 链下的 write/restore/challenge 身份认证；（doing）
+实现oktc client event listener的block number本地持久化；（doing）
+用户端工具：面向普通存储用户，一键备份和恢复；
+源数据管理服务：
+注册本地路径，生成链下挑战密码本 ——随机偏移和长度QA（done）
+ 生成链上挑战密码本 —— merkle根和半深度茎节点（doing）；
+账号服务：
+添加本地路径，选择bill id发起order（done）
+按参数匹配order ——依赖按参数索引的spv 服务（doing）
+生成order提交到交付服务；（done）
+交付服务
+注册order和关联的源数据（done）
+提交merkle根（done）
+监听链事件，同步order状态，向miner写入源数据；（done）
+使用密码本持续发起链下挑战（done）
+实现链上挑战（doing）
+恢复数据到本地目录（done）
+关键日志服务
+关键状态改变日志写入（doing）
+关键状态日志事件监听接口（doing）
+轻量命令行客户端
+服务部署脚本（doing）
+传入本地文件 和 order参数一键完成备份 （doing）
+一键恢复（doing）
+链下挑战失效时，自动发起链上挑战 ——依赖关键日志服务（doing）
+mysql实现移植到sqlite（看时间和问题多不多；undo）
+矿场端工具：本版本不是发布重点；保证能在自由的节点上稳定运行即可；结构上保证了一定的伸缩性；
+存储扇区管理服务：包括gateway 和 node；保证简单部署和扩展，可靠性暂时不需实现；
+扇区gateway服务：注册node，分发扇区读写请求到node；（doing）
+扇区node服务：注册本地目录到node，注册为可用扇区；（测试中单node可以直接接入，done）
+账号服务：
+添加扇区和挂单参数，生成bill（done）；
+监听链事件，响应新的order转入交付服务；（done）
+监听链事件，响应链上挑战转入交付服务；（doing）
+交付服务：
+注册order和关联的扇区；（done）
+对user提供交付相关http接口：
+查询miner端order状态（done）
+写入源数据（done）
+完成写入后准备生成merkle root准备证明（done）
+恢复源数据（done）
+接收链下挑战返回证明（done）
+自动提交链上挑战（doing）
+关键日志服务
+关键状态改变日志写入（可推迟实现；undo）
+自有节点部署

src/component/mail_environment/__init__.py

@@ -0,0 +1,3 @@
+from .issue import IssueParser
+from .local import LocalEmail
+from .spider import EmailSpider

src/component/mail_environment/issue.py

@@ -0,0 +1,200 @@
+# define a knowledge base class
+import json
+import string
+from aios_kernel import ComputeKernel, AIStorage, Environment, SimpleAIFunction, BaseAIAgent, AgentPrompt, AgentMsg
+from knowledge import *
+from .mail import MailStorage, Mail
+
+class IssueState(Enum):
+    Open = 1
+    InProgress = 2
+    Closed = 3
+
+class IssueUpdateHistory:
+    def __init__(self, source: str, changes: dict) -> None:
+        self.source = source
+        self.changes = changes
+             
+
+class Issue:
+    def __init__(self) -> None:
+        self.id = None
+        self.summary = ""
+        self.state = IssueState.Open
+        self.source: str = None
+        self.create_time: datetime = None
+        self.deadline: datetime = None
+        self.update_history = []
+        self.children = []
+        self.parent: ObjectID = None
+
+    @classmethod
+    def object_type(cls) -> ObjectType:
+        return ObjectType.from_user_def_type_code(0)
+        
+    def to_prompt(self) -> str:
+        prompt = {
+            "id": self.id,
+            "summary": self.summary,
+            "state": self.state.name, 
+            "deadline": self.deadline
+        }
+        return json.dumps(prompt)
+    
+    @classmethod
+    def prompt_desc(cls) -> str:
+        return '''a issue contains following fileds: {
+            id: a guid string to identify a issue
+            summary: summary of this issue
+            state: state of this issue, will be one of [Open, InProgress, Closed], 
+            deadline: if issue is not closed, deadline is the time to close this issue
+        }
+        '''
+    
+    def calculate_id(self) -> str:
+        desc = {
+            "summary": self.summary,
+            "source": self.source,
+            "create_time": self.create_time,
+            "deadline": self.deadline,
+            "parent": self.parent,
+        }
+        id = str(KnowledgeObject(Issue.object_type(), desc).calculate_id())
+        self.id = id
+        return id
+
+
+class IssueStorage:
+    def __init__(self, path: str, root: Issue=None) -> None:
+        self.path = path
+        if not os.path.exists(path):
+            self.root = root
+        else:
+            self.root = json.load(open(path, "r"))
+
+    def __flush(self):
+        json.dump(self.root, open(self.path, "w"))
+
+    def get_issue_by_id(self, id: str) -> Issue:
+        self.root()
+
+    def __get_issue_by_mail_in_subtree(self, root_issue: Issue, mail_id: str):
+        if root_issue.source == mail_id:
+            return root_issue
+        if root_issue.children is None or len(root_issue.children) == 0:
+            return None
+        for child_issue in root_issue.children:
+            this_issue = self.__get_issue_by_mail_in_subtree(child_issue, mail_id)
+            if this_issue is not None:
+                return this_issue
+        return None
+
+    def get_issue_by_mail(self, mail_storage: MailStorage, mail: Mail) -> Issue:
+        if mail.reply_to is None:
+            return self.root
+        this_mail = mail_storage.get_mail_by_id(mail.reply_to)
+        while True:
+            issue = self.__get_issue_by_mail_in_subtree(self.root, this_mail.id)
+            if issue is not None:
+                return issue
+            if this_mail.replay_to is None:
+                return self.root
+            this_mail = mail_storage.get_mail_by_id(this_mail.reply_to)
+
+    
+    def add_issue(self, source_id: str, issue: dict):
+        parent_id = issue.get("parent")
+        parent_issue = self.get_issue(parent_id)
+        issue: Issue = issue
+        issue["source"] = source_id
+        issue.calculate_id()
+        parent_issue.children.append(issue)
+        self.__flush()
+
+    def update_issue(self, source_id: str, update: dict):
+        issue = self.get_issue(update["id"])
+        source = update["source"]
+        changes = {}
+        for key, value in update.items():
+            if key != "id" and key is not "source":
+                changes[key] = {
+                    "old": issue[key],
+                    "new": value,
+                }
+                issue[key] = value
+        issue.update_history.append(IssueUpdateHistory(source, changes))      
+
+        self.__flush()
+        
+
+class IssueParserEnvironment(Environment):
+    def __init__(self, env_id: str, storage: IssueStorage) -> None:
+        super().__init__(env_id)
+        self.storage = storage
+
+        update_description = '''update issue with email object'''
+
+        update_param = {
+            "source_id": "update issue with which email object id", 
+            "update_content": '''issue fileds to update, json format; 
+                if id field exists, update the issue with the id;
+                if id filed not exists, create a new issue with the content;
+                other fileds in update_content will be updated to the issue;
+            ''',
+        }
+        self.add_ai_function(SimpleAIFunction("update_issue", 
+                                            update_description,
+                                            self._update, 
+                                            update_param))
+        
+    async def _update(self, source_id: str, update_content: str):
+        update_issue = json.loads(update_content)
+        issue_id = update_issue.get("id")
+        if issue_id:
+            self.storage.update_issue(source_id, update_issue)
+        else:
+            self.storage.add_issue(source_id, update_issue)
+
+
+
+class IssueParser:
+    def __init__(self, env: KnowledgePipelineEnvironment, config: dict):
+        mail_path = string.Template(config["mail_path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
+        issue_path = string.Template(config["issue_path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
+        config["path"] = issue_path
+        
+        self.env = env
+        self.config = config
+        self.mail_storage = MailStorage(mail_path)
+
+        root_issue = None
+        if "root_issue" in config:
+            root_issue = Issue()
+            root_issue.summary = config["root_issue"]
+        self.issue_storage = IssueStorage(issue_path, root_issue)
+        self.llm_env = IssueParserEnvironment("issue_parser", self.issue_storage)
+    
+    def get_path(self) -> str:
+        return self.config["path"]
+
+    async def parse(self, mail_id: ObjectID) -> str:
+        mail_id = str(mail_id)
+        mail = self.mail_storage.get_mail_by_id(mail_id)
+        issue = self.issue_storage.get_issue_by_mail(self.mail_storage, mail)
+        mail_str = mail.to_prompt()
+        issue_str = issue.to_prompt()
+
+        mail_desc = Mail.prompt_desc()
+        issue_desc = Issue.prompt_desc()
+        prompt = f'''I'll give a mail in json format, {mail_desc};
+        and a issue in json format, {issue_desc};
+        you should read this mail {mail_str}, see if this mail associated with this issue {issue_str};
+        if this mail is about a new child issue of this issue, create a new issue with this mail, fill param update_content's summary field will mail content, set parent field with id of this issue; 
+        if this mail will update this issue, set id filed to this issue, fill update_content param with new summary and new state with this mail content;
+        then you should call update_issue function with source_id set to this mail id, and update_content in json format;
+        if this mail is not associated with issue, you should ignore this mail without an function call;
+        '''
+
+        llm_result = await BaseAIAgent.do_llm_complection(self.llm_env, AgentPrompt(prompt), AgentMsg(), "gpt-4", 16000)
+        return "update issue"
+

src/component/mail_environment/local.py

@@ -0,0 +1,34 @@
+import os
+import logging
+import json
+import string
+from knowledge import *
+from aios_kernel.storage import AIStorage
+from .mail import Mail, MailStorage
+
+
+class LocalEmail:
+    def __init__(self, env: KnowledgePipelineEnvironment, config:dict): 
+        self.config = config
+        self.env = env
+        path = string.Template(config["path"]).substitute(myai_dir=AIStorage.get_instance().get_myai_dir())
+        self.mail_storage = MailStorage(path, config.get("watch"))
+
+    async def next(self):
+        while True:
+            parsed = None
+            journals = self.env.journal.latest_journals(1)
+            if len(journals) == 1:
+                latest_journal = journals[0]
+                if latest_journal.is_finish():
+                    yield None
+                    continue
+                parsed = str(latest_journal.get_object_id())
+            
+            mail_id = self.mail_storage.next_mail_id(parsed)
+            if mail_id is None:
+                yield (None, None)
+            else:
+                yield (mail_id, str(mail_id))
+
+    

src/component/mail_environment/mail.py

@@ -0,0 +1,265 @@
+import asyncio
+import json
+import mailparser
+import base64
+import requests
+import datetime
+from bs4 import BeautifulSoup
+import sqlite3
+import html2text
+from knowledge import *
+
+class Mail:
+    def __init__(self, **kwargs) -> None:
+        self.from_addr = kwargs.get("From")
+        self.to_addr = kwargs.get("To")
+        self.subject = kwargs.get("Subject")
+        self.date = kwargs.get("Date")
+        self.bcc = kwargs.get("BCC")
+        self.cc = kwargs.get("CC")
+        self.reply_to = None
+        self.id: str = None
+        self.content: str = None
+
+    def to_prompt(self) -> str:
+        prompt = {
+            "id": self.id,
+            "subject": self.subject,
+            "from": self.from_addr,
+            "date": self.date, 
+            "content": self.content 
+        }
+        return json.dumps(prompt)
+    
+    @classmethod
+    def prompt_desc(cls) -> dict:
+        return '''a mail contains following fileds: {
+            id: a guid string to identify a mail
+            subject: subject of this mail
+            from: sender address of this mail
+            date: date of this mail
+            content: content of this mail
+        }
+        '''
+
+    def get_date(self) -> datetime.datetime:
+        datetime.datetime.strptime(self.date, "%Y-%m-%d %H:%M")
+
+    def calculate_id(self) -> str:
+        desc = {
+            "from_addr": self.from_addr,
+            "to_addr": self.to_addr,
+            "subject": self.subject, 
+            "date": self.date,
+            "content": self.content, 
+            "reply_to": self.reply_to
+        }
+        id = str(KnowledgeObject(ObjectType.Email, desc).calculate_id())
+        self.id = id
+        return id
+
+class MailStorage:
+    def __init__(self, root, watch=False):
+        self.root = root
+        if not os.path.exists(root):
+            os.makedirs(root)
+        db_file = os.path.join(root, "mail.db")
+
+        self.conn = sqlite3.connect(db_file)
+        cursor = self.conn.cursor()
+        cursor.execute(
+            """
+            CREATE TABLE IF NOT EXISTS mails (
+                uid INTEGER PRIMARY KEY,
+                object_id TEXT,
+                date DATETIME,
+                from_addr TEXT
+            )
+        """
+        )
+
+        if watch:
+            asyncio.create_task(self.watch_root())
+
+    def object_id_to_uid(self, object_id):
+        cursor = self.conn.cursor()
+        cursor.execute(
+            """
+            SELECT uid FROM mails WHERE object_id = ?
+        """,
+            (object_id,),
+        )
+        row = cursor.fetchone()
+        if row:
+            return row[0]
+        return None
+
+    def uid_to_object_id(self, uid):
+        cursor = self.conn.cursor()
+        cursor.execute(
+            """
+            SELECT object_id FROM mails WHERE uid = ?
+        """,
+            (uid,),
+        )
+        row = cursor.fetchone()
+        if row:
+            return row[0]
+        return None
+    
+    def lastest_uid(self):
+        cursor = self.conn.cursor()
+        cursor.execute(
+            """
+            SELECT uid FROM mails ORDER BY uid DESC LIMIT 1
+        """
+        )
+        row = cursor.fetchone()
+        if row:
+            return row[0]
+        return None
+    
+    def lastest_mail_id(self):
+        cursor = self.conn.cursor()
+        cursor.execute(
+            """
+            SELECT object_id FROM mails ORDER BY uid DESC LIMIT 1
+        """
+        )
+        row = cursor.fetchone()
+        if row:
+            return row[0]
+        return None
+    
+    def next_mail_id(self, id):
+        uid = 0 if id is None else self.object_id_to_uid(id)
+        cursor = self.conn.cursor()
+        cursor.execute(
+            """
+            SELECT object_id FROM mails WHERE uid > ? ORDER BY uid ASC LIMIT 1
+        """,
+            (uid,),
+        )
+        row = cursor.fetchone()
+        if row:
+            return row[0]
+        return None
+        
+    
+
+    def get_mail_by_id(self, id):
+        uid = self.object_id_to_uid(id)
+        mail = Mail()
+        mail.id = id
+        mail_dir = self.mail_dir(uid)
+        mail_json = json.load(open(f"{mail_dir}/mail.json", "r", encoding='utf-8'))
+        mail.__dict__.update(mail_json)
+        with open(f"{mail_dir}/mail.txt", "r", encoding='utf-8') as f:
+            mail_content = f.read()
+        mail.content = mail_content
+        return mail
+
+    def mail_dir(self, uid):
+        return os.path.join(self.root, str(uid))
+    
+    # for debug
+    async def watch_root(self):
+        while True:
+            latest_uid = self.lastest_uid()
+            for uid in os.listdir(self.root):
+                mail_dir = os.path.join(self.root, uid)
+                if uid.isdigit() and os.path.isdir(mail_dir):
+                    uid = int(uid)
+                    if uid <= latest_uid:
+                        continue
+                    mail = Mail()
+                    mail_json = json.load(open(f"{mail_dir}/mail.json", "r", encoding='utf-8'))
+                    
+                    mail.__dict__.update(mail_json)
+                    # mail content
+                    with open(f"{mail_dir}/mail.txt", "r", encoding='utf-8') as f:
+                        mail_content = f.read()
+                    mail.content = mail_content
+                    mail.calculate_id()
+                    cursor = self.conn.cursor()
+                    cursor.execute(
+                        """
+                        INSERT INTO mails (uid, object_id, date, from_addr)
+                        VALUES (?, ?, ?, ?)
+                    """,
+                        (uid, mail.id, mail.get_date(), mail.from_addr),
+                    )
+                    self.conn.commit()
+            await asyncio.sleep(10)
+    
+    def download(self, uid, mail: mailparser.MailParser):
+        mail_dir = self.mail_dir(uid)
+        os.makedirs(dir)
+
+        meta = json.loads(mail.mail_json)
+        mail = Mail(**meta)
+        reply_to = meta.get("In-Reply-To")
+        if reply_to:
+            mail.reply_to = self.uid_to_object_id(reply_to)
+        
+        h = html2text.HTML2Text()
+        h.ignore_links = True
+        h.ignore_images = True
+        mail_content = h.handle(mail.body)    
+        mail.content = mail_content
+       
+        mail.calculate_id()
+        del mail.content
+        json.dump(mail.__dict__, open(f"{mail_dir}/mail.json", "w", encoding='utf-8'))
+        
+        # save mail content
+        with open(f"{mail_dir}/mail.txt", "w", encoding='utf-8') as f:
+            f.write(mail_content)
+        
+        for attachment in mail.attachments:
+            if attachment['mail_content_type'] in ['image/png', 'image/jpeg', 'image/gif']:
+                filename = attachment['filename']
+                filefullname = f"{mail_dir}/{filename}"
+                image_data = attachment['payload']
+                try:
+                    image_data = base64.b64decode(image_data)
+                except base64.binascii.Error:
+                    image_data = image_data.encode()
+                with open(filefullname, 'wb') as f:
+                    f.write(image_data)
+                    logging.info(f"save email image {filename} success")
+
+        # get all image urls
+        soup = BeautifulSoup(mail.body, 'html.parser')
+        img_tags = soup.find_all('img')
+        img_urls = [img['src'] for img in img_tags if 'src' in img.attrs]
+        logging.info(f'Found {len(img_urls)} images in email body')
+
+        name_count = 0
+        
+        for img_url in img_urls:
+            # keep the original image filename(last of url)
+            ext = img_url.split('/')[-1].split('.')[-1]
+            img_filename = os.path.join(mail_dir, f"{name_count}.{ext}")
+            name_count += 1
+            # download image 
+            response = requests.get(img_url, stream=True)
+            if response.status_code == 200:
+                with open(img_filename, 'wb') as img_file:
+                    for chunk in response.iter_content(1024):
+                        img_file.write(chunk)
+                logging.info(f'Downloaded {img_url} to {img_filename}')
+            else:
+                logging.info(f'Failed to download {img_url}')
+
+        cursor = self.conn.cursor()
+        cursor.execute(
+            """
+            INSERT INTO mails (uid, object_id, date, from_addr)
+            VALUES (?, ?, ?, ?)
+        """,
+            (uid, mail.id, mail.date, mail.from_addr),
+        )
+
+
+    

src/component/mail_environment/spider.py

@@ -0,0 +1,53 @@
+import os
+import logging
+import json
+import imaplib
+import mailparser
+from knowledge import *
+from aios_kernel.storage import AIStorage
+
+
+class EmailSpider:
+    def __init__(self, env: KnowledgePipelineEnvironment, config:dict): 
+        self.config = config
+        self.env = env
+        self.env.get_logger().info(f"read email config from {self.config.get('imap_server')}")
+        self.client = imaplib.IMAP4_SSL(
+            host=self.config.get('imap_server'), 
+            port=self.config.get('imap_port')
+        )
+        self.client.login(self.config.get('address'), self.config.get('password'))
+        self.mail_local_root = os.path.join(self.env.pipeline_path, self.config.get("address"))
+        os.makedirs(self.mail_local_root)
+
+    async def next(self):
+        while True:
+            _, data = self.client.uid('search', None, "ALL")
+            uid_list = data[0].split()
+            if uid_list.len() == 0:
+                yield (None, None)
+                continue
+            
+            journals = self.env.journal.latest_journals(1)
+            from_uid = 0
+            if len(journals) == 1:
+                latest_journal = journals[0]
+                if latest_journal.is_finish():
+                    yield None
+                    continue
+                from_uid = int(latest_journal.get_input())
+                if int.from_bytes(uid_list[-1]) <= from_uid:
+                    yield (None, None)
+                    continue
+            
+            for uid in uid_list:
+                _uid = int.from_bytes(uid)
+                if _uid > from_uid:
+                    message_parts = "(BODY.PEEK[])"
+                    _, email_data = self.client.uid('fetch', uid, message_parts)
+                    mail = mailparser.parse_from_bytes(email_data[0][1])
+                    self.save_email(_uid, mail)
+
+            yield (None, None)
+
+