read mail with issue tree pipeline works

src/component/mail_environment/1/mail.json

@@ -1,6 +0,0 @@
-{
-    "subject": "开发dmc开源客户端", 
-    "from_addr": "sichangjun@buckyos.com",
-    "to_addr": ["liuzhicong@buckyos.com"],
-    "date": "2023-4-10 21:00"
-}

src/component/mail_environment/1/mail.txt

@@ -1,51 +0,0 @@
-最小功能集：4.15准备好oktc合约和客户端工具；4.21之前完成一些矿场节点部署；
-Oktc合约和rust接口 （秋总）
-实现以DMC结算的bill 和 order （done）
-实现链上挑战—— merkle联通证明；用户提交低深度半路径；矿工提供叶子原文和高深度半路径；（doing）
-实现按照价格和质押率索引的spv节点 —— 支持用户按参数匹配下单；（doing）
-实现对eth发起的http请求 auth 头认证 —— 支持https实现 链下的 write/restore/challenge 身份认证；（doing）
-实现oktc client event listener的block number本地持久化；（doing）
-用户端工具：面向普通存储用户，一键备份和恢复；
-源数据管理服务：
-注册本地路径，生成链下挑战密码本 ——随机偏移和长度QA（done）
- 生成链上挑战密码本 —— merkle根和半深度茎节点（doing）；
-账号服务：
-添加本地路径，选择bill id发起order（done）
-按参数匹配order ——依赖按参数索引的spv 服务（doing）
-生成order提交到交付服务；（done）
-交付服务
-注册order和关联的源数据（done）
-提交merkle根（done）
-监听链事件，同步order状态，向miner写入源数据；（done）
-使用密码本持续发起链下挑战（done）
-实现链上挑战（doing）
-恢复数据到本地目录（done）
-关键日志服务
-关键状态改变日志写入（doing）
-关键状态日志事件监听接口（doing）
-轻量命令行客户端
-服务部署脚本（doing）
-传入本地文件 和 order参数一键完成备份 （doing）
-一键恢复（doing）
-链下挑战失效时，自动发起链上挑战 ——依赖关键日志服务（doing）
-mysql实现移植到sqlite（看时间和问题多不多；undo）
-矿场端工具：本版本不是发布重点；保证能在自由的节点上稳定运行即可；结构上保证了一定的伸缩性；
-存储扇区管理服务：包括gateway 和 node；保证简单部署和扩展，可靠性暂时不需实现；
-扇区gateway服务：注册node，分发扇区读写请求到node；（doing）
-扇区node服务：注册本地目录到node，注册为可用扇区；（测试中单node可以直接接入，done）
-账号服务：
-添加扇区和挂单参数，生成bill（done）；
-监听链事件，响应新的order转入交付服务；（done）
-监听链事件，响应链上挑战转入交付服务；（doing）
-交付服务：
-注册order和关联的扇区；（done）
-对user提供交付相关http接口：
-查询miner端order状态（done）
-写入源数据（done）
-完成写入后准备生成merkle root准备证明（done）
-恢复源数据（done）
-接收链下挑战返回证明（done）
-自动提交链上挑战（doing）
-关键日志服务
-关键状态改变日志写入（可推迟实现；undo）
-自有节点部署

src/component/mail_environment/issue.py

@@ -14,7 +14,16 @@ class IssueUpdateHistory:
     def __init__(self, source: str, changes: dict) -> None:
         self.source = source
         self.changes = changes
-             
+
+    def to_json_dict(self) -> dict:
+        return {
+            "source": self.source,
+            "changes": self.changes,
+        }
+    
+    @classmethod
+    def from_json_dict(cls, json_dict: dict) -> "IssueUpdateHistory":
+        return IssueUpdateHistory(json_dict["source"], json_dict["changes"])
 
 class Issue:
     def __init__(self) -> None:
@@ -26,20 +35,80 @@ class Issue:
         self.deadline: datetime = None
         self.update_history = []
         self.children = []
-        self.parent: ObjectID = None
+        self.parent: str = None
+
+    def to_json_dict(self) -> dict:
+        json_dict = {
+            "id": self.id,
+            "summary": self.summary,
+            "state": self.state.name, 
+            "create_time": self.create_time,
+            "deadline": self.deadline,
+            "source": self.source,
+            "parent": self.parent,
+        }
+        if self.children is not None and len(self.children) > 0:
+            json_dict["children"] = []
+            for child in self.children:
+                json_dict["children"].append(child.to_json_dict())
+        if self.update_history is not None and len(self.update_history) > 0:
+            json_dict["update_history"] = []
+            for history in self.update_history:
+                json_dict["update_history"].append(history.to_json_dict())
+        
+        return json_dict
+
+    @classmethod
+    def from_json_dict(cls, json_dict: dict) -> "Issue":
+        issue = Issue()
+        issue.id = json_dict["id"]
+        issue.summary = json_dict["summary"]
+        issue.state = IssueState[json_dict["state"]]
+        issue.create_time = json_dict["create_time"]
+        issue.deadline = json_dict["deadline"]
+        issue.source = json_dict["source"]
+        issue.parent = json_dict["parent"]
+        if "children" in json_dict:
+            issue.children = []
+            for child_json_dict in json_dict["children"]:
+                child = Issue.from_json_dict(child_json_dict)
+                issue.children.append(child)
+        if "update_history" in json_dict:
+            issue.update_history = []
+            for history_json_dict in json_dict["update_history"]:
+                history = IssueUpdateHistory.from_json_dict(history_json_dict)
+                issue.update_history.append(history)
+        return issue
+        
 
     @classmethod
     def object_type(cls) -> ObjectType:
         return ObjectType.from_user_def_type_code(0)
-        
-    def to_prompt(self) -> str:
-        prompt = {
+    
+    def __to_desc(self, desc_list:[], recursion=None):
+        desc = {
             "id": self.id,
             "summary": self.summary,
             "state": self.state.name, 
-            "deadline": self.deadline
+            "deadline": self.deadline,
         }
-        return json.dumps(prompt)
+        desc_list.append(desc)
+        if not recursion or not self.parent:
+            return 
+        else:
+            parent = recursion.get_issue_by_id(self.parent)
+            parent.__to_desc(desc_list, recursion)
+        
+    def to_prompt(self, recursion=None) -> str:
+        desc_list = []
+        self.__to_desc(desc_list, recursion)
+        root = desc_list.pop()
+        while len(desc_list) > 0:
+            child = desc_list.pop()
+            root["child"] = child
+            root = child
+        return json.dumps(root)
+            
     
     @classmethod
     def prompt_desc(cls) -> str:
@@ -47,7 +116,8 @@ class Issue:
             id: a guid string to identify a issue
             summary: summary of this issue
             state: state of this issue, will be one of [Open, InProgress, Closed], 
-            deadline: if issue is not closed, deadline is the time to close this issue
+            deadline: if issue is not closed, deadline is the time to close this issue,
+            children: child issues of this issue
         }
         '''
     
@@ -69,14 +139,27 @@ class IssueStorage:
         self.path = path
         if not os.path.exists(path):
             self.root = root
+            self.__flush()
         else:
-            self.root = json.load(open(path, "r"))
+            root_dict = json.load(open(path, "r", encoding="utf-8"))
+            self.root = Issue.from_json_dict(root_dict)
 
     def __flush(self):
-        json.dump(self.root, open(self.path, "w"))
+        json.dump(self.root.to_json_dict(), open(self.path, "w", encoding="utf-8"), ensure_ascii=False, indent=4)
+
+    def __get_issue_by_id_in_subtree(self, root_issue: Issue, id: str):
+        if root_issue.id == id:
+            return root_issue
+        if root_issue.children is None or len(root_issue.children) == 0:
+            return None
+        for child_issue in root_issue.children:
+            this_issue = self.__get_issue_by_id_in_subtree(child_issue, id)
+            if this_issue is not None:
+                return this_issue
+        return None
 
     def get_issue_by_id(self, id: str) -> Issue:
-        self.root()
+        return self.__get_issue_by_id_in_subtree(self.root, id)
 
     def __get_issue_by_mail_in_subtree(self, root_issue: Issue, mail_id: str):
         if root_issue.source == mail_id:
@@ -102,29 +185,30 @@ class IssueStorage:
             this_mail = mail_storage.get_mail_by_id(this_mail.reply_to)
 
     
-    def add_issue(self, source_id: str, issue: dict):
-        parent_id = issue.get("parent")
-        parent_issue = self.get_issue(parent_id)
-        issue: Issue = issue
-        issue["source"] = source_id
+    def add_issue(self, source_id: str, parent_id: str, summary: str):
+        parent_issue = self.get_issue_by_id(parent_id)
+        issue = Issue()
+        issue.summary = summary
+        issue.source = source_id
+        issue.parent = parent_id
         issue.calculate_id()
         parent_issue.children.append(issue)
         self.__flush()
+        return issue
 
-    def update_issue(self, source_id: str, update: dict):
-        issue = self.get_issue(update["id"])
-        source = update["source"]
+    def update_issue(self, source_id: str, issue_id: str, update: dict):
+        issue = self.get_issue_by_id(issue_id)
         changes = {}
         for key, value in update.items():
-            if key != "id" and key is not "source":
-                changes[key] = {
-                    "old": issue[key],
-                    "new": value,
-                }
-                issue[key] = value
-        issue.update_history.append(IssueUpdateHistory(source, changes))      
+            changes[key] = {
+                "old": issue[key],
+                "new": value,
+            }
+            issue.__dict__[key] = value
+        issue.update_history.append(IssueUpdateHistory(source_id, changes))      
 
         self.__flush()
+        return issue
         
 
 class IssueParserEnvironment(Environment):
@@ -132,29 +216,37 @@ class IssueParserEnvironment(Environment):
         super().__init__(env_id)
         self.storage = storage
 
-        update_description = '''update issue with email object'''
-
+        create_description = '''create a new issue'''
+        create_param = {
+            "mail_id": "new issue with which email object id", 
+            "issue_id": '''new issue's parent issue id''',
+            "summary": '''new issue's summary''',
+        }
+        self.add_ai_function(SimpleAIFunction("create_issue", 
+                                            create_description,
+                                            self._create, 
+                                            create_param))
+        
+        update_description = '''update an existing issue'''
         update_param = {
-            "source_id": "update issue with which email object id", 
-            "update_content": '''issue fileds to update, json format; 
-                if id field exists, update the issue with the id;
-                if id filed not exists, create a new issue with the content;
-                other fileds in update_content will be updated to the issue;
-            ''',
+            "mail_id": "update issue with which email object id", 
+            "issue_id": '''update issue's id''',
+            "summary": '''issue's new summary''',
         }
         self.add_ai_function(SimpleAIFunction("update_issue", 
                                             update_description,
                                             self._update, 
                                             update_param))
         
-    async def _update(self, source_id: str, update_content: str):
-        update_issue = json.loads(update_content)
-        issue_id = update_issue.get("id")
-        if issue_id:
-            self.storage.update_issue(source_id, update_issue)
-        else:
-            self.storage.add_issue(source_id, update_issue)
-
+    async def _create(self, mail_id: str, issue_id: str, summary: str):
+        issue = self.storage.add_issue(mail_id, issue_id, summary)
+        return issue.id
+            
+    async def _update(self, mail_id: str, issue_id: str, summary: str):
+        update = {}
+        update["summary"] = summary
+        issue = self.storage.update_issue(mail_id, issue_id, update)
+        return issue.id
 
 
 class IssueParser:
@@ -169,10 +261,30 @@ class IssueParser:
 
         root_issue = None
         if "root_issue" in config:
-            root_issue = Issue()
-            root_issue.summary = config["root_issue"]
+            root_config = config["root_issue"]
+            root_issue = IssueParser.__load_issue_config(root_config)
+            IssueParser.__calac_issue_id(root_issue)
+
         self.issue_storage = IssueStorage(issue_path, root_issue)
         self.llm_env = IssueParserEnvironment("issue_parser", self.issue_storage)
+
+    @classmethod
+    def __load_issue_config(cls, issue_config: dict) -> Issue: 
+        issue = Issue()
+        issue.summary = issue_config["summary"]
+        if "children" in issue_config:
+            for child_config in issue_config["children"]:
+                child_issue = cls.__load_issue_config(child_config)
+                issue.children.append(child_issue)
+        return issue
+    
+    @classmethod
+    def __calac_issue_id(cls, issue: Issue):
+        issue_id = issue.calculate_id()
+        for child in issue.children:
+            child.parent = issue_id
+            cls.__calac_issue_id(child)
+        
     
     def get_path(self) -> str:
         return self.config["path"]
@@ -182,19 +294,21 @@ class IssueParser:
         mail = self.mail_storage.get_mail_by_id(mail_id)
         issue = self.issue_storage.get_issue_by_mail(self.mail_storage, mail)
         mail_str = mail.to_prompt()
-        issue_str = issue.to_prompt()
+        issue_str = issue.to_prompt(recursion=self.issue_storage)
 
         mail_desc = Mail.prompt_desc()
         issue_desc = Issue.prompt_desc()
-        prompt = f'''I'll give a mail in json format, {mail_desc};
-        and a issue in json format, {issue_desc};
-        you should read this mail {mail_str}, see if this mail associated with this issue {issue_str};
-        if this mail is about a new child issue of this issue, create a new issue with this mail, fill param update_content's summary field will mail content, set parent field with id of this issue; 
-        if this mail will update this issue, set id filed to this issue, fill update_content param with new summary and new state with this mail content;
-        then you should call update_issue function with source_id set to this mail id, and update_content in json format;
-        if this mail is not associated with issue, you should ignore this mail without an function call;
-        '''
+        prompt = AgentPrompt()
+        prompt.system_message = {"role": "system", "content": f'''
+        I'm a CEO of a company named 巴克云; You'ar my assistant, and you should help me to manage my issues. Issues is a concept in software development of this company, but I use it to manage my work.
+        I'll give you a mail in json format, {mail_desc};
+        and a issue in json format, {issue_desc}. Read mail's fileds and issue's fileds, and decide if you should update the issue or create a new issue with this mail.
+        Then call the function create_issue or update_issue.
+        if this mail is not associated with issue, you should ignore this mail.'''}
+        
+        prompt.append(AgentPrompt(f'''Mail is {mail_str}, issue is {issue_str}. Answer me the function's return value or None if igonred.               
+        '''))
 
-        llm_result = await BaseAIAgent.do_llm_complection(self.llm_env, AgentPrompt(prompt), AgentMsg(), "gpt-4", 16000)
+        llm_result = await BaseAIAgent.do_llm_complection(prompt, "gpt-4-1106-preview", 4000, env=self.llm_env)
         return "update issue"
 

src/component/mail_environment/local.py

@@ -31,4 +31,7 @@ class LocalEmail:
             else:
                 yield (mail_id, str(mail_id))
 
-    
+
+class LocalEmailWithFilter:
+    def __init__(self, env: KnowledgePipelineEnvironment, config:dict):
+        pass

src/component/mail_environment/mail.py

@@ -261,5 +261,4 @@ class MailStorage:
             (uid, mail.id, mail.date, mail.from_addr),
         )
 
-