From 3cff2419a17d2e9108573e2c63c7c6a4e90430c0 Mon Sep 17 00:00:00 2001 From: alexsunxl Date: Thu, 31 Aug 2023 16:14:13 +0800 Subject: [PATCH 01/13] Add a service: email spider #43 --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f616e40..3bb2bad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .vscode/ *.pyc -rootfs/data *.log rootfs/email/config.local.toml rootfs/data @@ -11,3 +10,4 @@ aios_shell_history.txt math_school_env.db workflows.db + From dbb1ab44374e3d4bd23b3e1206d6d99b83480e05 Mon Sep 17 00:00:00 2001 From: Song Date: Sat, 9 Sep 2023 22:08:39 +0800 Subject: [PATCH 02/13] The argument for is_support should be task_type --- src/aios_kernel/compute_kernel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aios_kernel/compute_kernel.py b/src/aios_kernel/compute_kernel.py index 18ddeaf..97f9b9b 100644 --- a/src/aios_kernel/compute_kernel.py +++ b/src/aios_kernel/compute_kernel.py @@ -131,6 +131,7 @@ class ComputeKernel: return "error!" + def text_embedding(self,input:str,model_name:Optional[str] = None): task_req = ComputeTask() task_req.set_text_embedding_params(input,model_name) From f4933b878af303e9399db1f1af3735108211bc60 Mon Sep 17 00:00:00 2001 From: streetycat <305190374@qq.com> Date: Sun, 17 Sep 2023 13:16:21 +0000 Subject: [PATCH 03/13] add compute node for llama --- aios_shell.log | 10 + history.txt | 288 ++++++++++++++++++++++++++++ rootfs/workflows/math_school_env.db | Bin 0 -> 12288 bytes rootfs/workflows/workflows.db | Bin 0 -> 24576 bytes 4 files changed, 298 insertions(+) create mode 100644 aios_shell.log create mode 100644 history.txt create mode 100644 rootfs/workflows/math_school_env.db create mode 100644 rootfs/workflows/workflows.db diff --git a/aios_shell.log b/aios_shell.log new file mode 100644 index 0000000..c3927e7 --- /dev/null +++ b/aios_shell.log @@ -0,0 +1,10 @@ +[2023-09-17 13:10:22,448]aios_kernel.compute_kernel[INFO]: add compute_node OpenAI_ComputeNode: openai_node to compute_kernel +[2023-09-17 13:10:22,448]aios_kernel.compute_kernel[INFO]: add compute_node LocalLlama_ComputeNode: default to compute_kernel +[2023-09-17 13:10:33,359]package_manager.env[WARNING]: pkg_load math_school, cid:None error,not found ,search_parent=True +[2023-09-17 13:10:33,366]aios_kernel.environment[INFO]: set value GOAL in env math_school to 成为最好的学校 +[2023-09-17 13:10:33,372]aios_kernel.environment[WARNING]: get value now in env math_school failed!,type is not str +[2023-09-17 13:10:33,373]aios_kernel.compute_kernel[INFO]: compute_kernel get task: ComputeTask: 5bdfa49d8d63411aa3d8fa10c596f116 ComputeTaskType.LLM_COMPLETION ComputeTaskState.INIT +[2023-09-17 13:10:33,373]aios_kernel.queue_compute_node[INFO]: LocalLlama_ComputeNode: default push task: ComputeTask: 5bdfa49d8d63411aa3d8fa10c596f116 ComputeTaskType.LLM_COMPLETION ComputeTaskState.INIT +[2023-09-17 13:10:33,374]aios_kernel.queue_compute_node[INFO]: LocalLlama_ComputeNode: default get task: ComputeTask: 5bdfa49d8d63411aa3d8fa10c596f116 ComputeTaskType.LLM_COMPLETION ComputeTaskState.INIT +[2023-09-17 13:11:38,996]aios_kernel.local_llama_compute_node[INFO]: LocalLlama_ComputeNode task responsed, request: {'prompts': ['你是精通数学的老师', '现在时间是:2023-09-17 13:10:33你在学校任职，担任小学老师。学校由 小学老师、初中老师、高中老师、教导处主任 组成。\n当你发现学生的水平不是小学生时，应使用 sendmsg(老师名称,问题) 的方法，把学生的问题转发给学校里合适的老师\n当学生发来作业时，进行批改(满分5分)，并把批改结果以 postmsg(教导处主任,学生名_作业结果) 的方法，将一次作业情况汇报给教导处主任。\n你会根据教导处主任的指示，定期调整教学方法', '5+3=8', '', '5+3=8']}, status-code: 200, headers: {'Server': 'nginx/1.18.0 (Ubuntu)', 'Date': 'Sun, 17 Sep 2023 13:11:34 GMT', 'Content-Type': 'application/json', 'Content-Length': '5635', 'Connection': 'keep-alive'}, content: b'{"results":["\xe4\xbd\xa0\xe6\x98\xaf\xe7\xb2\xbe\xe9\x80\x9a\xe6\x95\xb0\xe5\xad\xa6\xe7\x9a\x84\xe8\x80\x81\xe5\xb8\x88,\xe8\x99\xbd\xe7\x84\xb6\xe6\x9a\x82\xe6\x97\xb6\xe5\xba\x86\xe7\xa5\x9d\xe5\x8f\xaa\xe6\x9c\x8910\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8a\xb3\xe9\x80\x83.\xe4\xbd\xa0\xe4\xbf\x9d\xe8\xaf\x81\xef\xbc\x8c25\xe5\xb9\xb4\xe5\x90\x8e\xe4\xb9\x9f\xe4\xbc\x9a\xe8\x87\xb4\xe5\x8a\x9b\xe4\xba\x8e\xe5\xa6\x82\xe6\xad\xa4\xe9\xab\x98\xe6\xb0\xb4\xe5\xb9\xb3\xe7\x9a\x84\xe5\x85\xa8\xe7\x90\x83\xe6\x95\x99\xe8\x82\xb2\xe5\x90\x97\xef\xbc\x9f\\nsevensnowy77\\nWhen I was an undergraduate student at Peking University in the late \'80s, my major was Mathematics. I joined a club of mathematics and physics enthusiast, named \\"beyond 3\\", which held open lectures about new mathematics from time to time. The lecturer came from different universities such as Shanghai Jiao Tong University, Nankai University, Fudan University, Nanjing University, etc., and they all gave us excellent lessons with great sense of humor. At that time I gained many precious advices for future career development: firstly, develop a continuous interest and passion toward math; secondly, keep learning other related disciplines like computer science or physical sciences; thirdly, apply whatever you have learned into real life, especially when it comes to doing something useful for society. So after graduation, I started to work in a local university. My boss told me that if I want to be more active in research area, then I should go back to school to pursue further study. But as far as I\'m concerned, it is much easier to do academic research than teaching since I used to spend most of my spare time on reading books during college years. And going back to school also means I will stop working and start paying tuition fees again. It seems no reason for me to go back. Then she had one good advice to me -- let your students see you are passionate about what you teach them, being kind enough, respectable but not afraid to make mistakes when demonstrating in front of them. After several discussions with her, we finally decided to send me back to school instead of stopping my job there. When I went back to school, I found that I really enjoy doing what I am doing right now! In fact, it is because I already received quite some instructions from my teachers before. Thanks God!\\nA Weibull distribution has two shape parameters. If the survival function (the probability) that a ship will last ________ years is fitted by this curve, we can conclude that","\xe7\x8e\xb0\xe5\x9c\xa8\xe6\x97\xb6\xe9\x97\xb4\xe6\x98\xaf:2023-09-17 13:10:33\xe4\xbd\xa0\xe5\x9c\xa8\xe5\xad\xa6\xe6\xa0\xa1\xe4\xbb\xbb\xe8\x81\x8c\xef\xbc\x8c\xe6\x8b\x85\xe4\xbb\xbb\xe5\xb0\x8f\xe5\xad\xa6\xe8\x80\x81\xe5\xb8\x88\xe3\x80\x82\xe5\xad\xa6\xe6\xa0\xa1\xe7\x94\xb1 \xe5\xb0\x8f\xe5\xad\xa6\xe8\x80\x81\xe5\xb8\x88\xe3\x80\x81\xe5\x88\x9d\xe4\xb8\xad\xe8\x80\x81\xe5\xb8\x88\xe3\x80\x81\xe9\xab\x98\xe4\xb8\xad\xe8\x80\x81\xe5\xb8\x88\xe3\x80\x81\xe6\x95\x99\xe5\xaf\xbc\xe5\xa4\x84\xe4\xb8\xbb\xe4\xbb\xbb \xe7\xbb\x84\xe6\x88\x90\xe3\x80\x82\\n\xe5\xbd\x93\xe4\xbd\xa0\xe5\x8f\x91\xe7\x8e\xb0\xe5\xad\xa6\xe7\x94\x9f\xe7\x9a\x84\xe6\xb0\xb4\xe5\xb9\xb3\xe4\xb8\x8d\xe6\x98\xaf\xe5\xb0\x8f\xe5\xad\xa6\xe7\x94\x9f\xe6\x97\xb6\xef\xbc\x8c\xe5\xba\x94\xe4\xbd\xbf\xe7\x94\xa8 sendmsg(\xe8\x80\x81\xe5\xb8\x88\xe5\x90\x8d\xe7\xa7\xb0,\xe9\x97\xae\xe9\xa2\x98) \xe7\x9a\x84\xe6\x96\xb9\xe6\xb3\x95\xef\xbc\x8c\xe6\x8a\x8a\xe5\xad\xa6\xe7\x94\x9f\xe7\x9a\x84\xe9\x97\xae\xe9\xa2\x98\xe8\xbd\xac\xe5\x8f\x91\xe7\xbb\x99\xe5\xad\xa6\xe6\xa0\xa1\xe9\x87\x8c\xe5\x90\x88\xe9\x80\x82\xe7\x9a\x84\xe8\x80\x81\xe5\xb8\x88\\n\xe5\xbd\x93\xe5\xad\xa6\xe7\x94\x9f\xe5\x8f\x91\xe6\x9d\xa5\xe4\xbd\x9c\xe4\xb8\x9a\xe6\x97\xb6\xef\xbc\x8c\xe8\xbf\x9b\xe8\xa1\x8c\xe6\x89\xb9\xe6\x94\xb9(\xe6\xbb\xa1\xe5\x88\x865\xe5\x88\x86)\xef\xbc\x8c\xe5\xb9\xb6\xe6\x8a\x8a\xe6\x89\xb9\xe6\x94\xb9\xe7\xbb\x93\xe6\x9e\x9c\xe4\xbb\xa5 postmsg(\xe6\x95\x99\xe5\xaf\xbc\xe5\xa4\x84\xe4\xb8\xbb\xe4\xbb\xbb,\xe5\xad\xa6\xe7\x94\x9f\xe5\x90\x8d_\xe4\xbd\x9c\xe4\xb8\x9a\xe7\xbb\x93\xe6\x9e\x9c) \xe7\x9a\x84\xe6\x96\xb9\xe6\xb3\x95\xef\xbc\x8c\xe5\xb0\x86\xe4\xb8\x80\xe6\xac\xa1\xe4\xbd\x9c\xe4\xb8\x9a\xe6\x83\x85\xe5\x86\xb5\xe6\xb1\x87\xe6\x8a\xa5\xe7\xbb\x99\xe6\x95\x99\xe5\xaf\xbc\xe5\xa4\x84\xe4\xb8\xbb\xe4\xbb\xbb\xe3\x80\x82\\n\xe4\xbd\xa0\xe4\xbc\x9a\xe6\xa0\xb9\xe6\x8d\xae\xe6\x95\x99\xe5\xaf\xbc\xe5\xa4\x84\xe4\xb8\xbb\xe4\xbb\xbb\xe7\x9a\x84\xe6\x8c\x87\xe7\xa4\xba\xef\xbc\x8c\xe5\xae\x9a\xe6\x9c\x9f\xe8\xb0\x83\xe6\x95\xb4\xe6\x95\x99\xe5\xad\xa6\xe6\x96\xb9\xe6\xb3\x95\xe5\x92\x8c\xe5\x88\xb6\xe5\xba\xa6\xe3\x80\x82","5+3=8: the new division of labour\\nThe following is a guest post by my colleague Rodrigo Cavalheiro. It outlines his research on the way that technology is changing traditional functions in organisations \xe2\x80\x93 particularly offices and business support roles \xe2\x80\x93 with interesting implications for how we think about work organisation.\\nWe are witnessing what I call the \xe2\x80\x9cnew division of labour\xe2\x80\x9d. In the industrial revolution, the main change was one of resources from manual to mechanised production; in this case, it\xe2\x80\x99s all about knowledge. To put it simply, computers have been decentralising the traditional hierarchical division of knowledge within firms into a networked structure of information, where some parts can only be accessed through an institution or firm (i.e., private databases). This new type of division of labour does not imply that people do not need to interact anymore to perform their tasks, but instead implies that they no longer need to rely exclusively on each other for specific activities, as long as there is shared access to required data and appropriate technological tools. The emergence of freelancers (\xe2\x80\x9cgig economy\xe2\x80\x9d) has significantly contributed to these changes in the workplace, since these workers depend much less on institutional needs. Also, organizations of various forms now face more competition than ever before in order to attract talent. For instance, technology giant Google offers space for employees to sleep overnight at its campuses because housing prices in Silicon Valley make living an unaffordable option. Thus, organizations tend to provide better working conditions in order to compete for human capital. However, this means that managers often lack feedback on whether specific initiatives result in improvements regarding productivity, quality, costs and customer satisfaction.\\nTo have an idea of how this impacts job design, consider the example of an office clerk who previously used photocopiers in her/his daily duties. Nowadays, anyone can send scanned documents directly to another person\xe2\x80\x99s email account using free cloud-storage applications such as Dropbox. Why should two persons have to go down to the copier when someone could just scan a document, compress it and send it to the recipient? I guess you may have already experienced similar situations yourself.\\nThis shift towards sharing also requires us to challenge previous understandings of hierarchy in organizational structures. In the past, organizational designs were based on formal authority and power relationships. Today, however, most jobs require skills beyond those associated with traditional","Live in-person with a prolific life coach.\\nGet 20% off Coaching when you sign up for the monthly subscription!","5+3=8 \xf0\x9f\x8c\x90 A new book by @peterberger The first, but hopefully not the last https://t.co/uJ7IvS6sT1\\n\xe2\x80\x94 Matthew Dear (@matthewdear) May 29, 2016"]}' +[2023-09-17 13:11:51,253]aios_kernel.workflow[INFO]: math_school.小学老师 process user:5+3=8,llm str is :['你是精通数学的老师,虽然暂时庆祝只有10年的劳逃.你保证，25年后也会致力于如此高水平的全球教育吗？\nsevensnowy77\nWhen I was an undergraduate student at Peking University in the late \'80s, my major was Mathematics. I joined a club of mathematics and physics enthusiast, named "beyond 3", which held open lectures about new mathematics from time to time. The lecturer came from different universities such as Shanghai Jiao Tong University, Nankai University, Fudan University, Nanjing University, etc., and they all gave us excellent lessons with great sense of humor. At that time I gained many precious advices for future career development: firstly, develop a continuous interest and passion toward math; secondly, keep learning other related disciplines like computer science or physical sciences; thirdly, apply whatever you have learned into real life, especially when it comes to doing something useful for society. So after graduation, I started to work in a local university. My boss told me that if I want to be more active in research area, then I should go back to school to pursue further study. But as far as I\'m concerned, it is much easier to do academic research than teaching since I used to spend most of my spare time on reading books during college years. And going back to school also means I will stop working and start paying tuition fees again. It seems no reason for me to go back. Then she had one good advice to me -- let your students see you are passionate about what you teach them, being kind enough, respectable but not afraid to make mistakes when demonstrating in front of them. After several discussions with her, we finally decided to send me back to school instead of stopping my job there. When I went back to school, I found that I really enjoy doing what I am doing right now! In fact, it is because I already received quite some instructions from my teachers before. Thanks God!\nA Weibull distribution has two shape parameters. If the survival function (the probability) that a ship will last ________ years is fitted by this curve, we can conclude that', '现在时间是:2023-09-17 13:10:33你在学校任职，担任小学老师。学校由 小学老师、初中老师、高中老师、教导处主任 组成。\n当你发现学生的水平不是小学生时，应使用 sendmsg(老师名称,问题) 的方法，把学生的问题转发给学校里合适的老师\n当学生发来作业时，进行批改(满分5分)，并把批改结果以 postmsg(教导处主任,学生名_作业结果) 的方法，将一次作业情况汇报给教导处主任。\n你会根据教导处主任的指示，定期调整教学方法和制度。', '5+3=8: the new division of labour\nThe following is a guest post by my colleague Rodrigo Cavalheiro. It outlines his research on the way that technology is changing traditional functions in organisations – particularly offices and business support roles – with interesting implications for how we think about work organisation.\nWe are witnessing what I call the “new division of labour”. In the industrial revolution, the main change was one of resources from manual to mechanised production; in this case, it’s all about knowledge. To put it simply, computers have been decentralising the traditional hierarchical division of knowledge within firms into a networked structure of information, where some parts can only be accessed through an institution or firm (i.e., private databases). This new type of division of labour does not imply that people do not need to interact anymore to perform their tasks, but instead implies that they no longer need to rely exclusively on each other for specific activities, as long as there is shared access to required data and appropriate technological tools. The emergence of freelancers (“gig economy”) has significantly contributed to these changes in the workplace, since these workers depend much less on institutional needs. Also, organizations of various forms now face more competition than ever before in order to attract talent. For instance, technology giant Google offers space for employees to sleep overnight at its campuses because housing prices in Silicon Valley make living an unaffordable option. Thus, organizations tend to provide better working conditions in order to compete for human capital. However, this means that managers often lack feedback on whether specific initiatives result in improvements regarding productivity, quality, costs and customer satisfaction.\nTo have an idea of how this impacts job design, consider the example of an office clerk who previously used photocopiers in her/his daily duties. Nowadays, anyone can send scanned documents directly to another person’s email account using free cloud-storage applications such as Dropbox. Why should two persons have to go down to the copier when someone could just scan a document, compress it and send it to the recipient? I guess you may have already experienced similar situations yourself.\nThis shift towards sharing also requires us to challenge previous understandings of hierarchy in organizational structures. In the past, organizational designs were based on formal authority and power relationships. Today, however, most jobs require skills beyond those associated with traditional', 'Live in-person with a prolific life coach.\nGet 20% off Coaching when you sign up for the monthly subscription!', '5+3=8 🌐 A new book by @peterberger The first, but hopefully not the last https://t.co/uJ7IvS6sT1\n— Matthew Dear (@matthewdear) May 29, 2016'] diff --git a/history.txt b/history.txt new file mode 100644 index 0000000..4dd328e --- /dev/null +++ b/history.txt @@ -0,0 +1,288 @@ + +# 2023-09-15 03:41:50.683879 ++hello + +# 2023-09-15 03:42:12.120490 ++help + +# 2023-09-15 06:37:37.070634 ++hel + +# 2023-09-15 06:37:43.470674 ++he + +# 2023-09-15 06:37:46.694774 ++help() + +# 2023-09-15 06:37:58.255558 ++open($target,$topic) + +# 2023-09-15 06:38:39.946346 ++open(math_school, hello) + +# 2023-09-15 06:38:52.255612 ++5+3 + +# 2023-09-15 06:40:08.434216 ++open(小学老师,math_school) + +# 2023-09-15 06:40:12.511473 ++5+3 + +# 2023-09-15 06:41:07.635875 ++send(小学老师, 5+3, math_school) + +# 2023-09-15 06:41:28.111632 ++send("小学老师", "5+3", "math_school") + +# 2023-09-15 06:46:03.361318 ++5+3 + +# 2023-09-15 06:58:04.568600 ++open(math_school, 小学老师) + +# 2023-09-15 06:58:10.940263 ++5+3 + +# 2023-09-15 08:14:18.370059 ++op + +# 2023-09-15 08:14:57.303640 ++open(math_school,小学老师) + +# 2023-09-15 08:16:13.045359 ++5+3 + +# 2023-09-15 08:24:53.045039 ++open(math_school,小学老师) + +# 2023-09-15 08:24:56.551048 ++5+3 + +# 2023-09-15 08:27:14.113933 ++open(math_school,小学老师) + +# 2023-09-15 08:27:31.679873 ++5+3=8 + +# 2023-09-15 08:28:39.132082 ++$open(math_school,小学老师) + +# 2023-09-15 08:29:11.896668 ++5+3=8 + +# 2023-09-15 08:36:34.342241 ++open(math_school,小学老师) + +# 2023-09-15 08:36:39.386247 ++5+3=8 + +# 2023-09-15 09:57:19.516838 ++open(math_school,小学老师) + +# 2023-09-15 09:57:33.371520 ++5+3=8 + +# 2023-09-15 10:00:53.355748 ++open(math_school,小学老师) + +# 2023-09-15 10:01:02.862918 ++5+3=8 + +# 2023-09-15 10:05:53.128794 ++open(math_school,小学老师) + +# 2023-09-15 10:05:58.133501 ++5+3=8 + +# 2023-09-15 10:07:42.454258 ++open(math_school,小学老师) + +# 2023-09-15 10:07:47.737608 ++5+3=8 + +# 2023-09-15 10:10:09.716805 ++open(math_school,小学老师) + +# 2023-09-15 10:10:16.097322 ++5+3=8 + +# 2023-09-15 10:12:08.449379 ++open(math_school,小学老师) + +# 2023-09-15 10:12:13.079025 ++5+3=8 + +# 2023-09-15 10:13:48.545683 ++open(math_school,小学老师) + +# 2023-09-15 10:13:54.927181 ++5+3=8 + +# 2023-09-15 10:19:17.745657 ++open(math_school,小学老师) + +# 2023-09-15 10:19:23.711550 ++5+3=8 + +# 2023-09-15 10:21:26.701757 ++open(math_school,小学老师) + +# 2023-09-15 10:21:32.767604 ++5+3=8 + +# 2023-09-15 10:26:04.480292 ++open(math_school,小学老师) + +# 2023-09-15 10:26:10.045387 ++5+3=8 + +# 2023-09-15 10:31:28.045250 ++open(match_school, 小学老师) + +# 2023-09-15 10:31:36.167547 ++5+3=8 + +# 2023-09-15 10:32:29.338152 ++open(match_school, 小学老师) + +# 2023-09-15 10:32:35.693949 ++3+5=8 + +# 2023-09-15 10:33:03.433018 ++open(match_school, 小学老师) + +# 2023-09-15 10:33:14.326229 ++4+5=9 + +# 2023-09-15 10:34:21.259810 ++open(math_school, 小学老师) + +# 2023-09-15 10:34:34.422484 ++1+1=2 + +# 2023-09-16 10:13:23.564438 ++open(math_school, 小学老师) + +# 2023-09-16 10:13:29.941319 ++5+3=8 + +# 2023-09-16 10:14:23.145831 ++open(math_school, 小学老师) + +# 2023-09-16 10:14:26.382580 ++5+3=8 + +# 2023-09-17 11:43:17.436353 ++open(math_school, 小学老师) + +# 2023-09-17 11:43:22.332117 ++5+3=8 + +# 2023-09-17 11:49:27.933651 ++open(math_school, 小学老师) + +# 2023-09-17 11:49:35.619278 ++5+3=8 + +# 2023-09-17 11:51:28.813245 ++open(math_school, 小学老师) + +# 2023-09-17 11:51:34.474472 ++5+3=8 + +# 2023-09-17 11:58:28.221803 ++open(math_school, 小学老师) + +# 2023-09-17 11:59:16.715335 ++5+3=8 + +# 2023-09-17 12:04:10.819566 ++open(math_school, 小学老师) + +# 2023-09-17 12:04:15.404546 ++5+3=8 + +# 2023-09-17 12:04:39.159679 ++open(math_school, 小学老师) + +# 2023-09-17 12:04:50.036365 ++5+3=8 + +# 2023-09-17 12:05:20.015642 ++open(math_school, 小学老师) + +# 2023-09-17 12:05:26.268601 ++5+3=8 + +# 2023-09-17 12:07:52.687876 ++open(math_school, 小学老师) + +# 2023-09-17 12:08:10.093561 ++5+3=8 + +# 2023-09-17 12:10:49.055737 ++open(math_school, 小学老师) + +# 2023-09-17 12:10:55.622177 ++5+3=8 + +# 2023-09-17 12:17:37.593976 ++open(math_school, 小学老师) + +# 2023-09-17 12:17:41.496433 ++5+3=8 + +# 2023-09-17 12:43:55.489719 ++open(math_school, 小学老师) + +# 2023-09-17 12:43:59.837491 ++5+3=8 + +# 2023-09-17 12:46:07.069625 ++open(math_school, 小学老师) + +# 2023-09-17 12:46:13.028847 ++5+3=8 + +# 2023-09-17 12:48:40.762746 ++open(math_school, 小学老师) + +# 2023-09-17 12:48:45.062939 ++5+3=8 + +# 2023-09-17 12:52:08.003428 ++open(math_school, 小学老师) + +# 2023-09-17 12:52:15.104145 ++5+3=8 + +# 2023-09-17 12:58:11.789619 ++open(math_school, 小学老师) + +# 2023-09-17 12:58:17.994040 ++5+3=8 + +# 2023-09-17 13:03:18.654092 ++open(math_school, 小学老师) + +# 2023-09-17 13:03:23.698141 ++5+3=8 + +# 2023-09-17 13:05:39.719027 ++open(math_school, 小学老师) + +# 2023-09-17 13:05:44.820351 ++5+3=8 + +# 2023-09-17 13:08:18.344888 ++open(math_school, 小学老师) + +# 2023-09-17 13:08:25.670674 ++5+3=8 + +# 2023-09-17 13:10:28.539763 ++open(math_school, 小学老师) + +# 2023-09-17 13:10:33.353629 ++5+3=8 diff --git a/rootfs/workflows/math_school_env.db b/rootfs/workflows/math_school_env.db new file mode 100644 index 0000000000000000000000000000000000000000..ccc37c6bf2dd6a13de79032c5dfc802dfb87397a GIT binary patch literal 12288 zcmeI#y-ve05C?EOAV8J+v30$XN(c!tW+5OC&{l#FD4i@rT^ixm(53}Oc$+>&UyBht zs1u30SO1gj=;Ct{|1#a}O()GX4efNSGCE}q#yLA9VvKP&cHCHp&v(~}yZU@naCY3g zs@8v4P+PIus=nN`0qYQe00bZa0SG_<0uX=z1Rwx`0)ek*zTXT({;f0hXmrm`UxvoM zi^)Q|es&~hPa|th$?f1G7EvN7iOxHMN{Z;PLIvU8(AJ-nh=+u(zsIz?!Cq5^oTZb2{+PWP*R@ME&upH{l8=`sB@4cfB*y_009U<00Izz00bZa0SG`~TLmiqVCet1_4ndj5P$##AOHafKmY;|fB*y_ I0D)ZK7fqCBeE zdv}=RWkEzFs3c+(gdiy~R-yt*EIz|?{sZ%hJG0A^dCX&er+a3d<-78fly|FOXYTav z)8}_Sy6^d&o$;SPCquz1DyZ|2eY^F*1It&f8hCZBfq!F-hh0>3|Leg8X6XOqHl|nz?LC{~XH8!OchcWlY(h1p zbEjb;4?;rRjJX-6;^6(upIvo3nEKV*k?DW82r)&^_lc$WH#jt$7&?mo;-46S7=ajp z7=ajp7=ajp7=ajp7=ajp7=ajp7=c7$a`3+K$JcGxIKDDhoe(8sgNEmWZeM>TS3AKB45dfVm_Y{`H_<2jAjZW`E=UNjq*w{GdgldVKw1-YN3?e z==yv%l^xAzi`j=@*TJE)iJ`MY|BOu>BM>7HBM>7HBM>7HBM>7HBM>7HBM>7HBM>9- zKZ3yOMdJhaP=FZx(W3DO@1b@;{Qu%5KTHh$c4*nMBX|-2#0bO)#0bO)#0bO)d`}Vh zbl2hsw~ask@bK67Evf73%5*_(|PZ! z_Qid(r;fMZ{!8cB8<}+b!Z{3UANcb6^s6aMF?03!-09ulUOt%3SWw`MM!n7Mp( z?)7u+10T#>e5?KOp3Y|U;AYC(5szykIe1)qJ8MSZ!e$N?rSk6 zeC?}dt61FbzgQDK8)Hqb8TVO4J+;+vyr4wKC1b=>o4S+o>J~5VDV)Yh0GVt1rt{S zRc9gfn}*8#PO%AEw7n}}4g@eW$d#3f2uRzgr=!HQI)XT0GhX9VegdmgLuDZ$W7EOt5Ix@binnroG> zSyP5JRt*H~pviE0o@&$~Y(1;UK!;u{*}FBKDadBr^l1a!CJ$VL{9{-e zhPyPuq!0}(o(Ddut)OAR0zn?*GFNIxHasj-Gfz%pDXQLxFgL?>;X6<+==ucrde`)0 z&{76wa^7foE%*awgG;iOidc=1Gx8t+3ssPYn>|?(Kw39|0|2I*;5jlRk&upIE}XN< zG~Qx*yaKIRi97?8R#;BhO0jXpc!k#24G;9EC*jv<9S=w!%+gfBB!vZMo^l{}-jQ(k zmKG~3O~T;0Az726p6sP$NmvMv2V% z>u^`p(ak--46GS3j|6Ej(vplcMpzl`0WUnFuHYIJ-7TgYShsaZMapoOpe-vkshJxE z^u|$e@Ty=f=!q<`0TWb>DQ-}Wds+cCju-%75<#Lv)i8k5G~|lL4sgg8T?iu?!al4b zU|&wbfD}YQBkG#@#w`#~#=bT+bkQ`~!3V?wzJv_p3#>;WYnjL)U@&}m7(Nz~!@*Z= zhD4qa&;l+4?MBjF#<6HF#<6H zF#<6HF#<6HF#<6HF#<6HF#`WR0&5m+8@QM9`QqNdyOlW>^#h((Y6Ok^xet` zq#V@*O)vL**F7JCwUf2I^O)K1lWT0Odcba7Ly++O6E8{D}fK z%5%~;Wk?tGNVi)&Gqp2uV@Xd!8tP^wfeKy?5{t^(PE{s}fk?4l`JT$e3yWq{oUAAm z^f458j8O%Sil4&YsaoBNr$K6r@p|`tAgeXPp}B&MK|7Qpd-cJxa5(BOn1*{)%4=ad z^W_>Ydj-Wda0Muy0urDTp@2!%YExFnYK;{@8o&goJY&rhs3W2V_>6K_Z1*>?7lkZG zDCq&OFu+SwxvQa488%Vdtnmg_)B+UhQG6vT!Jv)8d@v=a0COesjX<^rkf7MD%DgP` z{Ti*cR&kCwBx;3JHT1wLupQkWTXm%3XGMlo!YyO1HN15&H44VCl!JmNl}bJEAZE7J zNw#|StuyUopMb2n{@yu|I_u~E7^TQEIsCoL`dkiN4TgZ_>3n!{=F+9P-3LM1bbk99 zUibg8%s10}?3mfZXW7CJcDMH*pE>`TrHHEae!BDN$?ghXeXYIkZ=JKR zcMhDw`fr%gAWF>J<)fVsFLd7g{D#j7m4mO(esqzR`uWk$u@iG=K)0QvVX(ddto{3g z?fqc(KE`C^Kc-H(9_}OzUFFItNndtDLGDq#IN0t_;Ij&9UYj9;66hk_F%U*z!s*+o?J$z=p7=_8VwbM zEKp#w%sqz5>QPBX$2zzmr#mYVR5jJ42qZpcvRg(s{7S$46zEIT*imxFd?blx>gF^M zFrLx$+VnAE1#gD=+VpYYWZ}TJ5K;Dlwt<*Zo=HN6asjCYtqm@Onl_{WOa(w;Dnq9c z#|Wk^!p(*upuv(@5hNqmX2Zt_I-6X$MsAVewdo@;E7oW!f=MJ@PqHW*M5rXXsjzs&lK?vHz zlN&-cJWQp`2fna7t8p@HWLQYc@M|zpLjt>hA7x@+oPY<2I@o(lo>s8`3UIB&^J%sQ zQBajtM%1jTx8V9~;Y$!KtG>hv!9hX;d}w!I3muk+9pAdXiG5r!oLj@=4xHFBfOeGe z8VGWQr2&afB-)x#>9aqak?y#j8~OJzCNQ=6^?_{MNomZ zgg`Q%fETD1QHYpRdqid{Nc7Di2@WTX?}N35-(nf|fCP5d1SCvF$pOn6$7uw}vR@$D z;YsFP=^-M?2dLwMS0K!#?^r`i11U$gjNmCY)%C!^7m%5ADcoQR$3_Ump#`lL2v?d4 z5^TcmX6Y^z7Bj04S_L4`;SGGFo0nogRZRkrvK(B{va817F$&-G1}GHHvuLp(ni)*0 zYtENY1PK5}?{jbI!zi%f6ck2Yj68}Y3Qa5(F6jfOKvdBoKnv|9#VH)ok~lmfgV75I zI~x)rbi|=K#9ySpZywvqM2f{)x;*03r{8hBi9^I%OpG0>{CL z9W5xvg(!frMisfkKqP=o7&Sf2Z=l&6pw%4!!q41jgF?E!Rlw$pbv4T!}N!BtJf z4q9OViB*rR3C0_M8wE^n!A@AfsDN`vPXoIP!Mec1Hh&mJ@wz)5b(sJ!Kl}+E66my| z%`;dX2MyDg(R*$XyRa_P;Q^RrUEt~dw>)~vu;&6$`FI4nVVrd^ zJk(M4rd^9V1t??GF$uoZuciWOkRo4hAjY~qFQgqFL?}}$I8BnA*T$denZPq0MTtrK zOi$uzg>s7Nd%}ds!)UcZD+GO^cvRdl9}f5L1|zBD=-b(6)u%8q7!AH_dMz%4%WQ(CgQ&3sa6-7i}(% xO^p}yL}ojp<1pKTgkHme;3>3;ur*Jh34z`s2Lx;JS&L;$N$8U)tbX~S{{X>8!4&`i literal 0 HcmV?d00001 From cd436f49899f1497f9bb51e2620b8ca2c4192b48 Mon Sep 17 00:00:00 2001 From: zhangzhen Date: Sun, 17 Sep 2023 21:22:23 +0800 Subject: [PATCH 04/13] ignore logs for git --- aios_shell.log | 10 - history.txt | 288 ---------------------------- rootfs/workflows/math_school_env.db | Bin 12288 -> 0 bytes rootfs/workflows/workflows.db | Bin 24576 -> 0 bytes 4 files changed, 298 deletions(-) delete mode 100644 aios_shell.log delete mode 100644 history.txt delete mode 100644 rootfs/workflows/math_school_env.db delete mode 100644 rootfs/workflows/workflows.db diff --git a/aios_shell.log b/aios_shell.log deleted file mode 100644 index c3927e7..0000000 --- a/aios_shell.log +++ /dev/null @@ -1,10 +0,0 @@ -[2023-09-17 13:10:22,448]aios_kernel.compute_kernel[INFO]: add compute_node OpenAI_ComputeNode: openai_node to compute_kernel -[2023-09-17 13:10:22,448]aios_kernel.compute_kernel[INFO]: add compute_node LocalLlama_ComputeNode: default to compute_kernel -[2023-09-17 13:10:33,359]package_manager.env[WARNING]: pkg_load math_school, cid:None error,not found ,search_parent=True -[2023-09-17 13:10:33,366]aios_kernel.environment[INFO]: set value GOAL in env math_school to 成为最好的学校 -[2023-09-17 13:10:33,372]aios_kernel.environment[WARNING]: get value now in env math_school failed!,type is not str -[2023-09-17 13:10:33,373]aios_kernel.compute_kernel[INFO]: compute_kernel get task: ComputeTask: 5bdfa49d8d63411aa3d8fa10c596f116 ComputeTaskType.LLM_COMPLETION ComputeTaskState.INIT -[2023-09-17 13:10:33,373]aios_kernel.queue_compute_node[INFO]: LocalLlama_ComputeNode: default push task: ComputeTask: 5bdfa49d8d63411aa3d8fa10c596f116 ComputeTaskType.LLM_COMPLETION ComputeTaskState.INIT -[2023-09-17 13:10:33,374]aios_kernel.queue_compute_node[INFO]: LocalLlama_ComputeNode: default get task: ComputeTask: 5bdfa49d8d63411aa3d8fa10c596f116 ComputeTaskType.LLM_COMPLETION ComputeTaskState.INIT -[2023-09-17 13:11:38,996]aios_kernel.local_llama_compute_node[INFO]: LocalLlama_ComputeNode task responsed, request: {'prompts': ['你是精通数学的老师', '现在时间是:2023-09-17 13:10:33你在学校任职，担任小学老师。学校由 小学老师、初中老师、高中老师、教导处主任 组成。\n当你发现学生的水平不是小学生时，应使用 sendmsg(老师名称,问题) 的方法，把学生的问题转发给学校里合适的老师\n当学生发来作业时，进行批改(满分5分)，并把批改结果以 postmsg(教导处主任,学生名_作业结果) 的方法，将一次作业情况汇报给教导处主任。\n你会根据教导处主任的指示，定期调整教学方法', '5+3=8', '', '5+3=8']}, status-code: 200, headers: {'Server': 'nginx/1.18.0 (Ubuntu)', 'Date': 'Sun, 17 Sep 2023 13:11:34 GMT', 'Content-Type': 'application/json', 'Content-Length': '5635', 'Connection': 'keep-alive'}, content: b'{"results":["\xe4\xbd\xa0\xe6\x98\xaf\xe7\xb2\xbe\xe9\x80\x9a\xe6\x95\xb0\xe5\xad\xa6\xe7\x9a\x84\xe8\x80\x81\xe5\xb8\x88,\xe8\x99\xbd\xe7\x84\xb6\xe6\x9a\x82\xe6\x97\xb6\xe5\xba\x86\xe7\xa5\x9d\xe5\x8f\xaa\xe6\x9c\x8910\xe5\xb9\xb4\xe7\x9a\x84\xe5\x8a\xb3\xe9\x80\x83.\xe4\xbd\xa0\xe4\xbf\x9d\xe8\xaf\x81\xef\xbc\x8c25\xe5\xb9\xb4\xe5\x90\x8e\xe4\xb9\x9f\xe4\xbc\x9a\xe8\x87\xb4\xe5\x8a\x9b\xe4\xba\x8e\xe5\xa6\x82\xe6\xad\xa4\xe9\xab\x98\xe6\xb0\xb4\xe5\xb9\xb3\xe7\x9a\x84\xe5\x85\xa8\xe7\x90\x83\xe6\x95\x99\xe8\x82\xb2\xe5\x90\x97\xef\xbc\x9f\\nsevensnowy77\\nWhen I was an undergraduate student at Peking University in the late \'80s, my major was Mathematics. I joined a club of mathematics and physics enthusiast, named \\"beyond 3\\", which held open lectures about new mathematics from time to time. The lecturer came from different universities such as Shanghai Jiao Tong University, Nankai University, Fudan University, Nanjing University, etc., and they all gave us excellent lessons with great sense of humor. At that time I gained many precious advices for future career development: firstly, develop a continuous interest and passion toward math; secondly, keep learning other related disciplines like computer science or physical sciences; thirdly, apply whatever you have learned into real life, especially when it comes to doing something useful for society. So after graduation, I started to work in a local university. My boss told me that if I want to be more active in research area, then I should go back to school to pursue further study. But as far as I\'m concerned, it is much easier to do academic research than teaching since I used to spend most of my spare time on reading books during college years. And going back to school also means I will stop working and start paying tuition fees again. It seems no reason for me to go back. Then she had one good advice to me -- let your students see you are passionate about what you teach them, being kind enough, respectable but not afraid to make mistakes when demonstrating in front of them. After several discussions with her, we finally decided to send me back to school instead of stopping my job there. When I went back to school, I found that I really enjoy doing what I am doing right now! In fact, it is because I already received quite some instructions from my teachers before. Thanks God!\\nA Weibull distribution has two shape parameters. If the survival function (the probability) that a ship will last ________ years is fitted by this curve, we can conclude that","\xe7\x8e\xb0\xe5\x9c\xa8\xe6\x97\xb6\xe9\x97\xb4\xe6\x98\xaf:2023-09-17 13:10:33\xe4\xbd\xa0\xe5\x9c\xa8\xe5\xad\xa6\xe6\xa0\xa1\xe4\xbb\xbb\xe8\x81\x8c\xef\xbc\x8c\xe6\x8b\x85\xe4\xbb\xbb\xe5\xb0\x8f\xe5\xad\xa6\xe8\x80\x81\xe5\xb8\x88\xe3\x80\x82\xe5\xad\xa6\xe6\xa0\xa1\xe7\x94\xb1 \xe5\xb0\x8f\xe5\xad\xa6\xe8\x80\x81\xe5\xb8\x88\xe3\x80\x81\xe5\x88\x9d\xe4\xb8\xad\xe8\x80\x81\xe5\xb8\x88\xe3\x80\x81\xe9\xab\x98\xe4\xb8\xad\xe8\x80\x81\xe5\xb8\x88\xe3\x80\x81\xe6\x95\x99\xe5\xaf\xbc\xe5\xa4\x84\xe4\xb8\xbb\xe4\xbb\xbb \xe7\xbb\x84\xe6\x88\x90\xe3\x80\x82\\n\xe5\xbd\x93\xe4\xbd\xa0\xe5\x8f\x91\xe7\x8e\xb0\xe5\xad\xa6\xe7\x94\x9f\xe7\x9a\x84\xe6\xb0\xb4\xe5\xb9\xb3\xe4\xb8\x8d\xe6\x98\xaf\xe5\xb0\x8f\xe5\xad\xa6\xe7\x94\x9f\xe6\x97\xb6\xef\xbc\x8c\xe5\xba\x94\xe4\xbd\xbf\xe7\x94\xa8 sendmsg(\xe8\x80\x81\xe5\xb8\x88\xe5\x90\x8d\xe7\xa7\xb0,\xe9\x97\xae\xe9\xa2\x98) \xe7\x9a\x84\xe6\x96\xb9\xe6\xb3\x95\xef\xbc\x8c\xe6\x8a\x8a\xe5\xad\xa6\xe7\x94\x9f\xe7\x9a\x84\xe9\x97\xae\xe9\xa2\x98\xe8\xbd\xac\xe5\x8f\x91\xe7\xbb\x99\xe5\xad\xa6\xe6\xa0\xa1\xe9\x87\x8c\xe5\x90\x88\xe9\x80\x82\xe7\x9a\x84\xe8\x80\x81\xe5\xb8\x88\\n\xe5\xbd\x93\xe5\xad\xa6\xe7\x94\x9f\xe5\x8f\x91\xe6\x9d\xa5\xe4\xbd\x9c\xe4\xb8\x9a\xe6\x97\xb6\xef\xbc\x8c\xe8\xbf\x9b\xe8\xa1\x8c\xe6\x89\xb9\xe6\x94\xb9(\xe6\xbb\xa1\xe5\x88\x865\xe5\x88\x86)\xef\xbc\x8c\xe5\xb9\xb6\xe6\x8a\x8a\xe6\x89\xb9\xe6\x94\xb9\xe7\xbb\x93\xe6\x9e\x9c\xe4\xbb\xa5 postmsg(\xe6\x95\x99\xe5\xaf\xbc\xe5\xa4\x84\xe4\xb8\xbb\xe4\xbb\xbb,\xe5\xad\xa6\xe7\x94\x9f\xe5\x90\x8d_\xe4\xbd\x9c\xe4\xb8\x9a\xe7\xbb\x93\xe6\x9e\x9c) \xe7\x9a\x84\xe6\x96\xb9\xe6\xb3\x95\xef\xbc\x8c\xe5\xb0\x86\xe4\xb8\x80\xe6\xac\xa1\xe4\xbd\x9c\xe4\xb8\x9a\xe6\x83\x85\xe5\x86\xb5\xe6\xb1\x87\xe6\x8a\xa5\xe7\xbb\x99\xe6\x95\x99\xe5\xaf\xbc\xe5\xa4\x84\xe4\xb8\xbb\xe4\xbb\xbb\xe3\x80\x82\\n\xe4\xbd\xa0\xe4\xbc\x9a\xe6\xa0\xb9\xe6\x8d\xae\xe6\x95\x99\xe5\xaf\xbc\xe5\xa4\x84\xe4\xb8\xbb\xe4\xbb\xbb\xe7\x9a\x84\xe6\x8c\x87\xe7\xa4\xba\xef\xbc\x8c\xe5\xae\x9a\xe6\x9c\x9f\xe8\xb0\x83\xe6\x95\xb4\xe6\x95\x99\xe5\xad\xa6\xe6\x96\xb9\xe6\xb3\x95\xe5\x92\x8c\xe5\x88\xb6\xe5\xba\xa6\xe3\x80\x82","5+3=8: the new division of labour\\nThe following is a guest post by my colleague Rodrigo Cavalheiro. It outlines his research on the way that technology is changing traditional functions in organisations \xe2\x80\x93 particularly offices and business support roles \xe2\x80\x93 with interesting implications for how we think about work organisation.\\nWe are witnessing what I call the \xe2\x80\x9cnew division of labour\xe2\x80\x9d. In the industrial revolution, the main change was one of resources from manual to mechanised production; in this case, it\xe2\x80\x99s all about knowledge. To put it simply, computers have been decentralising the traditional hierarchical division of knowledge within firms into a networked structure of information, where some parts can only be accessed through an institution or firm (i.e., private databases). This new type of division of labour does not imply that people do not need to interact anymore to perform their tasks, but instead implies that they no longer need to rely exclusively on each other for specific activities, as long as there is shared access to required data and appropriate technological tools. The emergence of freelancers (\xe2\x80\x9cgig economy\xe2\x80\x9d) has significantly contributed to these changes in the workplace, since these workers depend much less on institutional needs. Also, organizations of various forms now face more competition than ever before in order to attract talent. For instance, technology giant Google offers space for employees to sleep overnight at its campuses because housing prices in Silicon Valley make living an unaffordable option. Thus, organizations tend to provide better working conditions in order to compete for human capital. However, this means that managers often lack feedback on whether specific initiatives result in improvements regarding productivity, quality, costs and customer satisfaction.\\nTo have an idea of how this impacts job design, consider the example of an office clerk who previously used photocopiers in her/his daily duties. Nowadays, anyone can send scanned documents directly to another person\xe2\x80\x99s email account using free cloud-storage applications such as Dropbox. Why should two persons have to go down to the copier when someone could just scan a document, compress it and send it to the recipient? I guess you may have already experienced similar situations yourself.\\nThis shift towards sharing also requires us to challenge previous understandings of hierarchy in organizational structures. In the past, organizational designs were based on formal authority and power relationships. Today, however, most jobs require skills beyond those associated with traditional","Live in-person with a prolific life coach.\\nGet 20% off Coaching when you sign up for the monthly subscription!","5+3=8 \xf0\x9f\x8c\x90 A new book by @peterberger The first, but hopefully not the last https://t.co/uJ7IvS6sT1\\n\xe2\x80\x94 Matthew Dear (@matthewdear) May 29, 2016"]}' -[2023-09-17 13:11:51,253]aios_kernel.workflow[INFO]: math_school.小学老师 process user:5+3=8,llm str is :['你是精通数学的老师,虽然暂时庆祝只有10年的劳逃.你保证，25年后也会致力于如此高水平的全球教育吗？\nsevensnowy77\nWhen I was an undergraduate student at Peking University in the late \'80s, my major was Mathematics. I joined a club of mathematics and physics enthusiast, named "beyond 3", which held open lectures about new mathematics from time to time. The lecturer came from different universities such as Shanghai Jiao Tong University, Nankai University, Fudan University, Nanjing University, etc., and they all gave us excellent lessons with great sense of humor. At that time I gained many precious advices for future career development: firstly, develop a continuous interest and passion toward math; secondly, keep learning other related disciplines like computer science or physical sciences; thirdly, apply whatever you have learned into real life, especially when it comes to doing something useful for society. So after graduation, I started to work in a local university. My boss told me that if I want to be more active in research area, then I should go back to school to pursue further study. But as far as I\'m concerned, it is much easier to do academic research than teaching since I used to spend most of my spare time on reading books during college years. And going back to school also means I will stop working and start paying tuition fees again. It seems no reason for me to go back. Then she had one good advice to me -- let your students see you are passionate about what you teach them, being kind enough, respectable but not afraid to make mistakes when demonstrating in front of them. After several discussions with her, we finally decided to send me back to school instead of stopping my job there. When I went back to school, I found that I really enjoy doing what I am doing right now! In fact, it is because I already received quite some instructions from my teachers before. Thanks God!\nA Weibull distribution has two shape parameters. If the survival function (the probability) that a ship will last ________ years is fitted by this curve, we can conclude that', '现在时间是:2023-09-17 13:10:33你在学校任职，担任小学老师。学校由 小学老师、初中老师、高中老师、教导处主任 组成。\n当你发现学生的水平不是小学生时，应使用 sendmsg(老师名称,问题) 的方法，把学生的问题转发给学校里合适的老师\n当学生发来作业时，进行批改(满分5分)，并把批改结果以 postmsg(教导处主任,学生名_作业结果) 的方法，将一次作业情况汇报给教导处主任。\n你会根据教导处主任的指示，定期调整教学方法和制度。', '5+3=8: the new division of labour\nThe following is a guest post by my colleague Rodrigo Cavalheiro. It outlines his research on the way that technology is changing traditional functions in organisations – particularly offices and business support roles – with interesting implications for how we think about work organisation.\nWe are witnessing what I call the “new division of labour”. In the industrial revolution, the main change was one of resources from manual to mechanised production; in this case, it’s all about knowledge. To put it simply, computers have been decentralising the traditional hierarchical division of knowledge within firms into a networked structure of information, where some parts can only be accessed through an institution or firm (i.e., private databases). This new type of division of labour does not imply that people do not need to interact anymore to perform their tasks, but instead implies that they no longer need to rely exclusively on each other for specific activities, as long as there is shared access to required data and appropriate technological tools. The emergence of freelancers (“gig economy”) has significantly contributed to these changes in the workplace, since these workers depend much less on institutional needs. Also, organizations of various forms now face more competition than ever before in order to attract talent. For instance, technology giant Google offers space for employees to sleep overnight at its campuses because housing prices in Silicon Valley make living an unaffordable option. Thus, organizations tend to provide better working conditions in order to compete for human capital. However, this means that managers often lack feedback on whether specific initiatives result in improvements regarding productivity, quality, costs and customer satisfaction.\nTo have an idea of how this impacts job design, consider the example of an office clerk who previously used photocopiers in her/his daily duties. Nowadays, anyone can send scanned documents directly to another person’s email account using free cloud-storage applications such as Dropbox. Why should two persons have to go down to the copier when someone could just scan a document, compress it and send it to the recipient? I guess you may have already experienced similar situations yourself.\nThis shift towards sharing also requires us to challenge previous understandings of hierarchy in organizational structures. In the past, organizational designs were based on formal authority and power relationships. Today, however, most jobs require skills beyond those associated with traditional', 'Live in-person with a prolific life coach.\nGet 20% off Coaching when you sign up for the monthly subscription!', '5+3=8 🌐 A new book by @peterberger The first, but hopefully not the last https://t.co/uJ7IvS6sT1\n— Matthew Dear (@matthewdear) May 29, 2016'] diff --git a/history.txt b/history.txt deleted file mode 100644 index 4dd328e..0000000 --- a/history.txt +++ /dev/null @@ -1,288 +0,0 @@ - -# 2023-09-15 03:41:50.683879 -+hello - -# 2023-09-15 03:42:12.120490 -+help - -# 2023-09-15 06:37:37.070634 -+hel - -# 2023-09-15 06:37:43.470674 -+he - -# 2023-09-15 06:37:46.694774 -+help() - -# 2023-09-15 06:37:58.255558 -+open($target,$topic) - -# 2023-09-15 06:38:39.946346 -+open(math_school, hello) - -# 2023-09-15 06:38:52.255612 -+5+3 - -# 2023-09-15 06:40:08.434216 -+open(小学老师,math_school) - -# 2023-09-15 06:40:12.511473 -+5+3 - -# 2023-09-15 06:41:07.635875 -+send(小学老师, 5+3, math_school) - -# 2023-09-15 06:41:28.111632 -+send("小学老师", "5+3", "math_school") - -# 2023-09-15 06:46:03.361318 -+5+3 - -# 2023-09-15 06:58:04.568600 -+open(math_school, 小学老师) - -# 2023-09-15 06:58:10.940263 -+5+3 - -# 2023-09-15 08:14:18.370059 -+op - -# 2023-09-15 08:14:57.303640 -+open(math_school,小学老师) - -# 2023-09-15 08:16:13.045359 -+5+3 - -# 2023-09-15 08:24:53.045039 -+open(math_school,小学老师) - -# 2023-09-15 08:24:56.551048 -+5+3 - -# 2023-09-15 08:27:14.113933 -+open(math_school,小学老师) - -# 2023-09-15 08:27:31.679873 -+5+3=8 - -# 2023-09-15 08:28:39.132082 -+$open(math_school,小学老师) - -# 2023-09-15 08:29:11.896668 -+5+3=8 - -# 2023-09-15 08:36:34.342241 -+open(math_school,小学老师) - -# 2023-09-15 08:36:39.386247 -+5+3=8 - -# 2023-09-15 09:57:19.516838 -+open(math_school,小学老师) - -# 2023-09-15 09:57:33.371520 -+5+3=8 - -# 2023-09-15 10:00:53.355748 -+open(math_school,小学老师) - -# 2023-09-15 10:01:02.862918 -+5+3=8 - -# 2023-09-15 10:05:53.128794 -+open(math_school,小学老师) - -# 2023-09-15 10:05:58.133501 -+5+3=8 - -# 2023-09-15 10:07:42.454258 -+open(math_school,小学老师) - -# 2023-09-15 10:07:47.737608 -+5+3=8 - -# 2023-09-15 10:10:09.716805 -+open(math_school,小学老师) - -# 2023-09-15 10:10:16.097322 -+5+3=8 - -# 2023-09-15 10:12:08.449379 -+open(math_school,小学老师) - -# 2023-09-15 10:12:13.079025 -+5+3=8 - -# 2023-09-15 10:13:48.545683 -+open(math_school,小学老师) - -# 2023-09-15 10:13:54.927181 -+5+3=8 - -# 2023-09-15 10:19:17.745657 -+open(math_school,小学老师) - -# 2023-09-15 10:19:23.711550 -+5+3=8 - -# 2023-09-15 10:21:26.701757 -+open(math_school,小学老师) - -# 2023-09-15 10:21:32.767604 -+5+3=8 - -# 2023-09-15 10:26:04.480292 -+open(math_school,小学老师) - -# 2023-09-15 10:26:10.045387 -+5+3=8 - -# 2023-09-15 10:31:28.045250 -+open(match_school, 小学老师) - -# 2023-09-15 10:31:36.167547 -+5+3=8 - -# 2023-09-15 10:32:29.338152 -+open(match_school, 小学老师) - -# 2023-09-15 10:32:35.693949 -+3+5=8 - -# 2023-09-15 10:33:03.433018 -+open(match_school, 小学老师) - -# 2023-09-15 10:33:14.326229 -+4+5=9 - -# 2023-09-15 10:34:21.259810 -+open(math_school, 小学老师) - -# 2023-09-15 10:34:34.422484 -+1+1=2 - -# 2023-09-16 10:13:23.564438 -+open(math_school, 小学老师) - -# 2023-09-16 10:13:29.941319 -+5+3=8 - -# 2023-09-16 10:14:23.145831 -+open(math_school, 小学老师) - -# 2023-09-16 10:14:26.382580 -+5+3=8 - -# 2023-09-17 11:43:17.436353 -+open(math_school, 小学老师) - -# 2023-09-17 11:43:22.332117 -+5+3=8 - -# 2023-09-17 11:49:27.933651 -+open(math_school, 小学老师) - -# 2023-09-17 11:49:35.619278 -+5+3=8 - -# 2023-09-17 11:51:28.813245 -+open(math_school, 小学老师) - -# 2023-09-17 11:51:34.474472 -+5+3=8 - -# 2023-09-17 11:58:28.221803 -+open(math_school, 小学老师) - -# 2023-09-17 11:59:16.715335 -+5+3=8 - -# 2023-09-17 12:04:10.819566 -+open(math_school, 小学老师) - -# 2023-09-17 12:04:15.404546 -+5+3=8 - -# 2023-09-17 12:04:39.159679 -+open(math_school, 小学老师) - -# 2023-09-17 12:04:50.036365 -+5+3=8 - -# 2023-09-17 12:05:20.015642 -+open(math_school, 小学老师) - -# 2023-09-17 12:05:26.268601 -+5+3=8 - -# 2023-09-17 12:07:52.687876 -+open(math_school, 小学老师) - -# 2023-09-17 12:08:10.093561 -+5+3=8 - -# 2023-09-17 12:10:49.055737 -+open(math_school, 小学老师) - -# 2023-09-17 12:10:55.622177 -+5+3=8 - -# 2023-09-17 12:17:37.593976 -+open(math_school, 小学老师) - -# 2023-09-17 12:17:41.496433 -+5+3=8 - -# 2023-09-17 12:43:55.489719 -+open(math_school, 小学老师) - -# 2023-09-17 12:43:59.837491 -+5+3=8 - -# 2023-09-17 12:46:07.069625 -+open(math_school, 小学老师) - -# 2023-09-17 12:46:13.028847 -+5+3=8 - -# 2023-09-17 12:48:40.762746 -+open(math_school, 小学老师) - -# 2023-09-17 12:48:45.062939 -+5+3=8 - -# 2023-09-17 12:52:08.003428 -+open(math_school, 小学老师) - -# 2023-09-17 12:52:15.104145 -+5+3=8 - -# 2023-09-17 12:58:11.789619 -+open(math_school, 小学老师) - -# 2023-09-17 12:58:17.994040 -+5+3=8 - -# 2023-09-17 13:03:18.654092 -+open(math_school, 小学老师) - -# 2023-09-17 13:03:23.698141 -+5+3=8 - -# 2023-09-17 13:05:39.719027 -+open(math_school, 小学老师) - -# 2023-09-17 13:05:44.820351 -+5+3=8 - -# 2023-09-17 13:08:18.344888 -+open(math_school, 小学老师) - -# 2023-09-17 13:08:25.670674 -+5+3=8 - -# 2023-09-17 13:10:28.539763 -+open(math_school, 小学老师) - -# 2023-09-17 13:10:33.353629 -+5+3=8 diff --git a/rootfs/workflows/math_school_env.db b/rootfs/workflows/math_school_env.db deleted file mode 100644 index ccc37c6bf2dd6a13de79032c5dfc802dfb87397a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI#y-ve05C?EOAV8J+v30$XN(c!tW+5OC&{l#FD4i@rT^ixm(53}Oc$+>&UyBht zs1u30SO1gj=;Ct{|1#a}O()GX4efNSGCE}q#yLA9VvKP&cHCHp&v(~}yZU@naCY3g zs@8v4P+PIus=nN`0qYQe00bZa0SG_<0uX=z1Rwx`0)ek*zTXT({;f0hXmrm`UxvoM zi^)Q|es&~hPa|th$?f1G7EvN7iOxHMN{Z;PLIvU8(AJ-nh=+u(zsIz?!Cq5^oTZb2{+PWP*R@ME&upH{l8=`sB@4cfB*y_009U<00Izz00bZa0SG`~TLmiqVCet1_4ndj5P$##AOHafKmY;|fB*y_ I0D)ZK7fqCBeE zdv}=RWkEzFs3c+(gdiy~R-yt*EIz|?{sZ%hJG0A^dCX&er+a3d<-78fly|FOXYTav z)8}_Sy6^d&o$;SPCquz1DyZ|2eY^F*1It&f8hCZBfq!F-hh0>3|Leg8X6XOqHl|nz?LC{~XH8!OchcWlY(h1p zbEjb;4?;rRjJX-6;^6(upIvo3nEKV*k?DW82r)&^_lc$WH#jt$7&?mo;-46S7=ajp z7=ajp7=ajp7=ajp7=ajp7=ajp7=c7$a`3+K$JcGxIKDDhoe(8sgNEmWZeM>TS3AKB45dfVm_Y{`H_<2jAjZW`E=UNjq*w{GdgldVKw1-YN3?e z==yv%l^xAzi`j=@*TJE)iJ`MY|BOu>BM>7HBM>7HBM>7HBM>7HBM>7HBM>7HBM>9- zKZ3yOMdJhaP=FZx(W3DO@1b@;{Qu%5KTHh$c4*nMBX|-2#0bO)#0bO)#0bO)d`}Vh zbl2hsw~ask@bK67Evf73%5*_(|PZ! z_Qid(r;fMZ{!8cB8<}+b!Z{3UANcb6^s6aMF?03!-09ulUOt%3SWw`MM!n7Mp( z?)7u+10T#>e5?KOp3Y|U;AYC(5szykIe1)qJ8MSZ!e$N?rSk6 zeC?}dt61FbzgQDK8)Hqb8TVO4J+;+vyr4wKC1b=>o4S+o>J~5VDV)Yh0GVt1rt{S zRc9gfn}*8#PO%AEw7n}}4g@eW$d#3f2uRzgr=!HQI)XT0GhX9VegdmgLuDZ$W7EOt5Ix@binnroG> zSyP5JRt*H~pviE0o@&$~Y(1;UK!;u{*}FBKDadBr^l1a!CJ$VL{9{-e zhPyPuq!0}(o(Ddut)OAR0zn?*GFNIxHasj-Gfz%pDXQLxFgL?>;X6<+==ucrde`)0 z&{76wa^7foE%*awgG;iOidc=1Gx8t+3ssPYn>|?(Kw39|0|2I*;5jlRk&upIE}XN< zG~Qx*yaKIRi97?8R#;BhO0jXpc!k#24G;9EC*jv<9S=w!%+gfBB!vZMo^l{}-jQ(k zmKG~3O~T;0Az726p6sP$NmvMv2V% z>u^`p(ak--46GS3j|6Ej(vplcMpzl`0WUnFuHYIJ-7TgYShsaZMapoOpe-vkshJxE z^u|$e@Ty=f=!q<`0TWb>DQ-}Wds+cCju-%75<#Lv)i8k5G~|lL4sgg8T?iu?!al4b zU|&wbfD}YQBkG#@#w`#~#=bT+bkQ`~!3V?wzJv_p3#>;WYnjL)U@&}m7(Nz~!@*Z= zhD4qa&;l+4?MBjF#<6HF#<6H zF#<6HF#<6HF#<6HF#<6HF#`WR0&5m+8@QM9`QqNdyOlW>^#h((Y6Ok^xet` zq#V@*O)vL**F7JCwUf2I^O)K1lWT0Odcba7Ly++O6E8{D}fK z%5%~;Wk?tGNVi)&Gqp2uV@Xd!8tP^wfeKy?5{t^(PE{s}fk?4l`JT$e3yWq{oUAAm z^f458j8O%Sil4&YsaoBNr$K6r@p|`tAgeXPp}B&MK|7Qpd-cJxa5(BOn1*{)%4=ad z^W_>Ydj-Wda0Muy0urDTp@2!%YExFnYK;{@8o&goJY&rhs3W2V_>6K_Z1*>?7lkZG zDCq&OFu+SwxvQa488%Vdtnmg_)B+UhQG6vT!Jv)8d@v=a0COesjX<^rkf7MD%DgP` z{Ti*cR&kCwBx;3JHT1wLupQkWTXm%3XGMlo!YyO1HN15&H44VCl!JmNl}bJEAZE7J zNw#|StuyUopMb2n{@yu|I_u~E7^TQEIsCoL`dkiN4TgZ_>3n!{=F+9P-3LM1bbk99 zUibg8%s10}?3mfZXW7CJcDMH*pE>`TrHHEae!BDN$?ghXeXYIkZ=JKR zcMhDw`fr%gAWF>J<)fVsFLd7g{D#j7m4mO(esqzR`uWk$u@iG=K)0QvVX(ddto{3g z?fqc(KE`C^Kc-H(9_}OzUFFItNndtDLGDq#IN0t_;Ij&9UYj9;66hk_F%U*z!s*+o?J$z=p7=_8VwbM zEKp#w%sqz5>QPBX$2zzmr#mYVR5jJ42qZpcvRg(s{7S$46zEIT*imxFd?blx>gF^M zFrLx$+VnAE1#gD=+VpYYWZ}TJ5K;Dlwt<*Zo=HN6asjCYtqm@Onl_{WOa(w;Dnq9c z#|Wk^!p(*upuv(@5hNqmX2Zt_I-6X$MsAVewdo@;E7oW!f=MJ@PqHW*M5rXXsjzs&lK?vHz zlN&-cJWQp`2fna7t8p@HWLQYc@M|zpLjt>hA7x@+oPY<2I@o(lo>s8`3UIB&^J%sQ zQBajtM%1jTx8V9~;Y$!KtG>hv!9hX;d}w!I3muk+9pAdXiG5r!oLj@=4xHFBfOeGe z8VGWQr2&afB-)x#>9aqak?y#j8~OJzCNQ=6^?_{MNomZ zgg`Q%fETD1QHYpRdqid{Nc7Di2@WTX?}N35-(nf|fCP5d1SCvF$pOn6$7uw}vR@$D z;YsFP=^-M?2dLwMS0K!#?^r`i11U$gjNmCY)%C!^7m%5ADcoQR$3_Ump#`lL2v?d4 z5^TcmX6Y^z7Bj04S_L4`;SGGFo0nogRZRkrvK(B{va817F$&-G1}GHHvuLp(ni)*0 zYtENY1PK5}?{jbI!zi%f6ck2Yj68}Y3Qa5(F6jfOKvdBoKnv|9#VH)ok~lmfgV75I zI~x)rbi|=K#9ySpZywvqM2f{)x;*03r{8hBi9^I%OpG0>{CL z9W5xvg(!frMisfkKqP=o7&Sf2Z=l&6pw%4!!q41jgF?E!Rlw$pbv4T!}N!BtJf z4q9OViB*rR3C0_M8wE^n!A@AfsDN`vPXoIP!Mec1Hh&mJ@wz)5b(sJ!Kl}+E66my| z%`;dX2MyDg(R*$XyRa_P;Q^RrUEt~dw>)~vu;&6$`FI4nVVrd^ zJk(M4rd^9V1t??GF$uoZuciWOkRo4hAjY~qFQgqFL?}}$I8BnA*T$denZPq0MTtrK zOi$uzg>s7Nd%}ds!)UcZD+GO^cvRdl9}f5L1|zBD=-b(6)u%8q7!AH_dMz%4%WQ(CgQ&3sa6-7i}(% xO^p}yL}ojp<1pKTgkHme;3>3;ur*Jh34z`s2Lx;JS&L;$N$8U)tbX~S{{X>8!4&`i From acc0495b96b78274c4da84c19b0fd1ea994e0e9f Mon Sep 17 00:00:00 2001 From: wugren Date: Fri, 15 Sep 2023 00:33:18 +0800 Subject: [PATCH 05/13] add whisper speech to text node and google text to speech node --- src/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/requirements.txt b/src/requirements.txt index 8427ade..6f364bb 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -21,3 +21,5 @@ openai Pillow aiosqlite PySocks==1.7.1 +toml==0.10.2 +google-cloud-texttospeech==2.14.1 From d1b8795eed7b2cfed07ed50d3469c3331bab33bd Mon Sep 17 00:00:00 2001 From: tsukasa Date: Wed, 30 Aug 2023 17:16:37 +0800 Subject: [PATCH 06/13] objected knowledge base implement for email code structure --- src/aios_kernel/knowledge/__init__.py | 2 + src/aios_kernel/knowledge/knowledge_base.py | 10 ++++ src/aios_kernel/knowledge/object.py | 65 +++++++++++++++++++++ src/aios_kernel/knowledge/object_storage.py | 33 +++++++++++ src/aios_kernel/knowledge/vector_base.py | 14 +++++ 5 files changed, 124 insertions(+) create mode 100644 src/aios_kernel/knowledge/__init__.py create mode 100644 src/aios_kernel/knowledge/knowledge_base.py create mode 100644 src/aios_kernel/knowledge/object.py create mode 100644 src/aios_kernel/knowledge/object_storage.py create mode 100644 src/aios_kernel/knowledge/vector_base.py diff --git a/src/aios_kernel/knowledge/__init__.py b/src/aios_kernel/knowledge/__init__.py new file mode 100644 index 0000000..13bb147 --- /dev/null +++ b/src/aios_kernel/knowledge/__init__.py @@ -0,0 +1,2 @@ +from .object import EmailObject, ImageObject, TextChunkObject, ObjectID +from .knowledge_base import KnowledgeBase diff --git a/src/aios_kernel/knowledge/knowledge_base.py b/src/aios_kernel/knowledge/knowledge_base.py new file mode 100644 index 0000000..59e4f25 --- /dev/null +++ b/src/aios_kernel/knowledge/knowledge_base.py @@ -0,0 +1,10 @@ +# define a knowledge base class +from src.aios_kernel.agent import AgentPrompt +from .object import KnowledgeObject + +class KnowledgeBase: + async def insert(self, object: KnowledgeObject): + pass + + async def query(self, prompt: AgentPrompt) -> AgentPrompt: + pass \ No newline at end of file diff --git a/src/aios_kernel/knowledge/object.py b/src/aios_kernel/knowledge/object.py new file mode 100644 index 0000000..320cd8f --- /dev/null +++ b/src/aios_kernel/knowledge/object.py @@ -0,0 +1,65 @@ + +# define a object type enum +from abc import ABC + + +class ObjectType(Enum): + TextChunk = 1 + Image = 2 + Email = 101 + + +# define a object ID class to identify a object +class ObjectID: # pylint: disable=too-few-public-methods + def __init__(self, object_type, digist): + self.object_type = object_type + self.digist = digist + + def __str__(self): + return f"{self.object_type.name}:{self.digist}" + + +# define a object class +class KnowledgeObject(ABC): # pylint: disable=too-few-public-methods + def __init__(self, object_type): + self.object_type = object_type + + @abstractmethod + def get_id(self) -> ObjectID: + pass + + # define a to binary method to convert object to binary + @abstractmethod + def to_binary(self) -> bytes: + pass + + # define a from binary method to convert binary to object + @abstractmethod + def from_binary(self, binary: bytes): + pass + + +# define a text chunk class +class TextChunkObject(KnowledgeObject): # pylint: disable=too-few-public-methods + def __init__(self, text): + super().__init__(ObjectType.TextChunk) + self.text = text + + +# define a image class +class ImageObject(KnowledgeObject): # pylint: disable=too-few-public-methods + def __init__(self, meta, path): + super().__init__(ObjectType.Image) + self.meta = meta + self.path = path + + +# define a email class +class EmailObject(KnowledgeObject): # pylint: disable=too-few-public-methods + def __init__(self, meta): + super().__init__(ObjectType.Email) + self.meta = meta + self.text = [] + self.images = [] + + diff --git a/src/aios_kernel/knowledge/object_storage.py b/src/aios_kernel/knowledge/object_storage.py new file mode 100644 index 0000000..1a6f85e --- /dev/null +++ b/src/aios_kernel/knowledge/object_storage.py @@ -0,0 +1,33 @@ +# import RDB LargeBinary +from sqlalchemy import Column, String, LargeBinary, create_engine, sessionmaker, pickle +from .object import KnowledgeObject + +# implement object storage with RDB +# define object storage table +class ObjectStorageTable(Base): + __tablename__ = 'object_storage' + id = Column(String, primary_key=True) + parent = Column(String, nullable=True) + object = Column(LargeBinary, nullable=False) + + def __init__(self, id, parent, object): # pylint: disable=redefined-builtin + self.id = id + self.parent = parent + self.object = object + +# define object storage class +class ObjectStorage: + async def __init__(self, db_url): + self.engine = create_engine(db_url) + self.session = sessionmaker(bind=self.engine)() # pylint: disable=not-callable + + async def get(self, id) -> [KnowledgeObject, KnowledgeObject]: + obj = self.session.query(ObjectStorageTable).filter(ObjectStorageTable.id == id).first() + if obj is None: + return None + return pickle.loads(obj.object) + + # define insert method + async def insert(self, object, parent): # pylint: disable=redefined-builtin + obj = ObjectStorageTable(id, parent, pickle.dumps(object)) + \ No newline at end of file diff --git a/src/aios_kernel/knowledge/vector_base.py b/src/aios_kernel/knowledge/vector_base.py new file mode 100644 index 0000000..b2cf442 --- /dev/null +++ b/src/aios_kernel/knowledge/vector_base.py @@ -0,0 +1,14 @@ +# import the ObjectID class +from .object import ObjectID + +# define a vector base class +class VectorBase: + def __init__(self, db_url, model_name) -> None: + self.db_url = db_url + self.model_name = model_name + + async def insert(self, vector: [float], id: ObjectID): + pass + + async def query(self, vector: [float], top_k: int) -> [ObjectID]: + pass \ No newline at end of file From 071452fe1983152b7ab1900f5153ea617d1a595a Mon Sep 17 00:00:00 2001 From: tsukasa Date: Thu, 31 Aug 2023 15:45:02 +0800 Subject: [PATCH 07/13] Add embeding compute task --- src/aios_kernel/compute_task.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aios_kernel/compute_task.py b/src/aios_kernel/compute_task.py index c6053db..dd875a2 100644 --- a/src/aios_kernel/compute_task.py +++ b/src/aios_kernel/compute_task.py @@ -63,6 +63,7 @@ class ComputeTask: self.params["model_name"] = model_name else: self.params["model_name"] = "text-embedding-ada-002" + self.params["input"] = input def display(self) -> str: From 0eebdf61d06eed6a079a1dc3d615ec7e2b411a85 Mon Sep 17 00:00:00 2001 From: tsukasa Date: Thu, 31 Aug 2023 16:32:20 +0800 Subject: [PATCH 08/13] Add openai text embedding compute task --- src/aios_kernel/compute_task.py | 1 - src/aios_kernel/knowledge/knowledge_base.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aios_kernel/compute_task.py b/src/aios_kernel/compute_task.py index dd875a2..c6053db 100644 --- a/src/aios_kernel/compute_task.py +++ b/src/aios_kernel/compute_task.py @@ -63,7 +63,6 @@ class ComputeTask: self.params["model_name"] = model_name else: self.params["model_name"] = "text-embedding-ada-002" - self.params["input"] = input def display(self) -> str: diff --git a/src/aios_kernel/knowledge/knowledge_base.py b/src/aios_kernel/knowledge/knowledge_base.py index 59e4f25..a576280 100644 --- a/src/aios_kernel/knowledge/knowledge_base.py +++ b/src/aios_kernel/knowledge/knowledge_base.py @@ -7,4 +7,6 @@ class KnowledgeBase: pass async def query(self, prompt: AgentPrompt) -> AgentPrompt: - pass \ No newline at end of file + pass + + \ No newline at end of file From 10d6f2fa1e6e825ead5f9f3e29f60d11e8e09ed4 Mon Sep 17 00:00:00 2001 From: tsukasa Date: Thu, 31 Aug 2023 20:13:41 +0800 Subject: [PATCH 09/13] knowledge base basic flow --- src/aios_kernel/knowledge/knowledge_base.py | 46 ++++++++++++++++++--- src/aios_kernel/knowledge/object.py | 8 ++-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/aios_kernel/knowledge/knowledge_base.py b/src/aios_kernel/knowledge/knowledge_base.py index a576280..5150ede 100644 --- a/src/aios_kernel/knowledge/knowledge_base.py +++ b/src/aios_kernel/knowledge/knowledge_base.py @@ -1,12 +1,46 @@ # define a knowledge base class -from src.aios_kernel.agent import AgentPrompt -from .object import KnowledgeObject +from ..agent import AgentPrompt +from ..compute_kernel import ComputeKernel +from .object import KnowledgeObject, ObjectType, EmailObject, TextChunkObject, ImageObject +from .object_storage import ObjectStorage +from .vector_base import VectorBase class KnowledgeBase: + def __init__(self) -> None: + self.object_store = ObjectStorage() + self.vector_base = VectorBase() + self.compute_kernel = ComputeKernel() + async def insert(self, object: KnowledgeObject): - pass + if object.object_type == ObjectType.Email: + email: EmailObject = object + for text_id in email.text: + [text, _] = self.object_store.get(text_id) + text: TextChunkObject = text + vector = await self.compute_kernel.do_text_embedding(text.text) + self.vector_base.insert(vector, text_id) + + for image_id in email.images: + [image, _] = self.object_store.get(image_id) + image: ImageObject = image + vector = await self.compute_kernel.do_text_embedding(image.meta) + self.vector_base.insert(vector, image_id) + + vector = await self.compute_kernel.do_text_embedding(email.meta) + self.vector_base.insert(vector, email.get_id()) + else: + pass async def query(self, prompt: AgentPrompt) -> AgentPrompt: - pass - - \ No newline at end of file + for msg in prompt.messages: + if msg.role == "user": + vector = await self.compute_kernel.do_text_embedding(msg.content) + object_ids = self.vector_base.query(vector, 10) + for object_id in object_ids: + if object_id.object_type == ObjectType.Email: + + [object, email] = self.object_store.get(object_id) + if object.object_type == ObjectType.Email: + email: EmailObject = object + prompt.append(AgentPrompt()) + prompt diff --git a/src/aios_kernel/knowledge/object.py b/src/aios_kernel/knowledge/object.py index 320cd8f..815d4e2 100644 --- a/src/aios_kernel/knowledge/object.py +++ b/src/aios_kernel/knowledge/object.py @@ -21,7 +21,7 @@ class ObjectID: # pylint: disable=too-few-public-methods # define a object class class KnowledgeObject(ABC): # pylint: disable=too-few-public-methods - def __init__(self, object_type): + def __init__(self, object_type: ObjectType): self.object_type = object_type @abstractmethod @@ -41,7 +41,7 @@ class KnowledgeObject(ABC): # pylint: disable=too-few-public-methods # define a text chunk class class TextChunkObject(KnowledgeObject): # pylint: disable=too-few-public-methods - def __init__(self, text): + def __init__(self, text: str): super().__init__(ObjectType.TextChunk) self.text = text @@ -59,7 +59,7 @@ class EmailObject(KnowledgeObject): # pylint: disable=too-few-public-methods def __init__(self, meta): super().__init__(ObjectType.Email) self.meta = meta - self.text = [] - self.images = [] + self.text = [ObjectID] + self.images = [ObjectID] From 218ee44de908cbde029f68ba712da0cfc6f67c95 Mon Sep 17 00:00:00 2001 From: liyaxing Date: Tue, 5 Sep 2023 20:42:22 +0800 Subject: [PATCH 10/13] Add chromadb based vector store impl and test case --- src/aios_kernel/knowledge/__init__.py | 2 - src/aios_kernel/knowledge/knowledge_base.py | 46 ------------------- src/aios_kernel/knowledge/vector_base.py | 14 ------ src/{aios_kernel => }/knowledge/object.py | 4 +- .../knowledge/object_storage.py | 0 5 files changed, 2 insertions(+), 64 deletions(-) delete mode 100644 src/aios_kernel/knowledge/__init__.py delete mode 100644 src/aios_kernel/knowledge/knowledge_base.py delete mode 100644 src/aios_kernel/knowledge/vector_base.py rename src/{aios_kernel => }/knowledge/object.py (96%) rename src/{aios_kernel => }/knowledge/object_storage.py (100%) diff --git a/src/aios_kernel/knowledge/__init__.py b/src/aios_kernel/knowledge/__init__.py deleted file mode 100644 index 13bb147..0000000 --- a/src/aios_kernel/knowledge/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .object import EmailObject, ImageObject, TextChunkObject, ObjectID -from .knowledge_base import KnowledgeBase diff --git a/src/aios_kernel/knowledge/knowledge_base.py b/src/aios_kernel/knowledge/knowledge_base.py deleted file mode 100644 index 5150ede..0000000 --- a/src/aios_kernel/knowledge/knowledge_base.py +++ /dev/null @@ -1,46 +0,0 @@ -# define a knowledge base class -from ..agent import AgentPrompt -from ..compute_kernel import ComputeKernel -from .object import KnowledgeObject, ObjectType, EmailObject, TextChunkObject, ImageObject -from .object_storage import ObjectStorage -from .vector_base import VectorBase - -class KnowledgeBase: - def __init__(self) -> None: - self.object_store = ObjectStorage() - self.vector_base = VectorBase() - self.compute_kernel = ComputeKernel() - - async def insert(self, object: KnowledgeObject): - if object.object_type == ObjectType.Email: - email: EmailObject = object - for text_id in email.text: - [text, _] = self.object_store.get(text_id) - text: TextChunkObject = text - vector = await self.compute_kernel.do_text_embedding(text.text) - self.vector_base.insert(vector, text_id) - - for image_id in email.images: - [image, _] = self.object_store.get(image_id) - image: ImageObject = image - vector = await self.compute_kernel.do_text_embedding(image.meta) - self.vector_base.insert(vector, image_id) - - vector = await self.compute_kernel.do_text_embedding(email.meta) - self.vector_base.insert(vector, email.get_id()) - else: - pass - - async def query(self, prompt: AgentPrompt) -> AgentPrompt: - for msg in prompt.messages: - if msg.role == "user": - vector = await self.compute_kernel.do_text_embedding(msg.content) - object_ids = self.vector_base.query(vector, 10) - for object_id in object_ids: - if object_id.object_type == ObjectType.Email: - - [object, email] = self.object_store.get(object_id) - if object.object_type == ObjectType.Email: - email: EmailObject = object - prompt.append(AgentPrompt()) - prompt diff --git a/src/aios_kernel/knowledge/vector_base.py b/src/aios_kernel/knowledge/vector_base.py deleted file mode 100644 index b2cf442..0000000 --- a/src/aios_kernel/knowledge/vector_base.py +++ /dev/null @@ -1,14 +0,0 @@ -# import the ObjectID class -from .object import ObjectID - -# define a vector base class -class VectorBase: - def __init__(self, db_url, model_name) -> None: - self.db_url = db_url - self.model_name = model_name - - async def insert(self, vector: [float], id: ObjectID): - pass - - async def query(self, vector: [float], top_k: int) -> [ObjectID]: - pass \ No newline at end of file diff --git a/src/aios_kernel/knowledge/object.py b/src/knowledge/object.py similarity index 96% rename from src/aios_kernel/knowledge/object.py rename to src/knowledge/object.py index 815d4e2..f7a127e 100644 --- a/src/aios_kernel/knowledge/object.py +++ b/src/knowledge/object.py @@ -1,7 +1,7 @@ # define a object type enum -from abc import ABC - +from abc import ABC, abstractmethod +from enum import Enum class ObjectType(Enum): TextChunk = 1 diff --git a/src/aios_kernel/knowledge/object_storage.py b/src/knowledge/object_storage.py similarity index 100% rename from src/aios_kernel/knowledge/object_storage.py rename to src/knowledge/object_storage.py From 3721f4aaa49c3ca9f7a83ba58aa0e4590693075d Mon Sep 17 00:00:00 2001 From: tsukasa Date: Mon, 18 Sep 2023 12:48:26 +0800 Subject: [PATCH 11/13] test prompt from knowledge object --- src/aios_kernel/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aios_kernel/__init__.py b/src/aios_kernel/__init__.py index 7436dbf..e324e15 100644 --- a/src/aios_kernel/__init__.py +++ b/src/aios_kernel/__init__.py @@ -12,10 +12,10 @@ from .bus import AIBus from .workflow_env import WorkflowEnvironment,CalenderEnvironment,CalenderEvent from .local_llama_compute_node import LocalLlama_ComputeNode from .whisper_node import WhisperComputeNode -from .google_text_to_speech_node import GoogleTextToSpeechNode +# from .google_text_to_speech_node import GoogleTextToSpeechNode from .tunnel import AgentTunnel -from .tg_tunnel import TelegramTunnel -from .email_tunnel import EmailTunnel +# from .tg_tunnel import TelegramTunnel +# from .email_tunnel import EmailTunnel from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem AIOS_Version = "0.5.1, build 2023-9-17" From edbd1c41dac2a9610d499dbe77c8a3e0570612c7 Mon Sep 17 00:00:00 2001 From: tsukasa Date: Wed, 20 Sep 2023 14:38:43 +0800 Subject: [PATCH 12/13] add knowledge pipeline --- aios_shell_history.txt | 6 + src/aios_kernel/__init__.py | 7 +- src/aios_kernel/knowledge_base.py | 7 +- src/aios_kernel/knowledge_pipeline.py | 192 ++++++++++++++++++++++++++ src/requirements.txt | 98 ++++++++++--- 5 files changed, 282 insertions(+), 28 deletions(-) create mode 100644 aios_shell_history.txt create mode 100644 src/aios_kernel/knowledge_pipeline.py diff --git a/aios_shell_history.txt b/aios_shell_history.txt new file mode 100644 index 0000000..b0a8786 --- /dev/null +++ b/aios_shell_history.txt @@ -0,0 +1,6 @@ + +# 2023-09-19 16:04:55.011656 ++tsukasa + +# 2023-09-19 16:05:27.714815 ++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 diff --git a/src/aios_kernel/__init__.py b/src/aios_kernel/__init__.py index e324e15..6e6e000 100644 --- a/src/aios_kernel/__init__.py +++ b/src/aios_kernel/__init__.py @@ -6,16 +6,17 @@ from .compute_kernel import ComputeKernel,ComputeTask from .compute_node import ComputeNode,LocalComputeNode from .open_ai_node import OpenAI_ComputeNode from .knowledge_base import KnowledgeBase +from .knowledge_pipeline import EmailSpider from .role import AIRole,AIRoleGroup from .workflow import Workflow from .bus import AIBus from .workflow_env import WorkflowEnvironment,CalenderEnvironment,CalenderEvent from .local_llama_compute_node import LocalLlama_ComputeNode from .whisper_node import WhisperComputeNode -# from .google_text_to_speech_node import GoogleTextToSpeechNode +from .google_text_to_speech_node import GoogleTextToSpeechNode from .tunnel import AgentTunnel -# from .tg_tunnel import TelegramTunnel -# from .email_tunnel import EmailTunnel +from .tg_tunnel import TelegramTunnel +from .email_tunnel import EmailTunnel from .storage import ResourceLocation,AIStorage,UserConfig,UserConfigItem AIOS_Version = "0.5.1, build 2023-9-17" diff --git a/src/aios_kernel/knowledge_base.py b/src/aios_kernel/knowledge_base.py index b229864..50423fe 100644 --- a/src/aios_kernel/knowledge_base.py +++ b/src/aios_kernel/knowledge_base.py @@ -1,7 +1,7 @@ # define a knowledge base class import json import logging -from . import AgentPrompt, ComputeKernel +from . import AgentPrompt, ComputeKernel, AIStorage from knowledge import * @@ -239,9 +239,6 @@ class KnowledgeBase: prompt.messages.append({"role": "knowledge", "content": content}) return prompt - - - - + \ No newline at end of file diff --git a/src/aios_kernel/knowledge_pipeline.py b/src/aios_kernel/knowledge_pipeline.py new file mode 100644 index 0000000..c60adad --- /dev/null +++ b/src/aios_kernel/knowledge_pipeline.py @@ -0,0 +1,192 @@ +""" +Capture your email locally, and parse out the pictures in the email body and the pictures, videos and other files in the attachment. Subsequently, it supports vectorized analysis of your personal data and serves as a knowledge base to enable large language model answers. Better results. + +An example of a local file is as follows: +├── data +│ └── alex0072@gmail.com +│ └── 5de3e52f3a6b90cabe6cbdd4ae3a5c5b +│ ├── email.txt +│ ├── meta.json +│ ├── image +│ │ ├── 0648B869@99C03070.DB94B354.jpg +│ └── body_image +│ ├── 11044884873.jpg +│ ├── 282985198265470.gif +│ └── dd-login-service-min.png + +""" + +import imaplib +import os +import toml +import logging +import mailparser +import hashlib +import json +import base64 +from bs4 import BeautifulSoup +import requests + +class EmailSpider: + def __init__(self): + # logger config + self.logger = logging.getLogger('email spider') + self.logger.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s [%(name)s] [%(levelname)s] %(message)s') + ch.setFormatter(formatter) + self.logger.addHandler(ch) + + # read config from toml file + # and read from config config.local.toml if exists (config.local.toml is ignored by git) + self.config = toml.load('./rootfs/email/config.toml') + if os.path.exists('./rootfs/email/config.local.toml'): + self.config = toml.load('./rootfs/email/config.local.toml') + + self.client = self.email_client() + + def email_client(self) -> imaplib.IMAP4_SSL: + self.logger.info(f"read email config from {self.config.get('EMAIL_IMAP_SERVER')}") + client = imaplib.IMAP4_SSL( + host=self.config.get('EMAIL_IMAP_SERVER'), + port=self.config.get('EMAIL_IMAP_PORT') + ) + client.login(self.config.get('EMAIL_ADDRESS'), self.config.get('EMAIL_PASSWORD')) + return client + + def list_box(self): + _, mailbox_list = self.client.list() + for mailbox in mailbox_list: + print(mailbox.decode()) + + def read_emails(self, folder: str = 'INBOX', imap_keyword: str = "UNSEEN"): + self.client.select(folder) + _, data = self.client.uid('search', None, imap_keyword) + + # get email uid list + email_list = data[0].split() + self.logger.info(f"got {len(email_list)} emails") + email_list.reverse() + for uid in email_list: + if self.check_email_saved(uid): + self.logger.info(f"email uid {uid} already saved") + else: + self.read_and_save_email(uid) + self.logger.info(f"email uid {uid} saved") + + def read_and_save_email(self, uid: str): + message_parts = "(BODY.PEEK[])" + _, email_data = self.client.uid('fetch', uid, message_parts) + mail = mailparser.parse_from_bytes(email_data[0][1]) + self.logger.info(f"got email subject [{mail.subject}]") + self.save_email(mail) + + def get_local_dir_name(self, mail: mailparser.MailParser) -> str: + dir = f"{self.config.get('LOCAL_DIR')}/{self.config.get('EMAIL_ADDRESS')}" + name = f"{mail.subject}__{mail.date}" + name = hashlib.md5(name.encode('utf-8')).hexdigest() + return f"{dir}/{name}" + + def check_email_saved(self, uid: str): + message_parts = "(BODY[HEADER])" + _, email_data = self.client.uid('fetch', uid, message_parts) + mail = mailparser.parse_from_bytes(email_data[0][1]) + self.logger.info(f"[{uid}]check email subject [{mail.subject}]") + dir = self.get_local_dir_name(mail) + self.logger.info(f"check email saved {dir}") + file = f"{dir}/email.txt" + if os.path.exists(file): + return False + return False + + # save email attachment(images) + def save_email_attachment(self, mail: mailparser.MailParser, email_dir: str): + for attachment in mail.attachments: + if attachment['mail_content_type'] in ['image/png', 'image/jpeg', 'image/gif']: + print('current mail have image attachment') + img_dir = f"{email_dir}/image" + if not os.path.exists(img_dir): + os.makedirs(img_dir) + filename = attachment['filename'] + filefullname = f"{img_dir}/{filename}" + image_data = attachment['payload'] + try: + image_data = base64.b64decode(image_data) + except base64.binascii.Error: + image_data = image_data.encode() + with open(filefullname, 'wb') as f: + f.write(image_data) + self.logger.info(f"save email image {filename} success") + + # save email body images(html content) + def save_body_images(self, html_content: str, email_dir: str): + # get all image urls + soup = BeautifulSoup(html_content, 'html.parser') + img_tags = soup.find_all('img') + img_urls = [img['src'] for img in img_tags if 'src' in img.attrs] + self.logger.info(f'Found {len(img_urls)} images in email body') + + if not os.path.exists(email_dir): + os.makedirs(email_dir) + + for img_url in img_urls: + # keep the original image filename(last of url) + img_filename = os.path.join(email_dir, img_url.split('/')[-1]) + # download image + response = requests.get(img_url, stream=True) + if response.status_code == 200: + with open(img_filename, 'wb') as img_file: + for chunk in response.iter_content(1024): + img_file.write(chunk) + self.logger.info(f'Downloaded {img_url} to {img_filename}') + else: + self.logger.info(f'Failed to download {img_url}') + + # save email content to local dir + def save_email(self, mail: mailparser.MailParser): + dir = f"{self.config.get('LOCAL_DIR')}/{self.config.get('EMAIL_ADDRESS')}" + if not os.path.exists(dir): + os.makedirs(dir) + email_dir = self.get_local_dir_name(mail) + self.logger.info(f"save email to {email_dir}") + if not os.path.exists(email_dir): + os.makedirs(email_dir) + with open(f"{email_dir}/email.txt", "w") as f: + f.write(mail.body) + with open(f"{email_dir}/meta.json", "w", encoding='utf-8') as f: + mail_dict = json.loads(mail.mail_json) + if 'body' in mail_dict: + del mail_dict['body'] + json.dump(mail_dict, f, ensure_ascii=False, indent=4) + self.logger.info(f"save email meta info {f.name}") + + self.save_email_attachment(mail, email_dir) + self.save_body_images(mail.body, f"{email_dir}/body_image") + + + +from . import AIStorage, KnowledgeBase + +# define singleton class knowledge pipline +class KnowledgePipline: + _instance = None + + @classmethod + def get_instance(cls): + if cls._instance is None: + cls._instance = KnowledgePipline() + cls._instance.__singleton_init__() + + return cls._instance + + def __singleton_init__(self) -> None: + self.knowledge_base = KnowledgeBase() + + @classmethod + def declare_user_config(cls): + user_config = AIStorage.get_instance().get_user_config() + user_config.add_user_config("email_spiders","email addresses to build knowledge base",True,None,"list") + user_config.add_user_config("personal_dirs", "personal directories to build knowledge base", True, None, "list") + + def add_email_address(self, ) -> None: + pass diff --git a/src/requirements.txt b/src/requirements.txt index 6f364bb..0c88c8a 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,25 +1,83 @@ -chromadb==0.4 -moviepy==1.0 -base58==2.1 -base36==0.1 aiofiles==23.2.1 -aiohttp==3.7.0 +aiohttp==3.8.5 aioimaplib==1.0.1 +aiosignal==1.3.1 aiosmtplib==2.0.2 +anyio==4.0.0 +async-timeout==4.0.3 +attrs==23.1.0 +backoff==2.2.1 +base36==0.1.1 +base58==2.1.1 beautifulsoup4==4.12.2 -mail_parser==3.15.0 -prompt_toolkit==3.0.39 -pydantic==1.10.11 -python-telegram-bot==20.5 -Requests==2.31.0 -protobuf -stability_sdk -toml -base58 -google-cloud-texttospeech -openai -Pillow -aiosqlite -PySocks==1.7.1 -toml==0.10.2 +cachetools==5.3.1 +certifi==2023.7.22 +charset-normalizer==3.2.0 +chroma-hnswlib==0.7.1 +chromadb==0.4.0 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +decorator==4.4.2 +fastapi==0.99.1 +filelock==3.12.3 +flatbuffers==23.5.26 +frozenlist==1.4.0 +fsspec==2023.9.0 +google==3.0.0 +google-api-core==2.11.1 +google-auth==2.23.0 +google-cloud==0.34.0 google-cloud-texttospeech==2.14.1 +googleapis-common-protos==1.60.0 +grpcio==1.58.0 +grpcio-status==1.58.0 +h11==0.14.0 +httpcore==0.17.3 +httptools==0.6.0 +httpx==0.24.1 +huggingface-hub==0.16.4 +humanfriendly==10.0 +idna==3.4 +imageio==2.31.3 +imageio-ffmpeg==0.4.8 +importlib-resources==6.0.1 +mail-parser==3.15.0 +monotonic==1.6 +moviepy==1.0.0 +mpmath==1.3.0 +multidict==6.0.4 +numpy==1.25.2 +onnxruntime==1.15.1 +openai==0.28.0 +overrides==7.4.0 +packaging==23.1 +pandas==2.1.0 +Pillow==10.0.0 +posthog==3.0.2 +proglog==0.1.10 +prompt-toolkit==3.0.39 +proto-plus==1.22.3 +protobuf==4.24.3 +pulsar-client==3.3.0 +pyasn1==0.5.0 +pyasn1-modules==0.3.0 +pydantic==1.10.12 +PyPika==0.48.9 +pyreadline3==3.4.1 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-telegram-bot==20.5 +pytz==2023.3.post1 +PyYAML==6.0.1 +requests==2.31.0 +rsa==4.9 +simplejson==3.19.1 +six==1.16.0 +sniffio==1.3.0 +soupsieve==2.5 +starlette==0.27.0 +sympy==1.12 +telegram==0.0.1 +tokenizers==0.14.0 +toml==0.10.0 \ No newline at end of file From a0a45b89988ca8f3b60c4211a2b91d52f0f5f761 Mon Sep 17 00:00:00 2001 From: tsukasa Date: Thu, 21 Sep 2023 18:32:17 +0800 Subject: [PATCH 13/13] shell knowledge commands --- aios_shell_history.txt | 105 ++++++++ src/aios_kernel/__init__.py | 2 +- src/aios_kernel/knowledge_base.py | 9 +- src/aios_kernel/knowledge_pipeline.py | 270 +++++++++++++++---- src/knowledge/core_object/document_object.py | 4 +- src/knowledge/core_object/email_object.py | 2 +- src/knowledge/core_object/image_object.py | 2 +- src/knowledge/data/writer.py | 2 +- src/knowledge/store.py | 16 +- src/knowledge/vector/chroma_store.py | 6 +- src/service/aios_shell/aios_shell.py | 65 ++++- 11 files changed, 409 insertions(+), 74 deletions(-) diff --git a/aios_shell_history.txt b/aios_shell_history.txt index b0a8786..34732fd 100644 --- a/aios_shell_history.txt +++ b/aios_shell_history.txt @@ -4,3 +4,108 @@ # 2023-09-19 16:05:27.714815 +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 + +# 2023-09-20 14:48:06.181582 ++/connect + +# 2023-09-20 14:48:22.678945 ++/connect Jarvis email + +# 2023-09-20 14:54:35.101026 ++/connect Jarvis telegram + +# 2023-09-20 14:54:55.229801 ++token abcd + +# 2023-09-20 16:22:34.229084 ++/knowledge add email + +# 2023-09-20 16:22:46.797842 ++puotosssa@live.com + +# 2023-09-20 16:22:53.622015 ++p19870626 + +# 2023-09-20 16:22:59.186145 ++imap@live.com + +# 2023-09-20 16:23:02.334411 ++993 + +# 2023-09-20 16:40:29.996329 ++/knowledge add email + +# 2023-09-20 16:40:52.860700 ++puotosssa@live.com + +# 2023-09-20 16:40:55.324994 ++fdafda + +# 2023-09-20 16:40:58.167154 ++fdafdaf + +# 2023-09-20 16:40:59.757363 ++993 + +# 2023-09-20 16:42:55.288680 ++/knowledge add email + +# 2023-09-20 16:42:56.916408 ++fdfa + +# 2023-09-20 16:42:57.965232 ++fdfds + +# 2023-09-20 16:42:59.149516 ++fdafd + +# 2023-09-20 16:43:00.949104 ++993 + +# 2023-09-21 14:11:33.500467 ++/knowledge add c:/users/tsukasa/myai/photos + +# 2023-09-21 14:14:31.488794 ++/knowledge add dir + +# 2023-09-21 14:14:49.298635 ++c:/users/tsukasa/myai/photos + +# 2023-09-21 15:27:58.550658 ++/knowledge add dir + +# 2023-09-21 15:28:10.190467 ++c:/users/tsukasa/myai/photos + +# 2023-09-21 15:32:32.431305 ++/knowledge add dir + +# 2023-09-21 15:32:46.695566 ++c:/users/tsukasa/myai/photos + +# 2023-09-21 15:33:36.391465 ++/knowledge add dir + +# 2023-09-21 15:33:47.574853 ++c:/users/tsukasa/myai/photos + +# 2023-09-21 17:44:26.308692 ++/knowledge journals + +# 2023-09-21 17:45:42.925359 ++/knowledge journal + +# 2023-09-21 18:13:08.720054 ++/knowledge query what a see is + +# 2023-09-21 18:20:09.275556 ++/knowledge journal + +# 2023-09-21 18:20:10.746056 ++/knowledge query what a see is + +# 2023-09-21 18:29:48.391223 ++/knowledge journal + +# 2023-09-21 18:29:51.406532 ++/knowledge query what a see is diff --git a/src/aios_kernel/__init__.py b/src/aios_kernel/__init__.py index 6e6e000..7240b92 100644 --- a/src/aios_kernel/__init__.py +++ b/src/aios_kernel/__init__.py @@ -6,7 +6,7 @@ from .compute_kernel import ComputeKernel,ComputeTask from .compute_node import ComputeNode,LocalComputeNode from .open_ai_node import OpenAI_ComputeNode from .knowledge_base import KnowledgeBase -from .knowledge_pipeline import EmailSpider +from .knowledge_pipeline import KnowledgeEmailSource, KnowledgeDirSource, KnowledgePipline from .role import AIRole,AIRoleGroup from .workflow import Workflow from .bus import AIBus diff --git a/src/aios_kernel/knowledge_base.py b/src/aios_kernel/knowledge_base.py index 50423fe..11d6ce7 100644 --- a/src/aios_kernel/knowledge_base.py +++ b/src/aios_kernel/knowledge_base.py @@ -1,7 +1,9 @@ # define a knowledge base class import json import logging -from . import AgentPrompt, ComputeKernel, AIStorage +from .agent import AgentPrompt +from .compute_kernel import ComputeKernel +from .storage import AIStorage from knowledge import * @@ -17,7 +19,7 @@ class KnowledgeBase: def __singleton_init__(self) -> None: self.store = KnowledgeStore() - self.compute_kernel = ComputeKernel() + self.compute_kernel = ComputeKernel.get_instance() async def __embedding_document(self, document: DocumentObject): for chunk_id in document.get_chunk_list(): @@ -155,7 +157,7 @@ class KnowledgeBase: # pass async def insert_object(self, object: KnowledgeObject): - # self.__save_object(object) + self.store.get_object_store().put_object(object.calculate_id(), object.encode()) await self.__do_embedding(object) async def query_prompt(self, prompt: AgentPrompt): @@ -164,6 +166,7 @@ class KnowledgeBase: knowledge_prompt = self.prompt_from_objects(objects) logging.info(f"prompt_from_objects result: {knowledge_prompt.as_str()}") prompt.append(knowledge_prompt) + return prompt async def query_objects(self, prompt: AgentPrompt) -> [ObjectID]: results = [] diff --git a/src/aios_kernel/knowledge_pipeline.py b/src/aios_kernel/knowledge_pipeline.py index c60adad..c1f0a72 100644 --- a/src/aios_kernel/knowledge_pipeline.py +++ b/src/aios_kernel/knowledge_pipeline.py @@ -15,10 +15,10 @@ An example of a local file is as follows: │ └── dd-login-service-min.png """ - +import asyncio +import datetime +import sqlite3 import imaplib -import os -import toml import logging import mailparser import hashlib @@ -26,63 +26,139 @@ import json import base64 from bs4 import BeautifulSoup import requests +import os +import toml +from .storage import AIStorage, UserConfigItem +from .knowledge_base import KnowledgeBase, ImageObjectBuilder, ObjectID, ObjectType, DocumentObjectBuilder -class EmailSpider: +class KnowledgeJournal: + def __init__(self, source_type: str, source_id: str, item_id: str, object_id: str, timestamp=None): + # define a timestamp variable + self.timestamp = datetime.datetime.now() if timestamp is None else timestamp + self.object_id = object_id + self.source_type = source_type + self.source_id = source_id + self.item_id = item_id + + def __str__(self) -> str: + if self.source_type == "dir": + object_id = ObjectID.from_base58(self.object_id) + object_type = None + if object_id.get_object_type() == ObjectType.Image: + object_type = "image" + else: + pass + return f"Add {object_type} from {os.path.join(self.source_id, self.item_id)}" + if self.source_type == "email": + pass + + +# init sqlite3 client +class KnowledgeJournalClient: def __init__(self): - # logger config - self.logger = logging.getLogger('email spider') - self.logger.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - formatter = logging.Formatter('%(asctime)s [%(name)s] [%(levelname)s] %(message)s') - ch.setFormatter(formatter) - self.logger.addHandler(ch) + knowledge_dir = os.path.join(AIStorage.get_instance().get_myai_dir(), "knowledge") + if not os.path.exists(knowledge_dir): + os.makedirs(knowledge_dir) + self.journal_path = os.path.join(knowledge_dir, "journal.db") + + conn = sqlite3.connect(self.journal_path) + conn.execute( + '''CREATE TABLE IF NOT EXISTS journal ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + time DATETIME DEFAULT CURRENT_TIMESTAMP, + source_type TEXT, + source_id TEXT, + item_id TEXT, + object_id TEXT)''' + ) + conn.commit() + def insert(self, journal: KnowledgeJournal): + conn = sqlite3.connect(self.journal_path) + conn.execute( + "INSERT INTO journal (time, source_type, source_id, item_id, object_id) VALUES (?, ?, ?, ?, ?)", + (journal.timestamp, journal.source_type, journal.source_id, journal.item_id, journal.object_id), + ) + conn.commit() + + def latest_journal(self, source_id: str) -> KnowledgeJournal: + conn = sqlite3.connect(self.journal_path) + cursor = conn.cursor() + cursor.execute("SELECT * FROM journal WHERE source_id = ? ORDER BY id DESC LIMIT 1", (source_id,)) + result = cursor.fetchone() + if result is None: + return None + else: + (_, timestamp, source_type, sorce_id, item_id, object_id) = result + return KnowledgeJournal(source_type, sorce_id, item_id, object_id, timestamp) + + def latest_journals(self, topn) -> [KnowledgeJournal]: + conn = sqlite3.connect(self.journal_path) + cursor = conn.cursor() + cursor.execute("SELECT * FROM journal ORDER BY id DESC LIMIT ?", (topn,)) + return [KnowledgeJournal(source_type, sorce_id, item_id, object_id, timestamp) for (_, timestamp, source_type, sorce_id, item_id, object_id) in cursor.fetchall()] + + +class KnowledgeEmailSource: + def __init__(self, config:dict): + self.config = config + self.config["type"] = "email" + + def id(self): + "::".join([self.config["imap_server"], self.config["address"]]) + + @classmethod + def user_config_items(cls): + return [("address", "email address"), + ("password", "email password"), + ("imap_server", "imap server"), + ("imap_port", "imap port") + ] + + @classmethod + def local_root(cls): + user_data_dir = AIStorage.get_instance().get_myai_dir() + return os.path.abspath(f"{user_data_dir}/email") + + async def run_once(self): # read config from toml file # and read from config config.local.toml if exists (config.local.toml is ignored by git) - self.config = toml.load('./rootfs/email/config.toml') - if os.path.exists('./rootfs/email/config.local.toml'): - self.config = toml.load('./rootfs/email/config.local.toml') - self.client = self.email_client() + await self.read_emails() def email_client(self) -> imaplib.IMAP4_SSL: - self.logger.info(f"read email config from {self.config.get('EMAIL_IMAP_SERVER')}") + logging.info(f"read email config from {self.config.get('imap_server')}") client = imaplib.IMAP4_SSL( - host=self.config.get('EMAIL_IMAP_SERVER'), - port=self.config.get('EMAIL_IMAP_PORT') + host=self.config.get('imap_server'), + port=self.config.get('imap_port') ) - client.login(self.config.get('EMAIL_ADDRESS'), self.config.get('EMAIL_PASSWORD')) + client.login(self.config.get('address'), self.config.get('password')) return client - def list_box(self): - _, mailbox_list = self.client.list() - for mailbox in mailbox_list: - print(mailbox.decode()) - - def read_emails(self, folder: str = 'INBOX', imap_keyword: str = "UNSEEN"): + async def read_emails(self, folder: str = 'INBOX', imap_keyword: str = "UNSEEN"): self.client.select(folder) _, data = self.client.uid('search', None, imap_keyword) # get email uid list email_list = data[0].split() - self.logger.info(f"got {len(email_list)} emails") + logging.info(f"got {len(email_list)} emails") email_list.reverse() for uid in email_list: if self.check_email_saved(uid): - self.logger.info(f"email uid {uid} already saved") + logging.info(f"email uid {uid} already saved") else: self.read_and_save_email(uid) - self.logger.info(f"email uid {uid} saved") + logging.info(f"email uid {uid} saved") def read_and_save_email(self, uid: str): message_parts = "(BODY.PEEK[])" _, email_data = self.client.uid('fetch', uid, message_parts) mail = mailparser.parse_from_bytes(email_data[0][1]) - self.logger.info(f"got email subject [{mail.subject}]") + logging.info(f"got email subject [{mail.subject}]") self.save_email(mail) def get_local_dir_name(self, mail: mailparser.MailParser) -> str: - dir = f"{self.config.get('LOCAL_DIR')}/{self.config.get('EMAIL_ADDRESS')}" + dir = f"{self.local_root()}/{self.config.get('address')}" name = f"{mail.subject}__{mail.date}" name = hashlib.md5(name.encode('utf-8')).hexdigest() return f"{dir}/{name}" @@ -91,9 +167,9 @@ class EmailSpider: message_parts = "(BODY[HEADER])" _, email_data = self.client.uid('fetch', uid, message_parts) mail = mailparser.parse_from_bytes(email_data[0][1]) - self.logger.info(f"[{uid}]check email subject [{mail.subject}]") + logging.info(f"[{uid}]check email subject [{mail.subject}]") dir = self.get_local_dir_name(mail) - self.logger.info(f"check email saved {dir}") + logging.info(f"check email saved {dir}") file = f"{dir}/email.txt" if os.path.exists(file): return False @@ -116,7 +192,7 @@ class EmailSpider: image_data = image_data.encode() with open(filefullname, 'wb') as f: f.write(image_data) - self.logger.info(f"save email image {filename} success") + logging.info(f"save email image {filename} success") # save email body images(html content) def save_body_images(self, html_content: str, email_dir: str): @@ -124,7 +200,7 @@ class EmailSpider: soup = BeautifulSoup(html_content, 'html.parser') img_tags = soup.find_all('img') img_urls = [img['src'] for img in img_tags if 'src' in img.attrs] - self.logger.info(f'Found {len(img_urls)} images in email body') + logging.info(f'Found {len(img_urls)} images in email body') if not os.path.exists(email_dir): os.makedirs(email_dir) @@ -138,17 +214,17 @@ class EmailSpider: with open(img_filename, 'wb') as img_file: for chunk in response.iter_content(1024): img_file.write(chunk) - self.logger.info(f'Downloaded {img_url} to {img_filename}') + logging.info(f'Downloaded {img_url} to {img_filename}') else: - self.logger.info(f'Failed to download {img_url}') + logging.info(f'Failed to download {img_url}') # save email content to local dir def save_email(self, mail: mailparser.MailParser): - dir = f"{self.config.get('LOCAL_DIR')}/{self.config.get('EMAIL_ADDRESS')}" + dir = f"{self.local_root()}/{self.config.get('address')}" if not os.path.exists(dir): os.makedirs(dir) email_dir = self.get_local_dir_name(mail) - self.logger.info(f"save email to {email_dir}") + logging.info(f"save email to {email_dir}") if not os.path.exists(email_dir): os.makedirs(email_dir) with open(f"{email_dir}/email.txt", "w") as f: @@ -158,14 +234,58 @@ class EmailSpider: if 'body' in mail_dict: del mail_dict['body'] json.dump(mail_dict, f, ensure_ascii=False, indent=4) - self.logger.info(f"save email meta info {f.name}") + logging.info(f"save email meta info {f.name}") self.save_email_attachment(mail, email_dir) self.save_body_images(mail.body, f"{email_dir}/body_image") +class KnowledgeDirSource: + def __init__(self, config): + self.config = config + self.config["type"] = "dir" + + @classmethod + def user_config_items(cls): + return [("path", "local dir path")] + + def id(self): + return self.config["path"] + + def path(self): + return self.config["path"] + + async def run_once(self): + logging.debug(f"knowledge dir source {self.id()} run once") + journal_client = KnowledgeJournalClient() + latest_journal = journal_client.latest_journal(self.id()) + if latest_journal is not None: + if os.path.getmtime(self.path()) <= latest_journal.timestamp: + logging.debug(f"knowledge dir source {self.id()} ingnored for no update") + return + file_pathes = sorted(os.listdir(self.path()), key=lambda x: os.path.getctime(os.path.join(self.path(), x))) + for rel_path in file_pathes: + file_path = os.path.join(self.path(), rel_path) + timestamp = os.path.getctime(file_path) + if latest_journal is not None: + if timestamp <= latest_journal.timestamp: + continue + ext = os.path.splitext(file_path)[1].lower() + if ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp']: + logging.info(f"knowledge dir source {self.id()} found image file {file_path}") + image = ImageObjectBuilder({}, {}, file_path).build() + await KnowledgeBase().insert_object(image) + journal_client.insert(KnowledgeJournal("dir", self.id(), rel_path, str(image.calculate_id()), timestamp)) + if ext in ['.txt']: + logging.info(f"knowledge dir source {self.id()} found text file {file_path}") + with open(file_path, "r", encoding="utf-8") as f: + text = f.read() + document = DocumentObjectBuilder({}, {}, text).build() + await KnowledgeBase().insert_object(document) + journal_client.insert(KnowledgeJournal("dir", self.id(), rel_path, str(document.calculate_id()), timestamp)) + + -from . import AIStorage, KnowledgeBase # define singleton class knowledge pipline class KnowledgePipline: @@ -178,15 +298,71 @@ class KnowledgePipline: cls._instance.__singleton_init__() return cls._instance + + def initial(self): + config_path = self.__config_path() + logging.info(f"initial knowledge pipline from {config_path}") + if os.path.exists(config_path): + config = toml.load(self.__config_path()) + for source_config in config["sources"]: + if source_config['type'] == 'email': + self.add_email_source(KnowledgeEmailSource(source_config)) + if source_config['type'] == 'dir': + self.add_dir_source(KnowledgeDirSource(source_config)) - def __singleton_init__(self) -> None: + def __singleton_init__(self): self.knowledge_base = KnowledgeBase() + self.email_sources = dict() + self.dir_sources = dict() + self.source_queue = list() + self.run_lock = asyncio.Lock() + asyncio.create_task(self.run_loop()) + + + def save_config(self): + config = dict() + config["sources"] = [source.config for source in self.source_queue] + with open(self.__config_path(), "w") as f: + toml.dump(config, f) + @classmethod - def declare_user_config(cls): - user_config = AIStorage.get_instance().get_user_config() - user_config.add_user_config("email_spiders","email addresses to build knowledge base",True,None,"list") - user_config.add_user_config("personal_dirs", "personal directories to build knowledge base", True, None, "list") + def __config_path(cls) -> str: + user_data_dir = AIStorage.get_instance().get_myai_dir() + return os.path.abspath(f"{user_data_dir}/etc/knowledge.cfg.toml") - def add_email_address(self, ) -> None: - pass + + def add_email_source(self, source: KnowledgeEmailSource): + if self.email_sources.get(source.id()) is not None: + return "already exists" + self.email_sources[source.id()] = source + self.source_queue.append(source) + return None + + def add_dir_source(self, source: KnowledgeDirSource): + if self.dir_sources.get(source.id()) is not None: + logging.info(f"knowledge add source {source.id()} failed for already exists") + return "already exists" + logging.info(f"knowledge added source {source.id()}") + self.dir_sources[source.id()] = source + self.source_queue.append(source) + return None + + def get_latest_journals(self, topn) -> [KnowledgeJournal]: + return KnowledgeJournalClient().latest_journals(topn) + + async def run_loop(self): + while True: + await self.run_once() + await asyncio.sleep(5) + + async def run_once(self): + logging.info(f"knowledge pipeline started") + # sources = list() + # async with self.run_lock: + # for source in self.source_queue: + # sources.append(source) + # for source in sources: + # await source.run_once() + for source in self.source_queue: + await source.run_once() diff --git a/src/knowledge/core_object/document_object.py b/src/knowledge/core_object/document_object.py index 7210f16..672b4a3 100644 --- a/src/knowledge/core_object/document_object.py +++ b/src/knowledge/core_object/document_object.py @@ -49,7 +49,7 @@ class DocumentObjectBuilder: self.text = text return self - def build(self, relation_store: ObjectRelationStore) -> DocumentObject: + def build(self) -> DocumentObject: chunk_list = KnowledgeStore().get_chunk_list_writer().create_chunk_list_from_text( self.text, 1024 * 4, @@ -60,6 +60,6 @@ class DocumentObjectBuilder: # Add relation to store for chunk_id in chunk_list.chunk_list: - relation_store.add_relation(chunk_id, doc_id) + KnowledgeStore().get_relation_store().add_relation(chunk_id, doc_id) return doc diff --git a/src/knowledge/core_object/email_object.py b/src/knowledge/core_object/email_object.py index c86c3a2..37531ff 100644 --- a/src/knowledge/core_object/email_object.py +++ b/src/knowledge/core_object/email_object.py @@ -94,7 +94,7 @@ class EmailObjectBuilder: with open(content_file, "r", encoding="utf-8") as f: text = f.read() - document = DocumentObjectBuilder({}, {}, text).build(relation_store=relation) + document = DocumentObjectBuilder({}, {}, text).build() document_id = document.calculate_id() store.put_object(document_id, document.encode()) documents = {"email.txt": document_id} diff --git a/src/knowledge/core_object/image_object.py b/src/knowledge/core_object/image_object.py index 30b0e8a..2c02ad6 100644 --- a/src/knowledge/core_object/image_object.py +++ b/src/knowledge/core_object/image_object.py @@ -52,7 +52,7 @@ def get_exif_data(image_path: str): return { TAGS.get(key): exif_data[key] for key in exif_data.keys() - if key in TAGS and isinstance(exif_data[key], (bytes, str)) + if key in TAGS and isinstance(exif_data[key], str) } else: return {} diff --git a/src/knowledge/data/writer.py b/src/knowledge/data/writer.py index 3ac0ed1..ad6c520 100644 --- a/src/knowledge/data/writer.py +++ b/src/knowledge/data/writer.py @@ -46,7 +46,7 @@ class ChunkListWriter: ) file_hash = HashValue(hash_obj.digest()) - print(f"calc file hash: {file_path}, {file_hash}") + # print(f"calc file hash: {file_path}, {file_hash}") return ChunkList(chunk_list, file_hash) diff --git a/src/knowledge/store.py b/src/knowledge/store.py index 03e48d0..56ec64b 100644 --- a/src/knowledge/store.py +++ b/src/knowledge/store.py @@ -4,7 +4,7 @@ from .object import ObjectStore, ObjectRelationStore from .data import ChunkStore, ChunkTracker, ChunkListWriter, ChunkReader from .vector import ChromaVectorStore, VectorBase import logging - +import aios_kernel # KnowledgeStore class, which aggregates ChunkStore, ChunkTracker, and ObjectStore, and is a global singleton that makes it easy to use these three built-in store examples class KnowledgeStore: @@ -13,16 +13,12 @@ class KnowledgeStore: def __new__(cls): if cls._instance is None: cls._instance = super().__new__(cls) - directory = os.path.join( - os.path.dirname(__file__), "../../rootfs/data/" - ) - directory = os.path.normpath(directory) - print(directory) + knowledge_dir = aios_kernel.storage.AIStorage().get_myai_dir() / "knowledge" - if not os.path.exists(directory): - os.makedirs(directory) + if not os.path.exists(knowledge_dir): + os.makedirs(knowledge_dir) - cls._instance.__singleton_init__(directory) + cls._instance.__singleton_init__(knowledge_dir) return cls._instance @@ -64,5 +60,5 @@ class KnowledgeStore: def get_vector_store(self, model_name: str) -> VectorBase: if model_name not in self.vector_store: - self.vector_store[model_name] = ChromaVectorStore(model_name) + self.vector_store[model_name] = ChromaVectorStore(self.root, model_name) return self.vector_store[model_name] diff --git a/src/knowledge/vector/chroma_store.py b/src/knowledge/vector/chroma_store.py index 03006ae..260a8e8 100644 --- a/src/knowledge/vector/chroma_store.py +++ b/src/knowledge/vector/chroma_store.py @@ -6,16 +6,14 @@ import os class ChromaVectorStore(VectorBase): - def __init__(self, model_name: str) -> None: + def __init__(self, root_dir, model_name: str) -> None: super().__init__(model_name) logging.info( "will init chroma vector store, model={}".format(model_name) ) - directory = os.path.join( - os.path.dirname(__file__), "../../../rootfs/data/vector" - ) + directory = os.path.join(root_dir, "vector") logging.info("will use vector store: {}".format(directory)) client = chromadb.PersistentClient( diff --git a/src/service/aios_shell/aios_shell.py b/src/service/aios_shell/aios_shell.py index f5b9dec..3577d28 100644 --- a/src/service/aios_shell/aios_shell.py +++ b/src/service/aios_shell/aios_shell.py @@ -22,8 +22,11 @@ from prompt_toolkit.styles import Style directory = os.path.dirname(__file__) sys.path.append(directory + '/../../') -from aios_kernel import AIOS_Version,UserConfigItem,AIStorage,Workflow,AIAgent,AgentMsg,AgentMsgStatus,ComputeKernel,OpenAI_ComputeNode,AIBus,AIChatSession,AgentTunnel,TelegramTunnel,CalenderEnvironment,Environment,EmailTunnel,LocalLlama_ComputeNode + import proxy +from aios_kernel import * + + sys.path.append(directory + '/../../component/') from agent_manager import AgentManager @@ -115,7 +118,8 @@ class AIOS_Shell: await AgentTunnel.load_all_tunnels_from_config(tunnel_config) except Exception as e: logger.warning(f"load tunnels config from {tunnels_config_path} failed!") - + + KnowledgePipline.get_instance().initial() return True @@ -163,8 +167,7 @@ class AIOS_Shell: tunnel_config[key] = user_input - return tunnel_config - + return tunnel_config async def append_tunnel_config(self,tunnel_config): user_data_dir = AIStorage.get_instance().get_myai_dir() @@ -179,6 +182,55 @@ class AIOS_Shell: except Exception as e: logger.warning(f"load tunnels config from {tunnels_config_path} failed!") + async def handle_knowledge_commands(self, args): + show_text = FormattedText([("class:title", "sub command not support!\n" + "/knowledge add email | dir\n" + "/knowledge journal [$topn]\n" + "/knowledge query $query\n")]) + if len(args) < 1: + return show_text + sub_cmd = args[0] + if sub_cmd == "add": + if len(args) < 2: + return show_text + if args[1] == "email": + config = dict() + for key, item in KnowledgeEmailSource.user_config_items(): + user_input = await try_get_input(f"{key} : {item}") + if user_input is None: + return show_text + config[key] = user_input + error = KnowledgePipline.get_instance().add_email_source(KnowledgeEmailSource(config)) + if error is not None: + return FormattedText([("class:title", f"/knowledge add email failed {error}\n")]) + else: + KnowledgePipline.get_instance().save_config() + if args[1] == "dir": + config = dict() + for key, item in KnowledgeDirSource.user_config_items(): + user_input = await try_get_input(f"{key} : {item}") + if user_input is None: + return show_text + config[key] = user_input + error = KnowledgePipline.get_instance().add_dir_source(KnowledgeDirSource(config)) + if error is not None: + return FormattedText([("class:title", f"/knowledge add dir failed {error}\n")]) + else: + KnowledgePipline.get_instance().save_config() + else: + return show_text + if sub_cmd == "journal": + topn = 10 if len(args) == 1 else int(args[1]) + journals = [str(journal) for journal in KnowledgePipline.get_instance().get_latest_journals(topn)] + print_formatted_text("\r\n".join(journals)) + if sub_cmd == "query": + if len(args) < 2: + return show_text + prompt = AgentPrompt() + prompt.messages.append({"role": "user", "content":" ".join(args[1:])}) + result = await KnowledgeBase().query_prompt(prompt) + print_formatted_text(result.as_str()) + async def call_func(self,func_name, args): match func_name: case 'send': @@ -221,6 +273,8 @@ class AIOS_Shell: show_text = FormattedText([("class:title", f"connect to {tunnel_target} success!")]) return show_text + case 'knowledge': + return await self.handle_knowledge_commands(args) case 'open': if len(args) >= 1: target_id = args[0] @@ -400,6 +454,9 @@ async def main(): '/history $num $offset', '/login $username', '/connect $target', + '/knowledge add email | dir', + '/knowledge journal [$topn]', + '/knowledge query $query' '/set_config $key', '/list_config', '/show',