gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adjust the directory structure to prepare for merging into Master.

Browse Source
This commit is contained in:
Liu Zhicong
2023-09-27 11:40:46 -07:00
parent 5146bc1871
commit 030e4c4f52
115 changed files with 413 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PoC/agent_jarvis/jarvis/ai_agent/agent_factory.py
+12
View File
@@ -0,0 +1,12 @@
from jarvis import CFG
from jarvis.ai_agent.gpt_agent import GptAgent
from jarvis.ai_agent.webui_agent import WebuiAgent
from jarvis.ai_agent.base_agent import BaseAgent
from jarvis.functional_modules.functional_module import CallerContext
def create_agent(context: CallerContext) -> BaseAgent:
if CFG.use_private_ai:
return WebuiAgent(context)
else:
return GptAgent(context)
PoC/agent_jarvis/jarvis/ai_agent/agent_utils.py
+72
View File
@@ -0,0 +1,72 @@
import json
from typing import Dict
from jarvis.functional_modules.functional_module import CallerContext, moduleRegistry
from jarvis.logger import logger
def must_not_be_valid_json(s: str):
"""
Simply check if the string is a JSON.
If the string does not contain even 1 pair of '{}',
it must not be a JSON, we treat it as a normal string.
"""
if s.count('{') < 1 and s.count("{") < 1:
return True
return False
def get_thoughts(reply: Dict, assistant_reply_json_valid: dict):
assistant_thoughts_reasoning = None
assistant_thoughts_speak = None
assistant_thoughts = assistant_reply_json_valid.get("thoughts", {})
assistant_thoughts_text = assistant_thoughts.get("text")
if assistant_thoughts:
assistant_thoughts_reasoning = assistant_thoughts.get("reasoning")
assistant_thoughts_speak = assistant_thoughts.get("speak")
reply["thoughts"] = assistant_thoughts_text
reply["reasoning"] = assistant_thoughts_reasoning
reply["speak"] = assistant_thoughts_speak
logger.debug(f" THOUGHTS: {assistant_thoughts_text}")
logger.debug(f"REASONING: {assistant_thoughts_reasoning}")
logger.debug(f" SPEAKING: {assistant_thoughts_speak}")
def get_function(reply: Dict, response_json: Dict):
try:
if "function" not in response_json:
return "Error:", "Missing 'function' object in JSON"
if not isinstance(response_json, dict):
return "Error:", f"'response_json' object is not dictionary {response_json}"
function = response_json["function"]
if not isinstance(function, dict):
return "Error:", "'function' object is not a dictionary"
if "name" not in function:
return "Error:", "Missing 'name' field in 'function' object"
function_name = function["name"]
# Use an empty dictionary if 'args' field is not present in 'function' object
arguments = function.get("args", {})
reply["function"] = function_name
reply["arguments"] = arguments
return function_name, arguments
except json.decoder.JSONDecodeError:
return "Error:", "Invalid JSON"
except Exception as e:
return "Error:", str(e)
async def execute_function(
context: CallerContext,
function_name: str,
**arguments,
):
logger.debug(f"Executing function: {function_name}({arguments})")
await context.push_notification(f"Executing function: {function_name}({arguments})")
return await moduleRegistry.execute_function(context, function_name, **arguments)
PoC/agent_jarvis/jarvis/ai_agent/base_agent.py
+23
View File
@@ -0,0 +1,23 @@
from jarvis.functional_modules.functional_module import CallerContext
class BaseAgent:
_caller_context: CallerContext = None
def __init__(self, context: CallerContext):
self._caller_context = context
async def feed_prompt(self, prompt):
raise NotImplementedError("Not implemented")
def append_history_message(self, role: str, content: str):
raise NotImplementedError("Not implemented")
def clear_history_messages(self):
raise NotImplementedError("Not implemented")
def save_history(self, to_where):
raise NotImplementedError("Not implemented")
def load_history(self, from_where):
raise NotImplementedError("Not implemented")
PoC/agent_jarvis/jarvis/ai_agent/gpt_agent.py
+251
View File
@@ -0,0 +1,251 @@
import asyncio
import contextlib
import json
import time
from typing import Dict, List
from openai.error import RateLimitError
from jarvis import CFG
from jarvis.ai_agent.agent_utils import must_not_be_valid_json, get_thoughts, get_function, execute_function
from jarvis.ai_agent.base_agent import BaseAgent
from jarvis.functional_modules.functional_module import CallerContext, moduleRegistry
from jarvis.gpt import token_counter, gpt
from jarvis.json_utils.json_fix_llm import fix_json_using_multiple_techniques
from jarvis.json_utils.utilities import validate_json
from jarvis.logger import logger
def _generate_first_prompt():
return """I will ask you questions or ask you to do something. You should:
First, determine if you know the answer of the question or you can accomplish the task directly.
If so, response directly.
If not, try to complete the task by calling the functions below.
If you can't accomplish the task by yourself and no function is able to accomplish the task, say "Dear master, sorry, I'm not able to do that."
Your setup:
```
{
"author": "OpenDAN",
"name": "Jarvis",
}
```"""
class GptAgent(BaseAgent):
_system_prompt: str
_full_message_history: List[dict] = []
_message_tokens: List[int] = []
def __init__(self, caller_context: CallerContext):
super().__init__(caller_context)
self._system_prompt = _generate_first_prompt()
logger.debug(f"Using GptAgent, system prompt is: {self._system_prompt}")
logger.debug(f"{json.dumps(moduleRegistry.to_json_schema())}")
async def _feed_prompt_to_get_response(self, prompt):
reply_type, assistant_reply = await self._chat_with_ai(
self._system_prompt,
prompt,
CFG.token_limit,
)
if reply_type == "content":
return {
"speak": assistant_reply,
}
elif reply_type == "function_call":
arguments_string = assistant_reply["arguments"]
try:
arguments = json.loads(arguments_string)
except:
arguments = await fix_json_using_multiple_techniques()
return {
"function": assistant_reply["name"],
"arguments": arguments
}
async def feed_prompt(self, prompt):
# Send message to AI, get response
logger.debug(f"Trigger: {prompt}")
reply: Dict = None
for i in range(3):
try:
if i == 0:
reply = await self._feed_prompt_to_get_response(prompt)
else:
reply = await self._feed_prompt_to_get_response(
prompt + ". Remember to reply using the specified JSON form")
break
except Exception as e:
# TODO: Feed the error to ChatGPT?
logger.debug(f"Failed to get reply, try again! {str(e)}")
continue
if reply is None:
await self._caller_context.reply_text("Sorry, but I don't understand what you want me to do.")
return
# Execute function
function_name: str = reply.get("function")
if function_name is None:
await self._caller_context.reply_text(reply["speak"])
else:
arguments: Dict = reply["arguments"]
function_result = "Failed"
try:
function_result = await execute_function(self._caller_context, function_name, **arguments)
finally:
result = f"{function_result}"
# Check if there's a result from the function append it to the message
# history
if result is not None:
self.append_history_message_raw({"role": "function", "name": function_name, "content": result})
logger.debug(f"function: {result}")
else:
self.append_history_message_raw({"role": "function", "name": function_name, "content": "Unable to execute function"})
logger.debug("function: Unable to execute function")
def append_history_message(self, role: str, content: str):
self._full_message_history.append({'role': role, 'content': content})
self._message_tokens.append(-1)
def append_history_message_raw(self, msg: dict):
self._full_message_history.append(msg)
self._message_tokens.append(-1)
def clear_history_messages(self):
self._full_message_history.clear()
self._message_tokens.clear()
def save_history(self, to_where):
with open(to_where, "w") as f:
assert len(self._message_tokens) == len(self._full_message_history)
s = json.dumps([
self._message_tokens,
self._full_message_history,
])
f.write(s)
def load_history(self, from_where):
with contextlib.suppress(Exception):
with open(from_where, "r") as f:
tmp = json.loads(f.read())
if isinstance(tmp, list) and len(tmp[0]) == len(tmp[1]):
self._message_tokens = tmp[0]
self._full_message_history = tmp[1]
async def _chat_with_ai(
self, prompt, user_input, token_limit
):
"""Interact with the OpenAI API, sending the prompt, user input, message history,
and permanent memory."""
while True:
try:
model = CFG.llm_model
# Reserve 1000 tokens for the response
send_token_limit = token_limit - 1000
(
next_message_to_add_index,
current_tokens_used,
insertion_index,
current_context,
) = await self._generate_context(prompt, model)
current_tokens_used += await token_counter.count_message_tokens(
[{"role": "user", "content": user_input}], model
) # Account for user input (appended later)
# TODO: OpenAI does not say how to count function tokens, we use this method to roughly get the tokens count
# It's result looks much larger than OpenAI's result
current_tokens_used += await token_counter.count_message_tokens(
[{"role": "user", "content": json.dumps(moduleRegistry.to_json_schema())}], model
)
while next_message_to_add_index >= 0:
# print (f"CURRENT TOKENS USED: {current_tokens_used}")
tokens_to_add = await self._get_history_message_tokens(next_message_to_add_index, model)
if current_tokens_used + tokens_to_add > send_token_limit:
break
message_to_add = self._full_message_history[next_message_to_add_index]
# Add the most recent message to the start of the current context,
# after the two system prompts.
current_context.insert(insertion_index, message_to_add)
# Count the currently used tokens
current_tokens_used += tokens_to_add
# Move to the next most recent message in the full message history
next_message_to_add_index -= 1
# Append user input, the length of this is accounted for above
current_context.extend([{"role": "user", "content": user_input}])
# Calculate remaining tokens
tokens_remaining = token_limit - current_tokens_used
assert tokens_remaining >= 0
async def on_single_chat_timeout(will_retry):
await self._caller_context.push_notification(
f'Thinking timeout{", retry" if will_retry else ", give up"}.')
reply_type, assistant_reply = await gpt.acreate_chat_completion(
model=model,
messages=current_context,
temperature=CFG.temperature,
max_tokens=tokens_remaining,
on_single_request_timeout=on_single_chat_timeout,
functions=moduleRegistry.to_json_schema()
)
# Update full message history
if reply_type == "content":
self.append_history_message("user", user_input)
self.append_history_message("assistant", assistant_reply)
pass
elif reply_type == "function_call":
self.append_history_message("user", user_input)
self.append_history_message_raw({"role": "assistant", "function_call": assistant_reply, "content": None})
pass
else:
assert False, "Unexpected reply type"
return reply_type, assistant_reply
except RateLimitError:
# TODO: When we switch to langchain, or something else this is built in
print("Error: ", "API Rate Limit Reached. Waiting 10 seconds...")
await asyncio.sleep(10)
async def _generate_context(self, prompt, model):
# We use the timezone of the session
timestamp = time.time() + time.timezone + self._caller_context.get_tz_offset() * 3600
time_str = time.strftime('%c', time.localtime(timestamp))
current_context = [
{"role": "system", "content": prompt},
{"role": "system", "content": f"The current time and date is {time_str}"},
]
# Add messages from the full message history until we reach the token limit
next_message_to_add_index = len(self._full_message_history) - 1
insertion_index = len(current_context)
# Count the currently used tokens
current_tokens_used = await token_counter.count_message_tokens(current_context, model)
return (
next_message_to_add_index,
current_tokens_used,
insertion_index,
current_context,
)
async def _get_history_message_tokens(self, index, model: str = "gpt-3.5-turbo-0301") -> int:
if self._message_tokens[index] == -1:
# since couting token is relatively slow, we store it here
self._message_tokens[index] = await token_counter.count_message_tokens([self._full_message_history[index]], model)
return self._message_tokens[index]
PoC/agent_jarvis/jarvis/ai_agent/webui_agent.py
+208
View File
@@ -0,0 +1,208 @@
import contextlib
import json
import aiohttp
from jarvis import CFG
from jarvis.ai_agent.agent_utils import must_not_be_valid_json, get_thoughts, get_function, execute_function
from jarvis.ai_agent.base_agent import BaseAgent
from jarvis.functional_modules.functional_module import CallerContext, moduleRegistry
from jarvis.json_utils.json_fix_llm import fix_json_using_multiple_techniques
from jarvis.json_utils.utilities import validate_json
from jarvis.logger import logger
def _generate_system_prompt():
return """Since now, every your response should satisfy the following JSON format, a 'function' must be chosen:
```
{
"thoughts": {
"text": "<Your thought>",
"reasoning": "<Your reasoning, think step by step>",
"speak": "<what you want to say to me>"
},
"function": {
"name": "<mandatory, one of listed functions>",
"args": {
"arg name": "<value>"
}
}
}
```
I will ask you questions or ask you to do something. You should:
First, you should determine if you know the answer of the question or you can accomplish the task directly.
If so, you should response directly.
If not, you should try to complete the task by calling the functions below.
If you can't accomplish the task by yourself and no function is able to accomplish the task, say "Dear master, sorry, I'm not able to do that."
```
Available functions:
```
""" + moduleRegistry.to_prompt() + """
```
Your setup:
```
{
"author": "OpenDAN",
"name": "Jarvis",
}
Example:
```
Tom: generate a picture of me.
Jarvis: {
"function": {
"name": "stable_diffusion",
"args": {
"prompt": "me"
}
}
}
```
"""
def _generate_request(prompt: str):
return {
'prompt': prompt,
'max_new_tokens': 1000,
'do_sample': True,
'temperature': 0.5,
'top_p': 0.5,
'typical_p': 1,
'repetition_penalty': 1.18,
'top_k': 40,
'min_length': 0,
'no_repeat_ngram_size': 0,
'num_beams': 1,
'penalty_alpha': 0,
'length_penalty': 1,
'early_stopping': False,
'seed': -1,
'add_bos_token': True,
'truncation_length': 2048,
'ban_eos_token': False,
'skip_special_tokens': True,
'stopping_strings': ["Tom: "]
}
def _convert_role(role: str):
if role == 'user':
return 'Tom'
if role == 'assistant':
return 'Jarvis'
return role
async def _completion(prompt):
async with aiohttp.ClientSession() as session:
# body = json.dumps(_generate_request(prompt))
async with session.post(CFG.private_ai_address, json=_generate_request(prompt)) as response:
if response.status == 200:
resp_obj = await response.json()
logger.debug(f"Completion result: {json.dumps(resp_obj, indent=2)}")
result = resp_obj["results"][0]['text']
return result
return None
class WebuiAgent(BaseAgent):
_system_prompt: str
_history = []
def __init__(self, context: CallerContext):
super().__init__(context)
self._system_prompt = _generate_system_prompt()
async def feed_prompt(self, prompt):
prompt = f'Tom: {prompt}'
self._history.append(prompt)
final_prompt = self._system_prompt + '\n' + '\n'.join(self._history)
logger.debug(f"Final prompt: {final_prompt}")
reply = await self._feed_prompt_to_get_respones(final_prompt)
await self._handle_reply(reply)
async def _feed_prompt_to_get_respones(self, prompt):
assistant_reply = await _completion(prompt)
reply = {
"thoughts": None,
"reasoning": None,
"speak": None,
"function": None,
"arguments": None,
}
if must_not_be_valid_json(assistant_reply):
raise Exception(f"AI replied an invalid response: {assistant_reply}!")
else:
assistant_reply_json = await fix_json_using_multiple_techniques(assistant_reply)
# Print Assistant thoughts
if assistant_reply_json != {}:
validate_json(assistant_reply_json, "llm_response_format_1")
try:
get_thoughts(reply, assistant_reply_json)
get_function(reply, assistant_reply_json)
except Exception as e:
logger.error(f"AI replied an invalid response: {assistant_reply}. Error: {str(e)}")
raise e
else:
raise Exception(f"AI replied an invalid response: {assistant_reply}!")
function_name = reply["function"]
if function_name is None or function_name == '':
raise Exception(f"Missing a function")
arguments = reply["arguments"]
if not isinstance(arguments, dict):
raise Exception(f"Invalid arguments, it MUST be a dict")
return reply
async def _handle_reply(self, reply):
# TODO: It's not reliable now, thus do nothing now.
return
if reply is None:
await self._caller_context.reply_text("Sorry, but I don't understand what you want me to do.")
return
# Execute function
function_name: str = reply["function"]
arguments: dict = reply["arguments"]
await self._caller_context.reply_text(reply["speak"])
execute_error = None
try:
function_result = await execute_function(self._caller_context, function_name, **arguments)
except Exception as e:
function_result = "Failed"
execute_error = e
result = f"Function {function_name} returned: " f"{function_result}"
if function_name is not None:
if result is not None:
self.append_history_message("system", result)
logger.debug(f"SYSTEM: {result}")
else:
self.append_history_message("system", "Unable to execute function")
logger.debug("SYSTEM: Unable to execute function")
if execute_error is not None:
raise execute_error
def append_history_message(self, role: str, content: str):
self._history.append({'role': role, 'content': content})
def clear_history_messages(self):
self._history.clear()
def save_history(self, to_where):
with open(to_where, "w") as f:
s = json.dumps(self._history)
f.write(s)
def load_history(self, from_where):
with contextlib.suppress(Exception):
with open(from_where, "r") as f:
self._history = json.loads(f.read())