gyt/opendan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix bugs for interactive of /node add

Browse Source
This commit is contained in:
streetycat
2023-12-04 08:15:28 +00:00
committed by zhangzhen
parent a001706752
commit 15046f0254
4 changed files with 84 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/component/llama_node/local_llama_compute_node.py
-3
View File
@@ -1,9 +1,6 @@
import json
import logging
import requests
from typing import Optional, List
from pydantic import BaseModel
from aios import ComputeTask,Queue_ComputeNode, ComputeTaskResult, ComputeTaskResultCode, ComputeTaskState, ComputeTaskType,AIStorage,UserConfig
src/service/aios_shell/local_compute_node_builder/__init__.py
+10 -2
View File
@@ -1,13 +1,19 @@
from prompt_toolkit import PromptSession
import os
from prompt_toolkit import HTML, PromptSession, print_formatted_text
from prompt_toolkit.styles import Style
from service.aios_shell.local_compute_node_builder.local_llama_node_builder import LocalLlamaNodeBuilder
from .local_compute_node_builder import BuilderState, LocalComputeNodeBuilder
from aios_kernel.storage import AIStorage
async def build(prompt_session: PromptSession, shell_style: Style) -> str or None:
# model_type = await prompt_session.prompt_async(f"Please select the node server type (default: llama.cpp):", style = shell_style)
model_type = 'llama.cpp'
download_dir = AIStorage.get_instance().get_download_dir()
if not os.path.exists(download_dir):
os.mkdir(download_dir)
state = BuilderState(prompt_session, shell_style)
match model_type:
@@ -19,7 +25,9 @@ async def build(prompt_session: PromptSession, shell_style: Style) -> str or Non
if param is None:
return None
value = await state.prompt_session.prompt_async(f"{state.last_result_prompt}{param.desc}:", style = state.shell_style)
if state.last_result_prompt or param.desc:
print_formatted_text(f"{state.last_result_prompt}{param.desc}", style = state.shell_style)
value = await state.prompt_session.prompt_async(f"{param.prompt}:", style = state.shell_style)
if value:
value = value.strip()
src/service/aios_shell/local_compute_node_builder/local_compute_node_builder.py
+2 -1
View File
@@ -24,8 +24,9 @@ class ParameterApplier:
pass
class BuildParameter:
def __init__(self, name: str, applier: ParameterApplier, desc: str or None = None, default_value: str or None = None):
def __init__(self, name: str, applier: ParameterApplier, prompt: str or None = None, desc: str or None = None, default_value: str or None = None):
self.name = name
self.prompt = prompt
self.desc = desc
self.default_value = default_value
self.applier = applier
src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py
+23 -17
View File
@@ -97,12 +97,13 @@ class ParameterExternParamsApplier:
async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None:
extern_params = value
docker_image = ""
gpu_options = None
gpu_options = []
state.next_step += 1
if state.params["n_gpu_layers"] == "0":
docker_image = "ghcr.io/abetlen/llama-cpp-python:latest"
else:
gpu_options = "--gpus all"
gpu_options = ["--gpus", "all"]
llama_cpp_python_repo_url = "https://github.com/abetlen/llama-cpp-python.git"
download_path = AIStorage.get_instance().get_download_dir()
llama_cpp_python_path = download_path + "/llama-cpp-python"
@@ -115,9 +116,10 @@ class ParameterExternParamsApplier:
if os.path.exists(llama_cpp_python_path):
result = subprocess.run(['git', 'pull'], cwd = llama_cpp_python_path, stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
else:
result = subprocess.run(['git', 'clone', llama_cpp_python_repo_url, download_path], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
result = subprocess.run(['git', 'clone', llama_cpp_python_repo_url, llama_cpp_python_path], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
if result.stderr:
print_formatted_text(FormattedText([("class:warn", result.stderr)]), style = state.shell_style)
while True:
sel = await state.prompt_session.prompt_async(f"Update 'llama-cpp-python' failed, you can press 'r' to retry, or 'c' to continue with the current version.", style = state.shell_style)
if sel == 'r':
@@ -139,6 +141,7 @@ class ParameterExternParamsApplier:
result = subprocess.run(['docker', 'build', '-t', docker_image, f"{llama_cpp_python_path}/docker/cuda_simple/"], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
if result.stderr:
print_formatted_text(FormattedText([("class:warn", result.stderr)]), style = state.shell_style)
while True:
sel = await state.prompt_session.prompt_async(f"Build the image failed, you can press 'r' to retry, or 'c' to continue with the current version.", style = state.shell_style)
if sel == 'r':
@@ -152,12 +155,12 @@ class ParameterExternParamsApplier:
break
retry = True
while True:
while retry:
retry = False
run_options = ['docker', 'run', '-d']
if gpu_options:
run_options.append(gpu_options)
run_options.extend(gpu_options)
run_options.extend([
'-p', f"{state.params['port']}:8000",
@@ -172,9 +175,12 @@ class ParameterExternParamsApplier:
if extern_params:
run_options.extend(extern_params.split(' '))
print_formatted_text(FormattedText([("class:prompt", f"Will start service with: {' '.join(run_options)}")]), style = state.shell_style)
result = subprocess.run(run_options, stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
if result.stderr:
print_formatted_text(FormattedText([("class:warn", result.stderr)]), style = state.shell_style)
while True:
sel = await state.prompt_session.prompt_async(f"Start the node service failed, you can press 'r' to retry, or 'a' to abort.", style = state.shell_style)
if sel == 'r':
@@ -209,31 +215,31 @@ class ParameterExternParamsApplier:
_recommend_model_urls = {
"1": {
"model": "Llama-2-70B-chat",
"model": "Llama-2-70B-Chat-GGUF",
"url": "https://huggingface.co/TheBloke/Llama-2-70B-chat-GGUF/resolve/main/llama-2-70b-chat.Q4_0.gguf"
},
"2": {
"model": "Llama-2-13B-chat",
"model": "Llama-2-13B-Chat-GGUF",
"url": "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGUF/resolve/main/llama-2-13b-chat.Q4_0.gguf"
},
"3": {
"model": "Llama-2-7B-Chat-GGUF",
"url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/blob/main/llama-2-7b-chat.Q4_K_M.gguf"
"url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf"
},
}
_recommend_model_url_table_str = map(lambda id, info: f"\t{id}\t{info['model']}\t{info['url']}\n", _recommend_model_urls)
_recommend_model_url_table_str = ""
for i in range(1, 999):
id = str(i)
info = _recommend_model_urls.get(id)
if info:
_recommend_model_url_table_str += f"\n\t{id}\t{info['model']}\t{info['url']}"
else:
break
_params = [
BuildParameter("model_path", BuildParameterModelPath(), "Please input the model file path (Press 'Enter' if you need to download it)"),
BuildParameter("model_url", BuildParameterModelUrl(),
f"""
Please input the url to download the model, or you can input the 'ID' in the follow table to select one:
ID\tmodel\turl
{_recommend_model_url_table_str}
Please input (default: Llama-2-70B-chat)
"""
),
BuildParameter("model_url", BuildParameterModelUrl(), "Please input (default: Llama-2-70B-chat)", f"Now you need input the url to download the model, or you can input the 'ID' in the follow table to select one:\n\tID\tmodel\t\turl{_recommend_model_url_table_str}"),
BuildParameter("node_name", ParameterNodeNameApplier(), "Please input name for your node, and you can set it in 'llm_model_name' of 'agent.toml' (default: the name of the model file)"),
BuildParameter("port", ParameterPortApplier(), "Please input the port which the node server will listen on (default: random)"),
BuildParameter("n_gpu_layers", ParameterNGpuLayersApplier(), "Please input layers offload to GPU (<=83 for Llama, 0 for CPU only, default: 83)"),