diff --git a/src/component/llama_node/local_llama_compute_node.py b/src/component/llama_node/local_llama_compute_node.py index 0a22369..efb9712 100644 --- a/src/component/llama_node/local_llama_compute_node.py +++ b/src/component/llama_node/local_llama_compute_node.py @@ -1,9 +1,6 @@ -import json import logging import requests -from typing import Optional, List -from pydantic import BaseModel from aios import ComputeTask,Queue_ComputeNode, ComputeTaskResult, ComputeTaskResultCode, ComputeTaskState, ComputeTaskType,AIStorage,UserConfig diff --git a/src/service/aios_shell/local_compute_node_builder/__init__.py b/src/service/aios_shell/local_compute_node_builder/__init__.py index cb84ca9..4fb1d16 100644 --- a/src/service/aios_shell/local_compute_node_builder/__init__.py +++ b/src/service/aios_shell/local_compute_node_builder/__init__.py @@ -1,13 +1,19 @@ -from prompt_toolkit import PromptSession +import os +from prompt_toolkit import HTML, PromptSession, print_formatted_text from prompt_toolkit.styles import Style from service.aios_shell.local_compute_node_builder.local_llama_node_builder import LocalLlamaNodeBuilder from .local_compute_node_builder import BuilderState, LocalComputeNodeBuilder +from aios_kernel.storage import AIStorage async def build(prompt_session: PromptSession, shell_style: Style) -> str or None: # model_type = await prompt_session.prompt_async(f"Please select the node server type (default: llama.cpp):", style = shell_style) model_type = 'llama.cpp' + download_dir = AIStorage.get_instance().get_download_dir() + if not os.path.exists(download_dir): + os.mkdir(download_dir) + state = BuilderState(prompt_session, shell_style) match model_type: @@ -19,7 +25,9 @@ async def build(prompt_session: PromptSession, shell_style: Style) -> str or Non if param is None: return None - value = await state.prompt_session.prompt_async(f"{state.last_result_prompt}{param.desc}:", style = state.shell_style) + if state.last_result_prompt or param.desc: + print_formatted_text(f"{state.last_result_prompt}{param.desc}", style = state.shell_style) + value = await state.prompt_session.prompt_async(f"{param.prompt}:", style = state.shell_style) if value: value = value.strip() diff --git a/src/service/aios_shell/local_compute_node_builder/local_compute_node_builder.py b/src/service/aios_shell/local_compute_node_builder/local_compute_node_builder.py index 7d9bc3d..33e1e36 100644 --- a/src/service/aios_shell/local_compute_node_builder/local_compute_node_builder.py +++ b/src/service/aios_shell/local_compute_node_builder/local_compute_node_builder.py @@ -24,8 +24,9 @@ class ParameterApplier: pass class BuildParameter: - def __init__(self, name: str, applier: ParameterApplier, desc: str or None = None, default_value: str or None = None): + def __init__(self, name: str, applier: ParameterApplier, prompt: str or None = None, desc: str or None = None, default_value: str or None = None): self.name = name + self.prompt = prompt self.desc = desc self.default_value = default_value self.applier = applier diff --git a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py index 25b7b14..5d818af 100644 --- a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py +++ b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py @@ -97,12 +97,13 @@ class ParameterExternParamsApplier: async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: extern_params = value docker_image = "" - gpu_options = None + gpu_options = [] + state.next_step += 1 if state.params["n_gpu_layers"] == "0": docker_image = "ghcr.io/abetlen/llama-cpp-python:latest" else: - gpu_options = "--gpus all" + gpu_options = ["--gpus", "all"] llama_cpp_python_repo_url = "https://github.com/abetlen/llama-cpp-python.git" download_path = AIStorage.get_instance().get_download_dir() llama_cpp_python_path = download_path + "/llama-cpp-python" @@ -115,9 +116,10 @@ class ParameterExternParamsApplier: if os.path.exists(llama_cpp_python_path): result = subprocess.run(['git', 'pull'], cwd = llama_cpp_python_path, stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) else: - result = subprocess.run(['git', 'clone', llama_cpp_python_repo_url, download_path], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) + result = subprocess.run(['git', 'clone', llama_cpp_python_repo_url, llama_cpp_python_path], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) if result.stderr: + print_formatted_text(FormattedText([("class:warn", result.stderr)]), style = state.shell_style) while True: sel = await state.prompt_session.prompt_async(f"Update 'llama-cpp-python' failed, you can press 'r' to retry, or 'c' to continue with the current version.", style = state.shell_style) if sel == 'r': @@ -139,6 +141,7 @@ class ParameterExternParamsApplier: result = subprocess.run(['docker', 'build', '-t', docker_image, f"{llama_cpp_python_path}/docker/cuda_simple/"], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) if result.stderr: + print_formatted_text(FormattedText([("class:warn", result.stderr)]), style = state.shell_style) while True: sel = await state.prompt_session.prompt_async(f"Build the image failed, you can press 'r' to retry, or 'c' to continue with the current version.", style = state.shell_style) if sel == 'r': @@ -151,89 +154,92 @@ class ParameterExternParamsApplier: else: break - retry = True - while True: - retry = False - run_options = ['docker', 'run', '-d'] + retry = True + while retry: + retry = False + run_options = ['docker', 'run', '-d'] - if gpu_options: - run_options.append(gpu_options) - - run_options.extend([ - '-p', f"{state.params['port']}:8000", - '-v', f"{os.path.dirname(state.params['model_path'])}:/models", '-e', f"MODEL=/models/{os.path.basename(state.params['model_path'])}", - 'llama-cpp-python-cuda', - 'python3', '-m', 'llama_cpp.server', - '--n_gpu_layers', state.params["n_gpu_layers"], - '--n_ctx', state.params["n_ctx"], - '--chat_format', state.params["chat_format"], - ]) - - if extern_params: - run_options.extend(extern_params.split(' ')) + if gpu_options: + run_options.extend(gpu_options) + + run_options.extend([ + '-p', f"{state.params['port']}:8000", + '-v', f"{os.path.dirname(state.params['model_path'])}:/models", '-e', f"MODEL=/models/{os.path.basename(state.params['model_path'])}", + 'llama-cpp-python-cuda', + 'python3', '-m', 'llama_cpp.server', + '--n_gpu_layers', state.params["n_gpu_layers"], + '--n_ctx', state.params["n_ctx"], + '--chat_format', state.params["chat_format"], + ]) + + if extern_params: + run_options.extend(extern_params.split(' ')) - result = subprocess.run(run_options, stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) + print_formatted_text(FormattedText([("class:prompt", f"Will start service with: {' '.join(run_options)}")]), style = state.shell_style) - if result.stderr: - while True: - sel = await state.prompt_session.prompt_async(f"Start the node service failed, you can press 'r' to retry, or 'a' to abort.", style = state.shell_style) - if sel == 'r': - retry = True - break - elif sel == 'a': - break - else: - pass # Select again - else: - local_url = f'http://localhost:{state.params["port"]}' - foreign_url = 'http://{your-host-address}:' + state.params["port"] - model_name = state.params['node_name'] + result = subprocess.run(run_options, stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) - ComputeNodeConfig.get_instance().add_node("llama", local_url, model_name) - ComputeNodeConfig.get_instance().save() - node = LocalLlama_ComputeNode(local_url, model_name) - node.start() - ComputeKernel.get_instance().add_compute_node(node) + if result.stderr: + print_formatted_text(FormattedText([("class:warn", result.stderr)]), style = state.shell_style) + while True: + sel = await state.prompt_session.prompt_async(f"Start the node service failed, you can press 'r' to retry, or 'a' to abort.", style = state.shell_style) + if sel == 'r': + retry = True + break + elif sel == 'a': + break + else: + pass # Select again + else: + local_url = f'http://localhost:{state.params["port"]}' + foreign_url = 'http://{your-host-address}:' + state.params["port"] + model_name = state.params['node_name'] + + ComputeNodeConfig.get_instance().add_node("llama", local_url, model_name) + ComputeNodeConfig.get_instance().save() + node = LocalLlama_ComputeNode(local_url, model_name) + node.start() + ComputeKernel.get_instance().add_compute_node(node) + + print_formatted_text(FormattedText([( + "class:prompt", +f""" +Congratulations! The node ({model_name}) service successed. +You can access it with follow url: +{local_url} +And 'http://{foreign_url}' in other computers. +Now you can refer it in agents as `llm_model_name={model_name}` +""" + )]), style = state.shell_style) + break - print_formatted_text(FormattedText([( - "class:prompt", - f""" - Congratulations! The node ({model_name}) service successed. - You can access it with follow url: - {local_url} - And 'http://{foreign_url}' in other computers. - Now you can refer it in agents as `llm_model_name={model_name}` - """ - )]), style = state.shell_style) - break - _recommend_model_urls = { "1": { - "model": "Llama-2-70B-chat", + "model": "Llama-2-70B-Chat-GGUF", "url": "https://huggingface.co/TheBloke/Llama-2-70B-chat-GGUF/resolve/main/llama-2-70b-chat.Q4_0.gguf" }, "2": { - "model": "Llama-2-13B-chat", + "model": "Llama-2-13B-Chat-GGUF", "url": "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGUF/resolve/main/llama-2-13b-chat.Q4_0.gguf" }, "3": { "model": "Llama-2-7B-Chat-GGUF", - "url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/blob/main/llama-2-7b-chat.Q4_K_M.gguf" + "url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf" }, } -_recommend_model_url_table_str = map(lambda id, info: f"\t{id}\t{info['model']}\t{info['url']}\n", _recommend_model_urls) +_recommend_model_url_table_str = "" +for i in range(1, 999): + id = str(i) + info = _recommend_model_urls.get(id) + if info: + _recommend_model_url_table_str += f"\n\t{id}\t{info['model']}\t{info['url']}" + else: + break _params = [ BuildParameter("model_path", BuildParameterModelPath(), "Please input the model file path (Press 'Enter' if you need to download it)"), - BuildParameter("model_url", BuildParameterModelUrl(), - f""" - Please input the url to download the model, or you can input the 'ID' in the follow table to select one: - ID\tmodel\turl - {_recommend_model_url_table_str} - Please input (default: Llama-2-70B-chat) - """ - ), + BuildParameter("model_url", BuildParameterModelUrl(), "Please input (default: Llama-2-70B-chat)", f"Now you need input the url to download the model, or you can input the 'ID' in the follow table to select one:\n\tID\tmodel\t\turl{_recommend_model_url_table_str}"), BuildParameter("node_name", ParameterNodeNameApplier(), "Please input name for your node, and you can set it in 'llm_model_name' of 'agent.toml' (default: the name of the model file)"), BuildParameter("port", ParameterPortApplier(), "Please input the port which the node server will listen on (default: random)"), BuildParameter("n_gpu_layers", ParameterNGpuLayersApplier(), "Please input layers offload to GPU (<=83 for Llama, 0 for CPU only, default: 83)"),