From 348c3be2099789feb7f70ba38465abf5ef370eaa Mon Sep 17 00:00:00 2001 From: streetycat <305190374@qq.com> Date: Tue, 5 Dec 2023 04:02:02 +0000 Subject: [PATCH 1/4] fix incorrect determination of subprocess failure --- .../local_compute_node_builder/local_llama_node_builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py index d9bdcde..b2606c8 100644 --- a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py +++ b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py @@ -118,7 +118,7 @@ class ParameterExternParamsApplier: else: result = subprocess.run(['git', 'clone', llama_cpp_python_repo_url, llama_cpp_python_path], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) - if result.stderr: + if result.returncode: print_formatted_text(FormattedText([("class:warn", result.stderr)]), style = state.shell_style) while True: sel = await state.prompt_session.prompt_async(f"Update 'llama-cpp-python' failed, you can press 'r' to retry, or 'c' to continue with the current version.", style = state.shell_style) @@ -140,7 +140,7 @@ class ParameterExternParamsApplier: result = subprocess.run(['docker', 'rmi', docker_image], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) result = subprocess.run(['docker', 'build', '-t', docker_image, f"{llama_cpp_python_path}/docker/cuda_simple/"], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) - if result.stderr: + if result.returncode: print_formatted_text(FormattedText([("class:warn", result.stderr)]), style = state.shell_style) while True: sel = await state.prompt_session.prompt_async(f"Build the image failed, you can press 'r' to retry, or 'c' to continue with the current version.", style = state.shell_style) @@ -179,7 +179,7 @@ class ParameterExternParamsApplier: result = subprocess.run(run_options, stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True) - if result.stderr: + if result.returncode: print_formatted_text(FormattedText([("class:warn", result.stderr)]), style = state.shell_style) while True: sel = await state.prompt_session.prompt_async(f"Start the node service failed, you can press 'r' to retry, or 'a' to abort.", style = state.shell_style) From 9f2fd50ba80c0724b0eae47be08200f96e38bca8 Mon Sep 17 00:00:00 2001 From: streetycat <305190374@qq.com> Date: Wed, 6 Dec 2023 06:19:15 +0000 Subject: [PATCH 2/4] fix default port error --- .../local_compute_node_builder/local_llama_node_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py index b2606c8..9eef8cf 100644 --- a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py +++ b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py @@ -69,7 +69,7 @@ class ParameterNodeNameApplier: class ParameterPortApplier: async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: - if value is None or value == "0": + if value is None or value == "0" or value == "": value = str(random.randint(10000, 60000)) state.params["port"] = value @@ -165,7 +165,7 @@ class ParameterExternParamsApplier: run_options.extend([ '-p', f"{state.params['port']}:8000", '-v', f"{os.path.dirname(state.params['model_path'])}:/models", '-e', f"MODEL=/models/{os.path.basename(state.params['model_path'])}", - 'llama-cpp-python-cuda', + docker_image, 'python3', '-m', 'llama_cpp.server', '--n_gpu_layers', state.params["n_gpu_layers"], '--n_ctx', state.params["n_ctx"], From 29f02138a5929ebadca678d1f66adb223cd19165 Mon Sep 17 00:00:00 2001 From: streetycat <305190374@qq.com> Date: Wed, 6 Dec 2023 08:06:51 +0000 Subject: [PATCH 3/4] Fix: build the service with CUDA as CPU only for build failed without CUDA. --- .../local_compute_node_builder/local_llama_node_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py index 9eef8cf..690242b 100644 --- a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py +++ b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py @@ -100,7 +100,7 @@ class ParameterExternParamsApplier: gpu_options = [] state.next_step += 1 - if state.params["n_gpu_layers"] == "0": + if False and state.params["n_gpu_layers"] == "0": docker_image = "ghcr.io/abetlen/llama-cpp-python:latest" else: gpu_options = ["--gpus", "all"] From e2b6f051cbde53fa98ebae8c1205aa0161517f8a Mon Sep 17 00:00:00 2001 From: streetycat <305190374@qq.com> Date: Wed, 6 Dec 2023 08:51:23 +0000 Subject: [PATCH 4/4] Fix singleton and Classbase --- .../local_llama_node_builder.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py index 690242b..163d6de 100644 --- a/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py +++ b/src/service/aios_shell/local_compute_node_builder/local_llama_node_builder.py @@ -10,10 +10,10 @@ from prompt_toolkit.formatted_text import FormattedText from aios.storage.storage import AIStorage from aios import ComputeKernel from component.llama_node.local_llama_compute_node import LocalLlama_ComputeNode -from service.aios_shell.compute_node_config import ComputeNodeConfig +from compute_node_config import ComputeNodeConfig from .local_compute_node_builder import BuildParameter, BuilderState, LocalComputeNodeBuilder, ParameterApplier -class BuildParameterModelPath: +class BuildParameterModelPath(ParameterApplier): async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: if value: if os.path.exists(value): @@ -24,7 +24,7 @@ class BuildParameterModelPath: state.next_step += 1 -class BuildParameterModelUrl: +class BuildParameterModelUrl(ParameterApplier): async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: if value is None: value = "1" @@ -61,13 +61,13 @@ class BuildParameterModelUrl: except Exception as e: print_formatted_text(FormattedText([("class:error", f"Download model failed: {e}\nYou can retry it or select another one.")]), style = state.shell_style) -class ParameterNodeNameApplier: +class ParameterNodeNameApplier(ParameterApplier): async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: value = value or os.path.basename(state.params["model_path"]) state.params["node_name"] = value state.next_step += 1 -class ParameterPortApplier: +class ParameterPortApplier(ParameterApplier): async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: if value is None or value == "0" or value == "": value = str(random.randint(10000, 60000)) @@ -75,25 +75,25 @@ class ParameterPortApplier: state.params["port"] = value state.next_step += 1 -class ParameterNGpuLayersApplier: +class ParameterNGpuLayersApplier(ParameterApplier): async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: value = value or "83" state.params["n_gpu_layers"] = value state.next_step += 1 -class ParameterNCtxApplier: +class ParameterNCtxApplier(ParameterApplier): async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: value = value or "4096" state.params["n_ctx"] = value state.next_step += 1 -class ParameterChatFormatApplier: +class ParameterChatFormatApplier(ParameterApplier): async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: value = value or "llama-2" state.params["chat_format"] = value state.next_step += 1 -class ParameterExternParamsApplier: +class ParameterExternParamsApplier(ParameterApplier): async def apply(self, state: BuilderState, name: str, value: str or None = None) -> str or None: extern_params = value docker_image = ""