diff --git a/.env.example b/.env.example index 569feba..d3bb4fa 100644 --- a/.env.example +++ b/.env.example @@ -104,6 +104,7 @@ ENCRYPTION_KEY=your-super-secret-key-3 # SUPABASE_HOSTNAME=supabase.yourdomain.com # LANGFUSE_HOSTNAME=langfuse.yourdomain.com # OLLAMA_HOSTNAME=ollama.yourdomain.com +# LLAMACPP_HOSTNAME=llamacpp.yourdomain.com # SEARXNG_HOSTNAME=searxng.yourdomain.com # NEO4J_HOSTNAME=neo4j.yourdomain.com # LETSENCRYPT_EMAIL=internal @@ -121,9 +122,10 @@ ENCRYPTION_KEY=your-super-secret-key-3 # n8n ############ +N8N_PORT=5678 + # Set as N8N_HOST=${N8N_HOSTNAME:-${N8N_HOST}} in compose.yaml N8N_HOST=localhost -N8N_PORT=5678 # Permissions 0644 for n8n settings file /home/node/.n8n/config are too wide. N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=true @@ -164,8 +166,6 @@ N8N_RESERVE_MEMORY=1G # Time GENERIC_TIMEZONE=France/Paris -PG_META_CRYPTO_KEY - ############ # PostgreSQL - You can change these to any PostgreSQL database that has logical replication enabled. ############ @@ -180,8 +180,8 @@ POSTGRES_PORT=5432 # Projects ############ -# Omitted '=' to dynamically set using '~' prefix on Compose 'up' -PROJECTS_PATH +# Omitted '' to dynamically set using '~' prefix on Compose 'up' +PROJECTS_PATH= ############ # Document Processing Settings @@ -199,35 +199,22 @@ MAX_CHUNK_SIZE=40000 # Larger chunks for CAG (character count) CHUNK_OVERLAP=500 # Overlap between chunks if needed MIN_DOCUMENT_SIZE_FOR_CHUNKING=100000 # Only chunk very large documents - -# Omit '=' to return 'None' vs empty 'str' when not set -LLAMA_PATH - ############ -# Ollama - Inferencce engine and model manager +# Ollama - LLM ############ -# When running OLLAMA in the Host and N8N in Docker: +OLLAMA_PORT=11434 + +# When running OLLAMA in the Host: #OLLAMA_HOST=host.docker.internal:11434 -OLLAMA_HOST=ollama:11434 - -# Ollama URL for Open-WebUI backend connect -# The path '/ollama' will be redirected to the specified backend URL. -# When running OLLAMA in the Host and Open-WebUI in Docker: -#OLLAMA_BASE_URL=http://host.docker.internal:11434 -OLLAMA_BASE_URL=http://localhost:11434 - -# Set if conecting to Ollama using OpenAI API connection -#OPENAI_API_BASE_URL=http://localhost:11434 -# When running OLLAMA in the Host and Open-WebUI in Docker: -#OPENAI_API_BASE_URL=http://host.docker.internal:11434 -OPENAI_API_BASE_URL= -# OpenAI API key OPENAI_API_KEY declared below at Studio +# When running OLLAMA in Docker: +#OLLAMA_HOST=ollama:11434 +OLLAMA_HOST=host.docker.internal:11434 # Tuning OLLAMA_CONTEXT_LENGTH=8192 OLLAMA_FLASH_ATTENTION=1 -OLLAMA_KV_CACHE_TYPE=q8_0 +OLLAMA_KV_CACHE_TYPE=q4_0 OLLAMA_MAX_LOADED_MODELS=2 # Models @@ -235,9 +222,81 @@ OLLAMA_DEFAULT_MODEL=llama3.2 OLLAMA_SUPPLEMENT_MODEL=qwen3:8b OLLAMA_EMBEDDING_MODEL=nomic-embed-text -# Ollama server arguments (exclude model file) - use 'ollama serve --help' for available arguments +# Ollama server arguments - use ollama serve --help for available 'serve' arguments OLLAMA_SERVER_ARGS=serve +############ +# llama.cpp - LLM +############ + +LLAMA_ARG_PORT=8040 + +# When running LLAMA.CPP in the host: +#LLAMA_ARG_HOST=host.docker.internal +# When running LLAMA.CPP in Docker: +#LLAMA_ARG_HOST=0.0.0.0 +LLAMA_ARG_HOST=0.0.0.0 + +# Backend connect +LLAMACPP_HOST=${LLAMA_ARG_HOST}:${LLAMA_ARG_PORT} + +# Model names - Dictionary keys for model download identifier values below. +# Keys, and values below include an empty slot for a user-defined model +LLAMACPP_MODEL_GEMMA=gemma-4b # Default +LLAMACPP_MODEL_DEEPSEEK=deepseek-7b +LLAMACPP_MODEL_MISTRAL=mistral-7b +LLAMACPP_MODEL_LLAMA=llama-8b +LLAMACPP_MODEL_QWEN=qwen-8b +LLAMACPP_MODEL_USER= + +# Model download identifier - Dictionary values for model keys above. +# Model selected by 'best match' to LLAMACPP_MODEL_NAME +# To specify a local model, change '-hf' to '-m' in LLAMACPP_SERVER_ARGS below +# and replace the respective model id value below with 'models/'. +LLAMACPP_MODEL_GEMMA_ID=ggml-org/gemma-3-4b-it-GGUF +LLAMACPP_MODEL_DEEPSEEK_ID=mradermacher/DeepSeek-R1-Distill-Qwen-7B-Uncensored-i1-GGUF +LLAMACPP_MODEL_MISTRAL_ID=bartowski/mistralai_Ministral-3-8B-Instruct-2512-GGUF +LLAMACPP_MODEL_LLAMA_ID=bartowski/allura-forge_Llama-3.3-8B-Instruct-GGUF +LLAMACPP_MODEL_QWEN_ID=bartowski/Qwen_Qwen3-8B-GGUF +LLAMACPP_MODEL_USER_ID= + +# Model and paths +LLAMACPP_PATH=llama.cpp +LLAMACPP_MODEL_NAME=${LLAMACPP_MODEL_GEMMA} # IMPORTANT: should reasonably match Dictionary model name above. +LLAMACPP_MODELS_DIR=${LLAMACPP_PATH}/models +LLAMACPP_MODEL_PATH=${LLAMACPP_MODELS_DIR}/${LLAMACPP_MODEL_NAME} + +# Model management - automatically download specified model if not downloaded. +LLAMA_ARG_HF_REPO=${LLAMACPP_MODEL_GEMMA_ID} + +# Tuning +LLAMA_ARG_CTX_SIZE=8192 +LLAMA_ARG_FLASH_ATTN=1 +LLAMA_ARG_N_GPU_LAYERS=0 +LLAMA_ARG_THREADS=4 +LLAMA_ARG_MODELS_MAX=4 + +# Llama.cpp server arguments - use 'llama-server --help' for available arguments +# To specify a local model, append '-m' or '––model'. +# To auto-download model (if not already downloaded) and if LLAMA_ARG_HF_REPO is +# not used (commented), append '-hf' or '--hf-file'. +LLAMACPP_SERVER_ARGS=--jinja + +############ +# LLAMA (Ollama/Llama.cpp) - Shared environment variables +############ + +# Application Installation path +# Set for Llama.cpp or if using custom Ollama installation path +# e.g. LLAMA_PATH=~\Projects\ai-suite\llama.cpp\bin\llama-server.exe +# Omit '' to return 'False' when queried +LLAMA_PATH= + +# Conecting to LLAMA using OpenAI API connection +# When running OLLAMA: ${OLLAMA_HOST} +# When running LLAMA.CPP: ${LLAMACPP_HOST} +OPENAI_API_BASE_URL=${OLLAMA_HOST} +#OPENAI_API_KEY - OpenAI API key declared below at Studio ############ # Open-WebUI @@ -253,8 +312,8 @@ CORS_ALLOW_ORIGIN='*' FORWARDED_ALLOW_IPS='*' # If running Open WebUI in an offline environment, set to 1 to prevent attempts to download models from the internet. -# Omitted '=' to return 'None' vs empty 'str' when not set -HF_HUB_OFFLINE +# Omit '' to return 'False' when queried +HF_HUB_OFFLINE= # AUTOMATIC1111_BASE_URL="http://localhost:7860" @@ -274,8 +333,8 @@ MCPO_HOT_RELOAD=true DOCKER_SOCKET_LOCATION=/var/run/docker.sock # Set Docker Compose ignore orphans to 'true' for locally built comtainers -# Omitted '=' to return 'None' vs empty 'str' when not set -COMPOSE_IGNORE_ORPHANS +# Omit '' to return 'False' when queried +COMPOSE_IGNORE_ORPHANS= ############ # Google @@ -308,8 +367,12 @@ GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER POOLER_PROXY_PORT_TRANSACTION=6543 POOLER_DEFAULT_POOL_SIZE=20 POOLER_MAX_CLIENT_CONN=100 +# Must be at least 64 characters; generate with 'openssl rand -base64 48' SECRET_KEY_BASE=UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq +# Must be exactly 32 characters; generate with 'openssl rand -hex 16' VAULT_ENC_KEY=your-32-character-encryption-key +# Must be at least 32 characters; generate with 'openssl rand -base64 24' +PG_META_CRYPTO_KEY=your-super-secret—long-encryption-key # Pool size for internal metadata storage used by Supavisor # This is separate from client connections and used only by Supavisor itself POOLER_DB_POOL_SIZE=5 @@ -395,6 +458,7 @@ FUNCTIONS_VERIFY_JWT=false ############ # Change vector.toml sinks to reflect this change -# these cannot be the same value +# These cannot be the same value +# Must be at least 32 characters; generate with 'openssl rand -base64 24' LOGFLARE_PUBLIC_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-public LOGFLARE_PRIVATE_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-private diff --git a/.gitignore b/.gitignore index db863ac..5247f3a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ supabase/ searxng/ neo4j/ caddy/ +llama.cpp/ +ollama/ opencode/* !opencode/opencode.jsonc !opencode/run_opencode_docker.py diff --git a/README.md b/README.md index a41552a..1dced35 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,7 @@ Before you begin, make sure you have the following software installed: # SUPABASE_HOSTNAME=supabase.yourdomain.com # LANGFUSE_HOSTNAME=langfuse.yourdomain.com # OLLAMA_HOSTNAME=ollama.yourdomain.com + # LLAMACPP_HOSTNAME=llama.cpp.yourdomain.com # SEARXNG_HOSTNAME=searxng.yourdomain.com # NEO4J_HOSTNAME=neo4j.yourdomain.com # LETSENCRYPT_EMAIL=internal @@ -245,22 +246,23 @@ Before you begin, make sure you have the following software installed: --- **AI-Suite** uses the `suite_services.py` script for the _installation_ command -that handles the AI-Suite functional module selection, Ollama GPU configuration, -and starting Supabase when specified. +that handles the AI-Suite functional module selection, llama CPU/GPU configuration, +and starting Supabase and Open WebUI Filesystem when specified. -Additionally, This script is also used for operation commands that _start_, _stop_, -_pause_ and _unpause_ the AI-Suite services using the optional `--operation` argument. -An Ollama check is performed when it is assumed Ollama is being run from the Docker -Host. If Ollama is determined to be installed but not running, an attempt to launch -the Ollama service is executed on _install_, _start_ and _unpause_. The check -will also attempt to _stop_ the Ollama service (in addition to stopping the -AI-Suite services) when the _stop-ollama_ operational command is specified. +This script is also used for operation commands that _start_, _stop_, _stop-llama_, +_pause_, _unpause_, _update_ and _install_ the AI-Suite services using the optional +`--operation` argument. A llama (Ollama/Llama.cpp) check is performed when it is +assumed llama is running from the Docker Host. If llama is determined to be installed +but not running, an attempt to launch the Ollama/Llama.cpp service is executed +on _install_, _start_ and _unpause_. The check will also attempt to _stop_ the +running llama service (in addition to stopping the AI-Suite services) when the +_stop-llama_ operational command is specified. Both installation and operation commands utilize the optional `--profile` -arguments to specify which AI-Suite functional modules and which Ollama CPU/GPU +arguments to specify which AI-Suite functional modules and which llama CPU/GPU configuration to use. When no functional profile argument is specified, the -default functional module `open-webui` is used, Likewise, if no GPU configuration -profile is specified, it is assumed Ollama is being run from the Docker Host. +default functional module `open-webui` is used, Likewise, if no CPU/GPU configuration +profile is specified, it is assumed llama is being run from the Docker Host. **Multiple profile arguments (functional modules) are supported**. The `--environment` command allows the installation to be defined as _private_ @@ -286,9 +288,17 @@ to the network. | `n8n-all` | n8n - complete bundle | | `open-webui-all` | Open WebUI - complete bundle | | `ai-all` | AI-Suite full stack - all modules | + +`suite_services.py` `--profile` llama CPU/GPU argument: + +| Argument | Llama CPU/GPU | +| -----------------------: | ------: | | `cpu` | Ollama - run on CPU | | `gpu-nvidia` | Ollama - run on Nvidia GPU | | `gpu-amd` | Ollama - run on AMD GPU | +| `cpp-cpu` | Llama.cpp - run on CPU | +| `cpp-gpu-nvidia` | Llama.cpp - run on Nvidia GPU | +| `cpp-gpu-amd` | Llama.cpp - run on AMD GPU | Example command: @@ -298,13 +308,13 @@ python suite_services.py --profile n8n opencode gpu-nvidia --- -`suite_services.py` ... `--operation` arguments: +`suite_services.py` ... `--operation` argument: | Argument | Operation | | -----------------------: | ------: | | `start` | Start - start the previously stopped, specified profile containers | | `stop` | Stop - shut down the specified profile containers | -| `stop-ollama` | Stop Ollama - perform stop plus shut down Ollama on the Host | +| `stop-llama` | Stop - perform `stop` and shut down Ollama/Llama.cpp on Host | | `pause` | Pause - pause the specified profile containers | | `unpause` | Unpause - unpause the previously paused profile containers | @@ -816,12 +826,28 @@ modules described above: > path `/root/projects` and the resulting path is set as _work_dir_ to form the > OpenCode Docker exec command's _workdir=work_dir_ keyword argument. -### Ollama - running on host +### Ollama or Llama.cpp - running on host -- **OLLAMA_PATH environment variable** +- **LLAMA_PATH environment variable** - - If Ollama is installed in a non-standard location, you can add `OLLAMA_PATH` - with its absolute path (including the Ollama file) to the _.env_ file. + - If _Ollama_ is installed in a custom location or you are using _Llama.cpp_, + Add `LLAMA_PATH` with its absolute path (including the file name) to your + _.env_ file. + +- **OLLAMA_SERVER_ARGS environment variable** + + - Add _OLLAMA_SERVER_ARGS_ with additional Ollama server process start arguments + to your _.env_ file. + +- **LLAMACPP_MODELS_DIR environment variable** + + - If you are using _Llama.cpp_ with models that were **not** downloaded with + that instance of _Llama.cpp_, add `LLAMACPP_MODELS_DIR` with said models path + to your _.env_ file. + +- **LLAMACPP_SERVER_ARGS environment variable** + - Add _LLAMACPP_SERVER_ARGS_ with additional Llama.cpp server process start arguments + to your _.env_ file. ## Upgrading diff --git a/docker-compose.override.private.yml b/docker-compose.override.private.yml index e0b5df8..fc5429f 100644 --- a/docker-compose.override.private.yml +++ b/docker-compose.override.private.yml @@ -77,3 +77,15 @@ services: ollama-gpu-amd: ports: - 127.0.0.1:11434:11434 + + llamacpp-cpu: + ports: + - 127.0.0.1:8040:8040 + + llamacpp-gpu: + ports: + - 127.0.0.1:8040:8040 + + llamacpp-gpu-amd: + ports: + - 127.0.0.1:8040:8040 diff --git a/docker-compose.yml b/docker-compose.yml index 2a62606..85accd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,7 @@ volumes: langfuse_clickhouse_data: langfuse_clickhouse_logs: langfuse_minio_data: + llamacpp_data: redis_valkey_data: caddy_data: caddy_config_data: @@ -68,6 +69,8 @@ x-n8n: &service-n8n #- N8N_PROXY_HOPS=${N8N_PROXY_HOPS} - OLLAMA_HOST=${OLLAMA_HOST} + - LLAMACPP_HOST=${LLAMACPP_HOST} + - PROJECTS_PATH=${PROJECTS_PATH} - WEBHOOK_URL=${N8N_HOSTNAME:+https://}${N8N_HOSTNAME:-http://localhost:5678} x-n8n-runner: &service-n8n-runner @@ -99,6 +102,25 @@ x-init-ollama: &init-ollama - "-c" - "sleep 3; ollama pull ${OLLAMA_DEFAULT_MODEL}; ollama pull ${OLLAMA_SUPPLEMENT_MODEL}; ollama pull ${OLLAMA_EMBEDDING_MODEL}" +x-llamacpp: &service-llamacpp + container_name: llamacpp + restart: unless-stopped + expose: + - 8040/tcp + entrypoint: /app/llama-server ${LLAMACPP_SERVER_ARGS} + environment: + - LLAMA_ARG_PORT + - LLAMA_ARG_HOST + - LLAMA_ARG_HF_REPO + - LLAMA_ARG_N_GPU_LAYERS + - LLAMA_ARG_CTX_SIZE + - LLAMA_ARG_FLASH_ATTN + - LLAMA_ARG_THREADS + - LLAMA_ARG_MODELS_MAX + volumes: + - llamacpp_data:/root/.cache + - ./llama.cpp/models:/models + name: ai-suite services: @@ -595,3 +617,28 @@ services: image: ollama/ollama:rocm depends_on: - ollama-gpu-amd + + llamacpp-cpu: + profiles: ["cpp-cpu"] + <<: *service-llamacpp + image: ghcr.io/ggml-org/llama.cpp:server + + llamacpp-gpu: + profiles: ["cpp-gpu-nvidia"] + <<: *service-llamacpp + image: ghcr.io/ggml-org/llama.cpp:server-cuda + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + llamacpp-gpu-amd: + profiles: ["cpp-gpu-amd"] + <<: *service-llamacpp + image: ghcr.io/ggml-org/llama.cpp:server-rocm + devices: + - "/dev/kfd" + - "/dev/dri" diff --git a/suite_services.py b/suite_services.py index 5128ac0..8ffc994 100644 --- a/suite_services.py +++ b/suite_services.py @@ -1,31 +1,32 @@ #!/usr/bin/env python3 """ Trevor SANDY -Last Update January 09, 2026 +Last Update January 10, 2026 Copyright (c) 2025-Present by Trevor SANDY AI-Suite uses this script for the installation command that handles the AI-Suite -functional module selection, Ollama CPU/GPU configuration, and starting Supabase and -Open WebUI Filesystem when specified. +functional module selection, llama CPU/GPU configuration, and starting Supabase +and Open WebUI Filesystem when specified. If specified, the Supabase stack is started first. The script waits for it to initialize, -then starts open-webui filesystem tool - if specified, and then starts the AI-Suite stack. -All stacks use the same Docker Compose services project name ("ai-suite") so they appear -grouped together in Docker Desktop. +then starts open-webui filesystem tool - if specified, and then starts the AI-Suite +stack. All stacks use the same Docker Compose services project name ("ai-suite") +so they appear grouped together in Docker Desktop. -This script is also used for operation commands that start, stop, stop-ollama, +This script is also used for operation commands that start, stop, stop-llama, pause, unpause, update and install the AI-Suite services using the optional ---operation argument. An Ollama check is performed when it is assumed Ollama is -running from the Docker Host. If Ollama is determined to be installed but not running, -an attempt to launch the Ollama service is executed on install, start and unpause. -The check will also attempt to stop the Ollama service (in addition to stopping the -AI-Suite services) when the stop-ollama operational command is specified. +--operation argument. A llama (Ollama/Llama.cpp) check is performed when it is +assumed llama is running from the Docker Host. If llama is determined to be installed +but not running, an attempt to launch the Ollama/Llama.cpp service is executed +on install, start and unpause. The check will also attempt to stop the llama service +(in addition to stopping the AI-Suite services) when the stop-llama operational +command is specified. Both installation and operation commands utilize the optional --profile -arguments to specify which AI-Suite functional modules and which Ollama CPU/GPU +arguments to specify which AI-Suite functional modules and which llama CPU/GPU configuration to use. When no functional profile argument is specified, the default functional module open-webui is used, Likewise, if no CPU/GPU configuration -profile is specified, it is assumed Ollama is being run from the Docker Host. +profile is specified, it is assumed llama is being run from the Docker Host. Multiple profile arguments (functional modules) are supported. The --environment command allows the installation to be defined as private (default) @@ -89,80 +90,173 @@ def run_command(cmd, cwd=None): except Exception as e: print(f"Command Exception: {e}.") - -def launch_ollama_process(): - """Launch Ollama inference server on host""" - cmd = " ".join([ollama_exe, "serve"]) - print("Running command:", cmd) +def launch_llama_process(args): + """Launch Ollama/Llama.cpp server on the host""" + log_dir = "llama.cpp" if llama_cpp else "" + log_path = os.path.join(os.getcwd(), log_dir, 'llama_start.log') + log = "".join(['>', log_path, ' 2>&1']) if system == "Windows": - path = tempfile.gettempdir() - ollama_launch = os.path.join(path, "ollama_launch.vbs") - print("Command script:", ollama_launch) - with open(ollama_launch, 'w') as f: - f.write(textwrap.dedent(f"""\ - ' Generated from {info.get("file")} on: {datetime.datetime.now().ctime()} - Set WshShell = CreateObject("WScript.Shell") - WshShell.Run "{cmd}", 0, False - Set WshShell = Nothing - """)) - os.startfile(ollama_launch) + win = "".join(['/c,"', llama_exe]) + cmd = ['powershell', '-Command', 'Start-Process cmd -Args', win, args, + "".join([log, '"']), '-WindowStyle Hidden'] else: # Unix-based systems (Linux, macOS) - os.system(cmd) + cmd = [llama_exe, args, log] + print("Running command:", " ".join(cmd)) + try: + completed = subprocess.run(cmd, capture_output=True, text=True, check=True) + if completed.returncode != 0: + print(f"Error: {llama} process: {completed.stderr}") + except Exception as e: + print(f"Exception: {llama} process: {e} - assuming {llama} did not start.") global attempted_launch attempted_launch = True - print("Waiting for Ollama on host to initialize...") + print(f"Waiting for {llama} on host to initialize...") time.sleep(4) - check_ollama_process(None) + check_llama_process(None, {}) -def check_ollama_process(operation=None): - """Check for Ollama (on host) and attempt to launch if not running.""" +def check_llama_cpp_model(operation, env_vars, using_hf): + """Check if the specified llama.cpp model exists, offer to download if not""" + model_path = normalize_path(env_vars.get('LLAMACPP_MODEL_PATH')) + model_name = env_vars.get('LLAMACPP_MODEL_NAME', 'gemma-4b') + if not os.path.exists(model_path): + proceed = True + # Dictionary of common llama.cpp model names and their download identifiers + llama_cpp_models = { + "LLAMACPP_MODEL_GEMMA": "LLAMACPP_MODEL_GEMMA_ID", + "LLAMACPP_MODEL_DEEPSEEK": "LLAMACPP_MODEL_DEEPSEEK_ID", + "LLAMACPP_MODEL_MISTRAL": "LLAMACPP_MODEL_MISTRAL_ID", + "LLAMACPP_MODEL_LLAMA": "LLAMACPP_MODEL_LLAMA_ID", + "LLAMACPP_MODEL_QWEN": "LLAMACPP_MODEL_QWEN_ID", + "LLAMACPP_MODEL_USER": "LLAMACPP_MODEL_USER_ID" + } + if operation != 'install' and not using_hf: + print(f"{llama} model not found at {model_path}") + response = input(f"Would you like to download the {model_name} model now? (y/n): ") + proceed = response.lower() == 'y' + if proceed: + model_dir = os.path.dirname(model_path) + if not os.path.exists(model_dir): + os.makedirs(model_dir,exist_ok=True) + best_match = None + best_match_key = None + best_match_score = 0 + for model_key in llama_cpp_models.keys(): + known_model = env_vars.get(model_key) + if not known_model: + continue + match_score = sum(c1 == c2 for c1, c2 in zip(model_name.lower(), known_model.lower())) + if match_score > best_match_score: + best_match = known_model + best_match_key = model_key + best_match_score = match_score + if best_match and best_match_key and best_match_score > len(best_match) / 2: + print(f"Using {llama} model: {best_match}...") + return env_vars.get(llama_cpp_models[best_match_key]) + else: + print(f"Error: Unknown model '{model_name}', download model manually - Models:") + proceed = False + else: # User elected not to proceed + print("Notice: Download the model manually and update your .env file - Models:") + if not proceed: + llama_server_args = env_vars.get('LLAMACPP_SERVER_ARGS') + for model_key, model_value in llama_cpp_models.items(): + model = env_vars.get(model_key) + model_id = env_vars.get(model_value) + print(f"- {model} command: {llama_app} {llama_server_args} {model_id}") + return None + print(f"Using {llama} model: {model_name}...") + return model_name + +def check_llama_process(operation=None, env_vars={}): + """Check for Ollama/Llama.cpp (on host) and attempt to launch if not running.""" if not attempted_launch: - print("Checking for Ollama process on host...") - ollama_running = False - ollama_proc = ollama_app.lower() + print(f"Checking for {llama} process on host...") + llama_running = False + llama_proc = llama_app.lower() try: if system == "Windows": cmd = ["tasklist"] else: # Unix-based systems (Linux, macOS) - cmd = ["pgrep", "-f", ollama_proc] + cmd = ["pgrep", "-f", llama_proc] print("Running command:", " ".join(cmd)) - check_proc = subprocess.run(cmd, capture_output=True, text=True, check=True) + completed = subprocess.run(cmd, capture_output=True, text=True, check=True) if system == "Windows": - ollama_running = True if ollama_proc in check_proc.stdout.lower() else False + llama_running = True if llama_proc in completed.stdout.lower() else False else: # Unix-based systems (Linux, macOS) - ollama_running = check_proc.returncode == 0 if check_proc else False + llama_running = completed.returncode == 0 if completed else False except Exception as e: - print(f"Exception: Ollama process: {e} - assuming Ollama is not running.") + print(f"Exception: {llama} process: {e} - assuming {llama} is not running.") - stop_ollama = operation == 'stop-ollama' + stop_llama = operation == 'stop-llama' + start_llama = not stop_llama and operation in ['start', 'unpause'] - if ollama_running: - if stop_ollama: - print("Stopping Ollama process on host...") + if llama_running: + if stop_llama: + print(f"Stopping {llama} process on host...") if system == "Windows": - cmd = ["taskkill", "/f", "/im", ollama_proc] + cmd = ["taskkill", "/f", "/im", llama_proc] else: # Unix-based systems (Linux, macOS) - cmd = ["ps", "-C", ollama_proc, "-o", "pid=|xargs", "kill", "-9"] + cmd = ["ps", "-C", llama_proc, "-o", "pid=|xargs", "kill", "-9"] print("Running command:", " ".join(cmd)) os.system(" ".join(cmd)) else: insert = "is now" if attempted_launch else "is" - print(f"Ollama on host {insert} running...") + print(f"{llama} on host {insert} running...") else: if attempted_launch: - print("Failed to launch Ollama on host - exiting...") + print(f"Failed to launch {llama} on host - exiting...") sys.exit(1) - print("Ollama is not running...") - if not stop_ollama: - if ollama_found: - print("Attempting to launch Ollama on host...") - launch_ollama_process() + print(f"{llama} is not running...") + if start_llama: + if llama_found: + print(f"Attempting to launch {llama} on host...") + llama_args = [] + if llama_cpp: + regex = r"(?:-hf|--hf-file|-m|--model|--model-url)" + llama_hf_repo = env_vars.get('LLAMA_ARG_HF_REPO') + llama_server_args = env_vars.get('LLAMACPP_SERVER_ARGS') + llama_models_dir = normalize_path(env_vars.get('LLAMACPP_MODELS_DIR')) + llama_model_arg = re.search(regex, llama_server_args) + if llama_models_dir: + if os.path.isdir(llama_models_dir): + default_models_dir = normalize_path(os.path.join('llama.cpp','models')) + if llama_models_dir != default_models_dir and os.listdir(llama_models_dir): + regex = r"\b{--models-dir}\b" + if not re.search(regex, llama_server_args): + llama_args.extend(["--models-dir", llama_models_dir]) + else: + print(f"Error: Models directory {llama_models_dir} does not exist.") + can_download = True if llama_hf_repo else False + if llama_server_args: + llama_args.extend([llama_server_args]) + if not llama_hf_repo: + using_hf = False + if llama_model_arg: + using_hf = llama_model_arg.group() in ['-hf', '--hf-file'] + can_download = llama_model_arg.group() not in ['-m', '--model'] + llama_model = check_llama_cpp_model(operation, env_vars, using_hf) + if llama_model: + if not llama_model_arg: + llama_args.extend(['-hf']) + llama_args.extend([llama_model]) + elif operation == 'install': + sys.exit(1) + if can_download and operation in ['install', 'update']: + print(f"""Configured to download {llama} model (this may take a while).\n + Check container log for download progress...""") + else: + llama_server_args = env_vars.get('OLLAMA_SERVER_ARGS') + if llama_server_args: + llama_args.extend([llama_server_args]) + + args = " ".join(llama_args) + launch_llama_process(args) else: print(textwrap.dedent(f"""\ - Error: The {ollama_app} file was not found at {ollama_exe}. - If Ollama is installed in a non-standard location, you can set the OLLAMA_PATH - environment variable with its full path (including the Ollama file) to .env and - re-run this file - exiting... + Error: The {llama_app} file was not found at {llama_exe}. + If {llama} is installed in a non-standard location, you can set the LLAMA_PATH + environment variable with its full path (including the {llama} file) in .env and + re-run {info.get("file")} - exiting... """)) sys.exit(1) @@ -324,7 +418,7 @@ def operate_ai_suite(operation, profile, environment, env_vars): operation = "stop" with open('.operation', 'w') as f: - f.write(operation) + f.write(operation + ':' + llama.lower()) supabase = False open_webui = False @@ -852,20 +946,41 @@ def main(): name = info.get('name', 'placeholder') file = info.get('file', 'placeholder.py') version = info.get('version', (-1, -1, -1)) - print(f"""{name} version: {'.'.join(map(str, version))}""") + + # Detect operational status and current llama (Ollama/Llama.cpp) configuration + global llama, llama_cpp + status = None + llama_cpp = False + if os.path.exists('.operation'): + with open('.operation', 'r') as f: + a = f.readline().split(':') + status = a[0].strip() if a else None + if len(a) > 1: + llama_cpp = a[1].strip() == 'llama.cpp' + else: + lp = dotenv.get_key(os.path.join(".env"), 'LLAMA_PATH') + llama_cpp = lp and os.path.basename(lp).lower().startswith('llama-server') + llama = "Llama.cpp" if llama_cpp else "Ollama" + + # Banner + print(f"""{name} version: {'.'.join(map(str, version))} LLM: {llama}""") # Profile, environment and operation arguments global open_webui_all_profiles - ollama_profiles = ['cpu', 'gpu-nvidia', 'gpu-amd'] - n8n_profiles = ["n8n", "n8n-all"] + llama_host_profiles = ['ollama', 'llama.cpp'] + ollama_docker_profiles = ['cpu', 'gpu-nvidia', 'gpu-amd'] + llamacpp_docker_profiles = ['cpp-cpu', 'cpp-gpu-nvidia', 'cpp-gpu-amd'] + llama_docker_profiles = ollama_docker_profiles + llamacpp_docker_profiles + n8n_profiles = ['n8n', 'n8n-all'] n8n_all_profiles = n8n_profiles + ['ai-all'] open_webui_utils_profiles = ['open-webui-mcpo', 'open-webui-pipe'] open_webui_profiles = ['open-webui', 'open-webui-all'] open_webui_all_profiles = open_webui_profiles + n8n_all_profiles agent_all_profiles = open_webui_all_profiles + ['opencode'] server_profiles = ['supabase', 'flowise', 'searxng', 'langfuse', 'neo4j', 'caddy'] - profiles = agent_all_profiles + open_webui_utils_profiles + server_profiles + ollama_profiles - operations = ['stop', 'stop-ollama', 'start', 'pause', 'unpause', 'update', 'install'] + profiles = agent_all_profiles + open_webui_utils_profiles + server_profiles + \ + llama_docker_profiles + llama_host_profiles + operations = ['stop', 'stop-llama', 'start', 'pause', 'unpause', 'update', 'install'] environments = ['private', 'public'] parser = argparse.ArgumentParser( prog=f'{file}', @@ -884,23 +999,27 @@ def main(): Example commands: - Install profile arguments n8n and opencode... - ...with Ollama running in the Host: + ...with {llama} running in the Host: >python {file} --profile n8n opencode - ...with Ollama runing in Docker using CPU: + ...with Ollama CPU running in Docker: >python {file} --profile n8n opencode cpu - ...using GPU and in public (production) environment: - >python {file} --profile n8n opencode --environment public + ...using Llama.cpp AMD GPU in Docker and on production environment: + >python {file} --profile n8n opencode cpp-gpu-amd --environment public - Perform stop (start, pause, unpause) suite operation: - >python {file} --profile n8n opencode cpu --operation stop + >python {file} --profile n8n opencode --operation stop - Perform suite operation to update all modules and restart: >python {file} --operation update - - Perform suite operation to install all functional modules and start: + - Perform suite operation to install all functional modules and start... >python {file} --operation install + + ...using Llama.cpp Nvidia GPU running in Docker: + >python {file} --profile ai-all cpp-gpu-nvidia --operation install + '''), epilog=textwrap.dedent(f'''\ - Title: {info.get("title")} @@ -913,14 +1032,14 @@ def main(): - Copyright: {info.get("copyright")} ''')) parser.add_argument('-p', '--profile', type=str, nargs='+', choices=profiles, - help='Docker Compose Profile arguments for functional modules and Ollama' - 'CPU/GPU options (default: open-webui - with Ollama running on Host)') + help='Docker Compose Profile arguments for functional modules and llama' + f'CPU/GPU options (default: open-webui - with {llama} running on Host)') parser.add_argument('-e', '--environment', type=str, choices=environments, default='private', help='Environment arguments used by Docker Compose to expose ' 'or restrict network communication ports (default: private)') parser.add_argument('-o', '--operation', type=str, choices=operations, - help='Docker container, volumes and image management arguments ' - 'along with argument to stop Ollama running on Host.') + help='Docker container, volumes and image management arguments along ' + f'with argument to stop {llama} running on Host.') args = parser.parse_args() @@ -942,51 +1061,89 @@ def main(): env_vars = get_dotenv_vars() if not env_vars: sys.exit(1) - + # Detect default profile - no arguments specified default_profile = False if args.profile else True - - # Set default projects path in working environment variables - if any(p for p in args.profile if p in agent_all_profiles): - if not env_vars['PROJECTS_PATH']: - env_vars['PROJECTS_PATH'] = normalize_path(os.path.join('~', 'projects')) + args.profile = [] if default_profile else args.profile - # Check Ollama status when running Ollama in the Host - ollama_in_host = default_profile or not \ - any(profile for profile in args.profile if profile in ollama_profiles) - if ollama_in_host: - global ollama_found, ollama_app, ollama_exe, attempted_launch - ollama_found = False + # Check llama (Ollama/Llama.cpp) status + llama_arg = "cpu" + llama_in_host = default_profile or not \ + any(p for p in args.profile if p in llama_docker_profiles) + if llama_in_host: + global llama_found, llama_app, llama_exe, attempted_launch attempted_launch = False - if system == "Windows": - ollama_app = 'ollama.exe' - ollama_exe = os.path.join(os.path.expanduser('~'), 'AppData\\Local\\Programs\\Ollama', ollama_app) - ollama_found = os.path.exists(ollama_exe) - else: # Unix-based systems (Linux, macOS) - ollama_app = 'ollama' - for ollama_path in ['/bin', '/usr/local/bin', '/usr/bin']: - ollama_exe = os.path.join(ollama_path, ollama_app) - if os.path.exists(ollama_exe): - ollama_found = True - break - if not ollama_found: - ollama_path = dotenv.get_key(env_file, 'OLLAMA_PATH') - if ollama_path is None: - ollama_path = os.environ.get('OLLAMA_PATH') - if ollama_path is not None: - ollama_exe = os.path.normpath(ollama_path) - ollama_app = os.path.basename(ollama_exe) - ollama_found = os.path.exists(ollama_exe) - check_ollama_process(args.operation) + llama_found = False + llama_path = normalize_path(env_vars.get('LLAMA_PATH')) + llama_cpp = any(p for p in args.profile if p == 'llama.cpp') + llama = "Llama.cpp" if llama_cpp else "Ollama" + if llama_path is not None: + llama_exe = os.path.normpath(llama_path) + llama_app = os.path.basename(llama_exe) + llama_found = os.path.exists(llama_exe) + if not llama_found: + llama_app = "llama-server" if llama_cpp else "ollama" + if system == "Windows": + llama_app = "".join([llama_app, '.exe']) + llama_dir = "llama.cpp" if llama_cpp else "Ollama" + for llama_sub in ['~\\AppData\\Local\\Programs', os.getcwd()]: + llama_exe = normalize_path(os.path.join(llama_sub, llama_dir, llama_app)) + if os.path.exists(llama_exe): + llama_found = True + break + else: # Unix-based systems (Linux, macOS) + for llama_path in ['/bin', '/usr/local/bin', '/usr/bin']: + llama_exe = os.path.join(llama_path, llama_app) + if os.path.exists(llama_exe): + llama_found = True + break + set_dotenv_key(os.path.join(".env"), 'LLAMA_PATH', llama_exe, None) + env_vars = get_dotenv_vars(force=True) + # Check if llama exe (llama-server, ollama) matches profile argument (llama.cpp, ollama) + if (llama_cpp and not llama_app.lower().startswith('llama-server')) or \ + (not llama_cpp and not llama_app.lower().startswith('ollama')): + llama_cpp = llama_app.lower().startswith('llama-server') + llama = "Llama.cpp" if llama_cpp else "Ollama" + llama_mismatch = "ollama" if llama_cpp else "llama.cpp" + print(f"""Notice: The executable '{llama_app}' did not match the '{llama}' + profile argument - argument updated to '{llama.lower()}'...""") + args.profile.remove(llama_mismatch) if llama_mismatch in args.profile else None + args.profile.extend([llama.lower()]) if llama_cpp else None + llama_host = "host.docker.internal" + llama_host_var = llama_host if llama_cpp else llama_host + ":${OLLAMA_PORT}" + check_llama_process(args.operation, env_vars) + else: + llama_cpp = any(p for p in args.profile if p in llamacpp_docker_profiles) + llama = "Llama.cpp" if llama_cpp else "Ollama" + llama_host_var = "0.0.0.0" if llama_cpp else "ollama:${OLLAMA_PORT}" + # Check if more than one llama CPU/GPU argument specified, use first argument + if any(p for p in args.profile if p in llama_docker_profiles): + first_argument = False + for profile_arg in llama_docker_profiles: + if not first_argument: + if any(p for p in args.profile if p == profile_arg): + print(f"""{name} will use {llama} CPU/GPU profile argument '{profile_arg}'...""") + first_argument = True + else: + args.profile.remove(profile_arg) if profile_arg in args.profile else None + # Check if any llama host profile arguments specified and remove if found + for profile_arg in llama_host_profiles: + if any(p for p in args.profile if p == profile_arg): + print(f"""A llama in host profile argument cannot be specified with + llama CPU/GPU profile argument - removing '{profile_arg}'...""") + args.profile.remove(profile_arg) + # Assemble .env updates, set respective keys in .env file and reload .env vars + llama_host_env = "LLAMA_ARG_HOST" if llama_cpp else "OLLAMA_HOST" + oai_base_url_var = "${LLAMACPP_HOST}" if llama_cpp else "${OLLAMA_HOST}" + mod_env_vars.update({llama_host_env: llama_host_var, 'OPENAI_API_BASE_URL': oai_base_url_var}) + for env, var in mod_env_vars.items(): + set_dotenv_var(env_file, env, var, None) + env_vars = get_dotenv_vars(env_file, True) # Process operation argument build = False if args.operation: - status = None - if os.path.exists('.operation'): - with open('.operation', 'r') as f: - status = f.readline() - if args.operation == 'stop-ollama': + if args.operation == 'stop-llama': args.operation = "stop" if args.operation == status: if status == 'stop': @@ -1016,8 +1173,9 @@ def main(): args.operation = 'pull' insert = "Installing" if install == 'install' else "Updating" if default_profile: - print(f"""{insert} all container images including Ollama...""") - args.profile.extend(["cpu"]) + print(f"""{insert} all container images including {llama}...""") + llama_arg = "cpp-cpu" if llama_cpp else "cpu" + args.profile.extend([llama_arg]) else: print(f"""{insert} container images for {args.profile}...""") docker_compose_include(True, True, True) @@ -1030,7 +1188,7 @@ def main(): # Manually set default profile argument if default_profile: if build: - args.profile.remove('cpu') if 'cpu' in args.profile else None + args.profile.remove(llama_arg) if llama_arg in args.profile else None else: args.profile = ['open-webui'] @@ -1132,7 +1290,7 @@ def main(): start_ai_suite(args.profile, args.environment, build) with open('.operation', 'w') as f: - f.write('start') + f.write('start' + ':' + llama.lower()) if __name__ == "__main__": main()