WEBDIENSTE/ai-suite
1
0
Fork 0
mirror of https://github.com/trevorsandy/ai-suite.git synced 2026-08-17 18:25:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add support for Llama.cpp in Docker and on host

Browse Source
This commit is contained in:
Trevor SANDY
2026-01-10 15:11:34 +01:00
parent 8ebee6544f
commit ab48e96d0d
6 changed files with 476 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+96 -32
View File
@@ -104,6 +104,7 @@ ENCRYPTION_KEY=your-super-secret-key-3
# SUPABASE_HOSTNAME=supabase.yourdomain.com
# LANGFUSE_HOSTNAME=langfuse.yourdomain.com
# OLLAMA_HOSTNAME=ollama.yourdomain.com
# LLAMACPP_HOSTNAME=llamacpp.yourdomain.com
# SEARXNG_HOSTNAME=searxng.yourdomain.com
# NEO4J_HOSTNAME=neo4j.yourdomain.com
# LETSENCRYPT_EMAIL=internal
@@ -121,9 +122,10 @@ ENCRYPTION_KEY=your-super-secret-key-3
# n8n
############
N8N_PORT=5678
# Set as N8N_HOST=${N8N_HOSTNAME:-${N8N_HOST}} in compose.yaml
N8N_HOST=localhost
N8N_PORT=5678
# Permissions 0644 for n8n settings file /home/node/.n8n/config are too wide.
N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=true
@@ -164,8 +166,6 @@ N8N_RESERVE_MEMORY=1G
# Time
GENERIC_TIMEZONE=France/Paris
PG_META_CRYPTO_KEY
############
# PostgreSQL - You can change these to any PostgreSQL database that has logical replication enabled.
############
@@ -180,8 +180,8 @@ POSTGRES_PORT=5432
# Projects
############
# Omitted '=<value>' to dynamically set using '~' prefix on Compose 'up'
PROJECTS_PATH
# Omitted '<value>' to dynamically set using '~' prefix on Compose 'up'
PROJECTS_PATH=
############
# Document Processing Settings
@@ -199,35 +199,22 @@ MAX_CHUNK_SIZE=40000 # Larger chunks for CAG (character count)
CHUNK_OVERLAP=500 # Overlap between chunks if needed
MIN_DOCUMENT_SIZE_FOR_CHUNKING=100000 # Only chunk very large documents
# Omit '=' to return 'None' vs empty 'str' when not set
LLAMA_PATH
############
# Ollama - Inferencce engine and model manager
# Ollama - LLM
############
# When running OLLAMA in the Host and N8N in Docker:
OLLAMA_PORT=11434
# When running OLLAMA in the Host:
#OLLAMA_HOST=host.docker.internal:11434
OLLAMA_HOST=ollama:11434
# Ollama URL for Open-WebUI backend connect
# The path '/ollama' will be redirected to the specified backend URL.
# When running OLLAMA in the Host and Open-WebUI in Docker:
#OLLAMA_BASE_URL=http://host.docker.internal:11434
OLLAMA_BASE_URL=http://localhost:11434
# Set if conecting to Ollama using OpenAI API connection
#OPENAI_API_BASE_URL=http://localhost:11434
# When running OLLAMA in the Host and Open-WebUI in Docker:
#OPENAI_API_BASE_URL=http://host.docker.internal:11434
OPENAI_API_BASE_URL=
# OpenAI API key OPENAI_API_KEY declared below at Studio
# When running OLLAMA in Docker:
#OLLAMA_HOST=ollama:11434
OLLAMA_HOST=host.docker.internal:11434
# Tuning
OLLAMA_CONTEXT_LENGTH=8192
OLLAMA_FLASH_ATTENTION=1
OLLAMA_KV_CACHE_TYPE=q8_0
OLLAMA_KV_CACHE_TYPE=q4_0
OLLAMA_MAX_LOADED_MODELS=2
# Models
@@ -235,9 +222,81 @@ OLLAMA_DEFAULT_MODEL=llama3.2
OLLAMA_SUPPLEMENT_MODEL=qwen3:8b
OLLAMA_EMBEDDING_MODEL=nomic-embed-text
# Ollama server arguments (exclude model file) - use 'ollama serve --help' for available arguments
# Ollama server arguments - use ollama serve --help for available 'serve' arguments
OLLAMA_SERVER_ARGS=serve
############
# llama.cpp - LLM
############
LLAMA_ARG_PORT=8040
# When running LLAMA.CPP in the host:
#LLAMA_ARG_HOST=host.docker.internal
# When running LLAMA.CPP in Docker:
#LLAMA_ARG_HOST=0.0.0.0
LLAMA_ARG_HOST=0.0.0.0
# Backend connect
LLAMACPP_HOST=${LLAMA_ARG_HOST}:${LLAMA_ARG_PORT}
# Model names - Dictionary keys for model download identifier values below.
# Keys, and values below include an empty slot for a user-defined model
LLAMACPP_MODEL_GEMMA=gemma-4b # Default
LLAMACPP_MODEL_DEEPSEEK=deepseek-7b
LLAMACPP_MODEL_MISTRAL=mistral-7b
LLAMACPP_MODEL_LLAMA=llama-8b
LLAMACPP_MODEL_QWEN=qwen-8b
LLAMACPP_MODEL_USER=
# Model download identifier - Dictionary values for model keys above.
# Model selected by 'best match' to LLAMACPP_MODEL_NAME
# To specify a local model, change '-hf' to '-m' in LLAMACPP_SERVER_ARGS below
# and replace the respective model id value below with 'models/<model filename>'.
LLAMACPP_MODEL_GEMMA_ID=ggml-org/gemma-3-4b-it-GGUF
LLAMACPP_MODEL_DEEPSEEK_ID=mradermacher/DeepSeek-R1-Distill-Qwen-7B-Uncensored-i1-GGUF
LLAMACPP_MODEL_MISTRAL_ID=bartowski/mistralai_Ministral-3-8B-Instruct-2512-GGUF
LLAMACPP_MODEL_LLAMA_ID=bartowski/allura-forge_Llama-3.3-8B-Instruct-GGUF
LLAMACPP_MODEL_QWEN_ID=bartowski/Qwen_Qwen3-8B-GGUF
LLAMACPP_MODEL_USER_ID=
# Model and paths
LLAMACPP_PATH=llama.cpp
LLAMACPP_MODEL_NAME=${LLAMACPP_MODEL_GEMMA} # IMPORTANT: should reasonably match Dictionary model name above.
LLAMACPP_MODELS_DIR=${LLAMACPP_PATH}/models
LLAMACPP_MODEL_PATH=${LLAMACPP_MODELS_DIR}/${LLAMACPP_MODEL_NAME}
# Model management - automatically download specified model if not downloaded.
LLAMA_ARG_HF_REPO=${LLAMACPP_MODEL_GEMMA_ID}
# Tuning
LLAMA_ARG_CTX_SIZE=8192
LLAMA_ARG_FLASH_ATTN=1
LLAMA_ARG_N_GPU_LAYERS=0
LLAMA_ARG_THREADS=4
LLAMA_ARG_MODELS_MAX=4
# Llama.cpp server arguments - use 'llama-server --help' for available arguments
# To specify a local model, append '-m' or '––model'.
# To auto-download model (if not already downloaded) and if LLAMA_ARG_HF_REPO is
# not used (commented), append '-hf' or '--hf-file'.
LLAMACPP_SERVER_ARGS=--jinja
############
# LLAMA (Ollama/Llama.cpp) - Shared environment variables
############
# Application Installation path
# Set for Llama.cpp or if using custom Ollama installation path
# e.g. LLAMA_PATH=~\Projects\ai-suite\llama.cpp\bin\llama-server.exe
# Omit '<value>' to return 'False' when queried
LLAMA_PATH=
# Conecting to LLAMA using OpenAI API connection
# When running OLLAMA: ${OLLAMA_HOST}
# When running LLAMA.CPP: ${LLAMACPP_HOST}
OPENAI_API_BASE_URL=${OLLAMA_HOST}
#OPENAI_API_KEY - OpenAI API key declared below at Studio
############
# Open-WebUI
@@ -253,8 +312,8 @@ CORS_ALLOW_ORIGIN='*'
FORWARDED_ALLOW_IPS='*'
# If running Open WebUI in an offline environment, set to 1 to prevent attempts to download models from the internet.
# Omitted '=' to return 'None' vs empty 'str' when not set
HF_HUB_OFFLINE
# Omit '<value>' to return 'False' when queried
HF_HUB_OFFLINE=
# AUTOMATIC1111_BASE_URL="http://localhost:7860"
@@ -274,8 +333,8 @@ MCPO_HOT_RELOAD=true
DOCKER_SOCKET_LOCATION=/var/run/docker.sock
# Set Docker Compose ignore orphans to 'true' for locally built comtainers
# Omitted '=' to return 'None' vs empty 'str' when not set
COMPOSE_IGNORE_ORPHANS
# Omit '<value>' to return 'False' when queried
COMPOSE_IGNORE_ORPHANS=
############
# Google
@@ -308,8 +367,12 @@ GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER
POOLER_PROXY_PORT_TRANSACTION=6543
POOLER_DEFAULT_POOL_SIZE=20
POOLER_MAX_CLIENT_CONN=100
# Must be at least 64 characters; generate with 'openssl rand -base64 48'
SECRET_KEY_BASE=UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq
# Must be exactly 32 characters; generate with 'openssl rand -hex 16'
VAULT_ENC_KEY=your-32-character-encryption-key
# Must be at least 32 characters; generate with 'openssl rand -base64 24'
PG_META_CRYPTO_KEY=your-super-secret—long-encryption-key
# Pool size for internal metadata storage used by Supavisor
# This is separate from client connections and used only by Supavisor itself
POOLER_DB_POOL_SIZE=5
@@ -395,6 +458,7 @@ FUNCTIONS_VERIFY_JWT=false
############
# Change vector.toml sinks to reflect this change
# these cannot be the same value
# These cannot be the same value
# Must be at least 32 characters; generate with 'openssl rand -base64 24'
LOGFLARE_PUBLIC_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-public
LOGFLARE_PRIVATE_ACCESS_TOKEN=your-super-secret-and-long-logflare-key-private
.gitignore
+2
View File
@@ -4,6 +4,8 @@ supabase/
searxng/
neo4j/
caddy/
llama.cpp/
ollama/
opencode/*
!opencode/opencode.jsonc
!opencode/run_opencode_docker.py
README.md
+44 -18
View File
@@ -217,6 +217,7 @@ Before you begin, make sure you have the following software installed:
# SUPABASE_HOSTNAME=supabase.yourdomain.com
# LANGFUSE_HOSTNAME=langfuse.yourdomain.com
# OLLAMA_HOSTNAME=ollama.yourdomain.com
# LLAMACPP_HOSTNAME=llama.cpp.yourdomain.com
# SEARXNG_HOSTNAME=searxng.yourdomain.com
# NEO4J_HOSTNAME=neo4j.yourdomain.com
# LETSENCRYPT_EMAIL=internal
@@ -245,22 +246,23 @@ Before you begin, make sure you have the following software installed:
---
**AI-Suite** uses the `suite_services.py` script for the _installation_ command
that handles the AI-Suite functional module selection, Ollama GPU configuration,
and starting Supabase when specified.
that handles the AI-Suite functional module selection, llama CPU/GPU configuration,
and starting Supabase and Open WebUI Filesystem when specified.
Additionally, This script is also used for operation commands that _start_, _stop_,
_pause_ and _unpause_ the AI-Suite services using the optional `--operation` argument.
An Ollama check is performed when it is assumed Ollama is being run from the Docker
Host. If Ollama is determined to be installed but not running, an attempt to launch
the Ollama service is executed on _install_, _start_ and _unpause_. The check
will also attempt to _stop_ the Ollama service (in addition to stopping the
AI-Suite services) when the _stop-ollama_ operational command is specified.
This script is also used for operation commands that _start_, _stop_, _stop-llama_,
_pause_, _unpause_, _update_ and _install_ the AI-Suite services using the optional
`--operation` argument. A llama (Ollama/Llama.cpp) check is performed when it is
assumed llama is running from the Docker Host. If llama is determined to be installed
but not running, an attempt to launch the Ollama/Llama.cpp service is executed
on _install_, _start_ and _unpause_. The check will also attempt to _stop_ the
running llama service (in addition to stopping the AI-Suite services) when the
_stop-llama_ operational command is specified.
Both installation and operation commands utilize the optional `--profile`
arguments to specify which AI-Suite functional modules and which Ollama CPU/GPU
arguments to specify which AI-Suite functional modules and which llama CPU/GPU
configuration to use. When no functional profile argument is specified, the
default functional module `open-webui` is used, Likewise, if no GPU configuration
profile is specified, it is assumed Ollama is being run from the Docker Host.
default functional module `open-webui` is used, Likewise, if no CPU/GPU configuration
profile is specified, it is assumed llama is being run from the Docker Host.
**Multiple profile arguments (functional modules) are supported**.
The `--environment` command allows the installation to be defined as _private_
@@ -286,9 +288,17 @@ to the network.
| `n8n-all` | n8n - complete bundle |
| `open-webui-all` | Open WebUI - complete bundle |
| `ai-all` | AI-Suite full stack - all modules |
`suite_services.py` `--profile` llama CPU/GPU argument:
| Argument | Llama CPU/GPU |
| -----------------------: | ------: |
| `cpu` | Ollama - run on CPU |
| `gpu-nvidia` | Ollama - run on Nvidia GPU |
| `gpu-amd` | Ollama - run on AMD GPU |
| `cpp-cpu` | Llama.cpp - run on CPU |
| `cpp-gpu-nvidia` | Llama.cpp - run on Nvidia GPU |
| `cpp-gpu-amd` | Llama.cpp - run on AMD GPU |
Example command:
@@ -298,13 +308,13 @@ python suite_services.py --profile n8n opencode gpu-nvidia
---
`suite_services.py` ... `--operation` arguments:
`suite_services.py` ... `--operation` argument:
| Argument | Operation |
| -----------------------: | ------: |
| `start` | Start - start the previously stopped, specified profile containers |
| `stop` | Stop - shut down the specified profile containers |
| `stop-ollama` | Stop Ollama - perform stop plus shut down Ollama on the Host |
| `stop-llama` | Stop - perform `stop` and shut down Ollama/Llama.cpp on Host |
| `pause` | Pause - pause the specified profile containers |
| `unpause` | Unpause - unpause the previously paused profile containers |
@@ -816,12 +826,28 @@ modules described above:
> path `/root/projects` and the resulting path is set as _work_dir_ to form the
> OpenCode Docker exec command's _workdir=work_dir_ keyword argument.
### Ollama - running on host
### Ollama or Llama.cpp - running on host
- **OLLAMA_PATH environment variable**
- **LLAMA_PATH environment variable**
- If Ollama is installed in a non-standard location, you can add `OLLAMA_PATH`
with its absolute path (including the Ollama file) to the _.env_ file.
- If _Ollama_ is installed in a custom location or you are using _Llama.cpp_,
Add `LLAMA_PATH` with its absolute path (including the file name) to your
_.env_ file.
- **OLLAMA_SERVER_ARGS environment variable**
- Add _OLLAMA_SERVER_ARGS_ with additional Ollama server process start arguments
to your _.env_ file.
- **LLAMACPP_MODELS_DIR environment variable**
- If you are using _Llama.cpp_ with models that were **not** downloaded with
that instance of _Llama.cpp_, add `LLAMACPP_MODELS_DIR` with said models path
to your _.env_ file.
- **LLAMACPP_SERVER_ARGS environment variable**
- Add _LLAMACPP_SERVER_ARGS_ with additional Llama.cpp server process start arguments
to your _.env_ file.
## Upgrading
docker-compose.override.private.yml
+12
View File
@@ -77,3 +77,15 @@ services:
ollama-gpu-amd:
ports:
- 127.0.0.1:11434:11434
llamacpp-cpu:
ports:
- 127.0.0.1:8040:8040
llamacpp-gpu:
ports:
- 127.0.0.1:8040:8040
llamacpp-gpu-amd:
ports:
- 127.0.0.1:8040:8040
docker-compose.yml
+47
View File
@@ -10,6 +10,7 @@ volumes:
langfuse_clickhouse_data:
langfuse_clickhouse_logs:
langfuse_minio_data:
llamacpp_data:
redis_valkey_data:
caddy_data:
caddy_config_data:
@@ -68,6 +69,8 @@ x-n8n: &service-n8n
#- N8N_PROXY_HOPS=${N8N_PROXY_HOPS}
- OLLAMA_HOST=${OLLAMA_HOST}
- LLAMACPP_HOST=${LLAMACPP_HOST}
- PROJECTS_PATH=${PROJECTS_PATH}
- WEBHOOK_URL=${N8N_HOSTNAME:+https://}${N8N_HOSTNAME:-http://localhost:5678}
x-n8n-runner: &service-n8n-runner
@@ -99,6 +102,25 @@ x-init-ollama: &init-ollama
- "-c"
- "sleep 3; ollama pull ${OLLAMA_DEFAULT_MODEL}; ollama pull ${OLLAMA_SUPPLEMENT_MODEL}; ollama pull ${OLLAMA_EMBEDDING_MODEL}"
x-llamacpp: &service-llamacpp
container_name: llamacpp
restart: unless-stopped
expose:
- 8040/tcp
entrypoint: /app/llama-server ${LLAMACPP_SERVER_ARGS}
environment:
- LLAMA_ARG_PORT
- LLAMA_ARG_HOST
- LLAMA_ARG_HF_REPO
- LLAMA_ARG_N_GPU_LAYERS
- LLAMA_ARG_CTX_SIZE
- LLAMA_ARG_FLASH_ATTN
- LLAMA_ARG_THREADS
- LLAMA_ARG_MODELS_MAX
volumes:
- llamacpp_data:/root/.cache
- ./llama.cpp/models:/models
name: ai-suite
services:
@@ -595,3 +617,28 @@ services:
image: ollama/ollama:rocm
depends_on:
- ollama-gpu-amd
llamacpp-cpu:
profiles: ["cpp-cpu"]
<<: *service-llamacpp
image: ghcr.io/ggml-org/llama.cpp:server
llamacpp-gpu:
profiles: ["cpp-gpu-nvidia"]
<<: *service-llamacpp
image: ghcr.io/ggml-org/llama.cpp:server-cuda
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
llamacpp-gpu-amd:
profiles: ["cpp-gpu-amd"]
<<: *service-llamacpp
image: ghcr.io/ggml-org/llama.cpp:server-rocm
devices:
- "/dev/kfd"
- "/dev/dri"
suite_services.py
+275 -117
View File
@@ -1,31 +1,32 @@
#!/usr/bin/env python3
"""
Trevor SANDY
Last Update January 09, 2026
Last Update January 10, 2026
Copyright (c) 2025-Present by Trevor SANDY
AI-Suite uses this script for the installation command that handles the AI-Suite
functional module selection, Ollama CPU/GPU configuration, and starting Supabase and
Open WebUI Filesystem when specified.
functional module selection, llama CPU/GPU configuration, and starting Supabase
and Open WebUI Filesystem when specified.
If specified, the Supabase stack is started first. The script waits for it to initialize,
then starts open-webui filesystem tool - if specified, and then starts the AI-Suite stack.
All stacks use the same Docker Compose services project name ("ai-suite") so they appear
grouped together in Docker Desktop.
then starts open-webui filesystem tool - if specified, and then starts the AI-Suite
stack. All stacks use the same Docker Compose services project name ("ai-suite")
so they appear grouped together in Docker Desktop.
This script is also used for operation commands that start, stop, stop-ollama,
This script is also used for operation commands that start, stop, stop-llama,
pause, unpause, update and install the AI-Suite services using the optional
--operation argument. An Ollama check is performed when it is assumed Ollama is
running from the Docker Host. If Ollama is determined to be installed but not running,
an attempt to launch the Ollama service is executed on install, start and unpause.
The check will also attempt to stop the Ollama service (in addition to stopping the
AI-Suite services) when the stop-ollama operational command is specified.
--operation argument. A llama (Ollama/Llama.cpp) check is performed when it is
assumed llama is running from the Docker Host. If llama is determined to be installed
but not running, an attempt to launch the Ollama/Llama.cpp service is executed
on install, start and unpause. The check will also attempt to stop the llama service
(in addition to stopping the AI-Suite services) when the stop-llama operational
command is specified.
Both installation and operation commands utilize the optional --profile
arguments to specify which AI-Suite functional modules and which Ollama CPU/GPU
arguments to specify which AI-Suite functional modules and which llama CPU/GPU
configuration to use. When no functional profile argument is specified, the
default functional module open-webui is used, Likewise, if no CPU/GPU configuration
profile is specified, it is assumed Ollama is being run from the Docker Host.
profile is specified, it is assumed llama is being run from the Docker Host.
Multiple profile arguments (functional modules) are supported.
The --environment command allows the installation to be defined as private (default)
@@ -89,80 +90,173 @@ def run_command(cmd, cwd=None):
except Exception as e:
print(f"Command Exception: {e}.")
def launch_ollama_process():
"""Launch Ollama inference server on host"""
cmd = " ".join([ollama_exe, "serve"])
print("Running command:", cmd)
def launch_llama_process(args):
"""Launch Ollama/Llama.cpp server on the host"""
log_dir = "llama.cpp" if llama_cpp else ""
log_path = os.path.join(os.getcwd(), log_dir, 'llama_start.log')
log = "".join(['>', log_path, ' 2>&1'])
if system == "Windows":
path = tempfile.gettempdir()
ollama_launch = os.path.join(path, "ollama_launch.vbs")
print("Command script:", ollama_launch)
with open(ollama_launch, 'w') as f:
f.write(textwrap.dedent(f"""\
' Generated from {info.get("file")} on: {datetime.datetime.now().ctime()}
Set WshShell = CreateObject("WScript.Shell")
WshShell.Run "{cmd}", 0, False
Set WshShell = Nothing
"""))
os.startfile(ollama_launch)
win = "".join(['/c,"', llama_exe])
cmd = ['powershell', '-Command', 'Start-Process cmd -Args', win, args,
"".join([log, '"']), '-WindowStyle Hidden']
else: # Unix-based systems (Linux, macOS)
os.system(cmd)
cmd = [llama_exe, args, log]
print("Running command:", " ".join(cmd))
try:
completed = subprocess.run(cmd, capture_output=True, text=True, check=True)
if completed.returncode != 0:
print(f"Error: {llama} process: {completed.stderr}")
except Exception as e:
print(f"Exception: {llama} process: {e} - assuming {llama} did not start.")
global attempted_launch
attempted_launch = True
print("Waiting for Ollama on host to initialize...")
print(f"Waiting for {llama} on host to initialize...")
time.sleep(4)
check_ollama_process(None)
check_llama_process(None, {})
def check_ollama_process(operation=None):
"""Check for Ollama (on host) and attempt to launch if not running."""
def check_llama_cpp_model(operation, env_vars, using_hf):
"""Check if the specified llama.cpp model exists, offer to download if not"""
model_path = normalize_path(env_vars.get('LLAMACPP_MODEL_PATH'))
model_name = env_vars.get('LLAMACPP_MODEL_NAME', 'gemma-4b')
if not os.path.exists(model_path):
proceed = True
# Dictionary of common llama.cpp model names and their download identifiers
llama_cpp_models = {
"LLAMACPP_MODEL_GEMMA": "LLAMACPP_MODEL_GEMMA_ID",
"LLAMACPP_MODEL_DEEPSEEK": "LLAMACPP_MODEL_DEEPSEEK_ID",
"LLAMACPP_MODEL_MISTRAL": "LLAMACPP_MODEL_MISTRAL_ID",
"LLAMACPP_MODEL_LLAMA": "LLAMACPP_MODEL_LLAMA_ID",
"LLAMACPP_MODEL_QWEN": "LLAMACPP_MODEL_QWEN_ID",
"LLAMACPP_MODEL_USER": "LLAMACPP_MODEL_USER_ID"
}
if operation != 'install' and not using_hf:
print(f"{llama} model not found at {model_path}")
response = input(f"Would you like to download the {model_name} model now? (y/n): ")
proceed = response.lower() == 'y'
if proceed:
model_dir = os.path.dirname(model_path)
if not os.path.exists(model_dir):
os.makedirs(model_dir,exist_ok=True)
best_match = None
best_match_key = None
best_match_score = 0
for model_key in llama_cpp_models.keys():
known_model = env_vars.get(model_key)
if not known_model:
continue
match_score = sum(c1 == c2 for c1, c2 in zip(model_name.lower(), known_model.lower()))
if match_score > best_match_score:
best_match = known_model
best_match_key = model_key
best_match_score = match_score
if best_match and best_match_key and best_match_score > len(best_match) / 2:
print(f"Using {llama} model: {best_match}...")
return env_vars.get(llama_cpp_models[best_match_key])
else:
print(f"Error: Unknown model '{model_name}', download model manually - Models:")
proceed = False
else: # User elected not to proceed
print("Notice: Download the model manually and update your .env file - Models:")
if not proceed:
llama_server_args = env_vars.get('LLAMACPP_SERVER_ARGS')
for model_key, model_value in llama_cpp_models.items():
model = env_vars.get(model_key)
model_id = env_vars.get(model_value)
print(f"- {model} command: {llama_app} {llama_server_args} {model_id}")
return None
print(f"Using {llama} model: {model_name}...")
return model_name
def check_llama_process(operation=None, env_vars={}):
"""Check for Ollama/Llama.cpp (on host) and attempt to launch if not running."""
if not attempted_launch:
print("Checking for Ollama process on host...")
ollama_running = False
ollama_proc = ollama_app.lower()
print(f"Checking for {llama} process on host...")
llama_running = False
llama_proc = llama_app.lower()
try:
if system == "Windows":
cmd = ["tasklist"]
else: # Unix-based systems (Linux, macOS)
cmd = ["pgrep", "-f", ollama_proc]
cmd = ["pgrep", "-f", llama_proc]
print("Running command:", " ".join(cmd))
check_proc = subprocess.run(cmd, capture_output=True, text=True, check=True)
completed = subprocess.run(cmd, capture_output=True, text=True, check=True)
if system == "Windows":
ollama_running = True if ollama_proc in check_proc.stdout.lower() else False
llama_running = True if llama_proc in completed.stdout.lower() else False
else: # Unix-based systems (Linux, macOS)
ollama_running = check_proc.returncode == 0 if check_proc else False
llama_running = completed.returncode == 0 if completed else False
except Exception as e:
print(f"Exception: Ollama process: {e} - assuming Ollama is not running.")
print(f"Exception: {llama} process: {e} - assuming {llama} is not running.")
stop_ollama = operation == 'stop-ollama'
stop_llama = operation == 'stop-llama'
start_llama = not stop_llama and operation in ['start', 'unpause']
if ollama_running:
if stop_ollama:
print("Stopping Ollama process on host...")
if llama_running:
if stop_llama:
print(f"Stopping {llama} process on host...")
if system == "Windows":
cmd = ["taskkill", "/f", "/im", ollama_proc]
cmd = ["taskkill", "/f", "/im", llama_proc]
else: # Unix-based systems (Linux, macOS)
cmd = ["ps", "-C", ollama_proc, "-o", "pid=|xargs", "kill", "-9"]
cmd = ["ps", "-C", llama_proc, "-o", "pid=|xargs", "kill", "-9"]
print("Running command:", " ".join(cmd))
os.system(" ".join(cmd))
else:
insert = "is now" if attempted_launch else "is"
print(f"Ollama on host {insert} running...")
print(f"{llama} on host {insert} running...")
else:
if attempted_launch:
print("Failed to launch Ollama on host - exiting...")
print(f"Failed to launch {llama} on host - exiting...")
sys.exit(1)
print("Ollama is not running...")
if not stop_ollama:
if ollama_found:
print("Attempting to launch Ollama on host...")
launch_ollama_process()
print(f"{llama} is not running...")
if start_llama:
if llama_found:
print(f"Attempting to launch {llama} on host...")
llama_args = []
if llama_cpp:
regex = r"(?:-hf|--hf-file|-m|--model|--model-url)"
llama_hf_repo = env_vars.get('LLAMA_ARG_HF_REPO')
llama_server_args = env_vars.get('LLAMACPP_SERVER_ARGS')
llama_models_dir = normalize_path(env_vars.get('LLAMACPP_MODELS_DIR'))
llama_model_arg = re.search(regex, llama_server_args)
if llama_models_dir:
if os.path.isdir(llama_models_dir):
default_models_dir = normalize_path(os.path.join('llama.cpp','models'))
if llama_models_dir != default_models_dir and os.listdir(llama_models_dir):
regex = r"\b{--models-dir}\b"
if not re.search(regex, llama_server_args):
llama_args.extend(["--models-dir", llama_models_dir])
else:
print(f"Error: Models directory {llama_models_dir} does not exist.")
can_download = True if llama_hf_repo else False
if llama_server_args:
llama_args.extend([llama_server_args])
if not llama_hf_repo:
using_hf = False
if llama_model_arg:
using_hf = llama_model_arg.group() in ['-hf', '--hf-file']
can_download = llama_model_arg.group() not in ['-m', '--model']
llama_model = check_llama_cpp_model(operation, env_vars, using_hf)
if llama_model:
if not llama_model_arg:
llama_args.extend(['-hf'])
llama_args.extend([llama_model])
elif operation == 'install':
sys.exit(1)
if can_download and operation in ['install', 'update']:
print(f"""Configured to download {llama} model (this may take a while).\n
Check container log for download progress...""")
else:
llama_server_args = env_vars.get('OLLAMA_SERVER_ARGS')
if llama_server_args:
llama_args.extend([llama_server_args])
args = " ".join(llama_args)
launch_llama_process(args)
else:
print(textwrap.dedent(f"""\
Error: The {ollama_app} file was not found at {ollama_exe}.
If Ollama is installed in a non-standard location, you can set the OLLAMA_PATH
environment variable with its full path (including the Ollama file) to .env and
re-run this file - exiting...
Error: The {llama_app} file was not found at {llama_exe}.
If {llama} is installed in a non-standard location, you can set the LLAMA_PATH
environment variable with its full path (including the {llama} file) in .env and
re-run {info.get("file")} - exiting...
"""))
sys.exit(1)
@@ -324,7 +418,7 @@ def operate_ai_suite(operation, profile, environment, env_vars):
operation = "stop"
with open('.operation', 'w') as f:
f.write(operation)
f.write(operation + ':' + llama.lower())
supabase = False
open_webui = False
@@ -852,20 +946,41 @@ def main():
name = info.get('name', 'placeholder')
file = info.get('file', 'placeholder.py')
version = info.get('version', (-1, -1, -1))
print(f"""{name} version: {'.'.join(map(str, version))}""")
# Detect operational status and current llama (Ollama/Llama.cpp) configuration
global llama, llama_cpp
status = None
llama_cpp = False
if os.path.exists('.operation'):
with open('.operation', 'r') as f:
a = f.readline().split(':')
status = a[0].strip() if a else None
if len(a) > 1:
llama_cpp = a[1].strip() == 'llama.cpp'
else:
lp = dotenv.get_key(os.path.join(".env"), 'LLAMA_PATH')
llama_cpp = lp and os.path.basename(lp).lower().startswith('llama-server')
llama = "Llama.cpp" if llama_cpp else "Ollama"
# Banner
print(f"""{name} version: {'.'.join(map(str, version))} LLM: {llama}""")
# Profile, environment and operation arguments
global open_webui_all_profiles
ollama_profiles = ['cpu', 'gpu-nvidia', 'gpu-amd']
n8n_profiles = ["n8n", "n8n-all"]
llama_host_profiles = ['ollama', 'llama.cpp']
ollama_docker_profiles = ['cpu', 'gpu-nvidia', 'gpu-amd']
llamacpp_docker_profiles = ['cpp-cpu', 'cpp-gpu-nvidia', 'cpp-gpu-amd']
llama_docker_profiles = ollama_docker_profiles + llamacpp_docker_profiles
n8n_profiles = ['n8n', 'n8n-all']
n8n_all_profiles = n8n_profiles + ['ai-all']
open_webui_utils_profiles = ['open-webui-mcpo', 'open-webui-pipe']
open_webui_profiles = ['open-webui', 'open-webui-all']
open_webui_all_profiles = open_webui_profiles + n8n_all_profiles
agent_all_profiles = open_webui_all_profiles + ['opencode']
server_profiles = ['supabase', 'flowise', 'searxng', 'langfuse', 'neo4j', 'caddy']
profiles = agent_all_profiles + open_webui_utils_profiles + server_profiles + ollama_profiles
operations = ['stop', 'stop-ollama', 'start', 'pause', 'unpause', 'update', 'install']
profiles = agent_all_profiles + open_webui_utils_profiles + server_profiles + \
llama_docker_profiles + llama_host_profiles
operations = ['stop', 'stop-llama', 'start', 'pause', 'unpause', 'update', 'install']
environments = ['private', 'public']
parser = argparse.ArgumentParser(
prog=f'{file}',
@@ -884,23 +999,27 @@ def main():
Example commands:
- Install profile arguments n8n and opencode...
...with Ollama running in the Host:
...with {llama} running in the Host:
>python {file} --profile n8n opencode
...with Ollama runing in Docker using CPU:
...with Ollama CPU running in Docker:
>python {file} --profile n8n opencode cpu
...using GPU and in public (production) environment:
>python {file} --profile n8n opencode --environment public
...using Llama.cpp AMD GPU in Docker and on production environment:
>python {file} --profile n8n opencode cpp-gpu-amd --environment public
- Perform stop (start, pause, unpause) suite operation:
>python {file} --profile n8n opencode cpu --operation stop
>python {file} --profile n8n opencode --operation stop
- Perform suite operation to update all modules and restart:
>python {file} --operation update
- Perform suite operation to install all functional modules and start:
- Perform suite operation to install all functional modules and start...
>python {file} --operation install
...using Llama.cpp Nvidia GPU running in Docker:
>python {file} --profile ai-all cpp-gpu-nvidia --operation install
'''),
epilog=textwrap.dedent(f'''\
- Title: {info.get("title")}
@@ -913,14 +1032,14 @@ def main():
- Copyright: {info.get("copyright")}
'''))
parser.add_argument('-p', '--profile', type=str, nargs='+', choices=profiles,
help='Docker Compose Profile arguments for functional modules and Ollama'
'CPU/GPU options (default: open-webui - with Ollama running on Host)')
help='Docker Compose Profile arguments for functional modules and llama'
f'CPU/GPU options (default: open-webui - with {llama} running on Host)')
parser.add_argument('-e', '--environment', type=str, choices=environments, default='private',
help='Environment arguments used by Docker Compose to expose '
'or restrict network communication ports (default: private)')
parser.add_argument('-o', '--operation', type=str, choices=operations,
help='Docker container, volumes and image management arguments '
'along with argument to stop Ollama running on Host.')
help='Docker container, volumes and image management arguments along '
f'with argument to stop {llama} running on Host.')
args = parser.parse_args()
@@ -942,51 +1061,89 @@ def main():
env_vars = get_dotenv_vars()
if not env_vars:
sys.exit(1)
# Detect default profile - no arguments specified
default_profile = False if args.profile else True
# Set default projects path in working environment variables
if any(p for p in args.profile if p in agent_all_profiles):
if not env_vars['PROJECTS_PATH']:
env_vars['PROJECTS_PATH'] = normalize_path(os.path.join('~', 'projects'))
args.profile = [] if default_profile else args.profile
# Check Ollama status when running Ollama in the Host
ollama_in_host = default_profile or not \
any(profile for profile in args.profile if profile in ollama_profiles)
if ollama_in_host:
global ollama_found, ollama_app, ollama_exe, attempted_launch
ollama_found = False
# Check llama (Ollama/Llama.cpp) status
llama_arg = "cpu"
llama_in_host = default_profile or not \
any(p for p in args.profile if p in llama_docker_profiles)
if llama_in_host:
global llama_found, llama_app, llama_exe, attempted_launch
attempted_launch = False
if system == "Windows":
ollama_app = 'ollama.exe'
ollama_exe = os.path.join(os.path.expanduser('~'), 'AppData\\Local\\Programs\\Ollama', ollama_app)
ollama_found = os.path.exists(ollama_exe)
else: # Unix-based systems (Linux, macOS)
ollama_app = 'ollama'
for ollama_path in ['/bin', '/usr/local/bin', '/usr/bin']:
ollama_exe = os.path.join(ollama_path, ollama_app)
if os.path.exists(ollama_exe):
ollama_found = True
break
if not ollama_found:
ollama_path = dotenv.get_key(env_file, 'OLLAMA_PATH')
if ollama_path is None:
ollama_path = os.environ.get('OLLAMA_PATH')
if ollama_path is not None:
ollama_exe = os.path.normpath(ollama_path)
ollama_app = os.path.basename(ollama_exe)
ollama_found = os.path.exists(ollama_exe)
check_ollama_process(args.operation)
llama_found = False
llama_path = normalize_path(env_vars.get('LLAMA_PATH'))
llama_cpp = any(p for p in args.profile if p == 'llama.cpp')
llama = "Llama.cpp" if llama_cpp else "Ollama"
if llama_path is not None:
llama_exe = os.path.normpath(llama_path)
llama_app = os.path.basename(llama_exe)
llama_found = os.path.exists(llama_exe)
if not llama_found:
llama_app = "llama-server" if llama_cpp else "ollama"
if system == "Windows":
llama_app = "".join([llama_app, '.exe'])
llama_dir = "llama.cpp" if llama_cpp else "Ollama"
for llama_sub in ['~\\AppData\\Local\\Programs', os.getcwd()]:
llama_exe = normalize_path(os.path.join(llama_sub, llama_dir, llama_app))
if os.path.exists(llama_exe):
llama_found = True
break
else: # Unix-based systems (Linux, macOS)
for llama_path in ['/bin', '/usr/local/bin', '/usr/bin']:
llama_exe = os.path.join(llama_path, llama_app)
if os.path.exists(llama_exe):
llama_found = True
break
set_dotenv_key(os.path.join(".env"), 'LLAMA_PATH', llama_exe, None)
env_vars = get_dotenv_vars(force=True)
# Check if llama exe (llama-server, ollama) matches profile argument (llama.cpp, ollama)
if (llama_cpp and not llama_app.lower().startswith('llama-server')) or \
(not llama_cpp and not llama_app.lower().startswith('ollama')):
llama_cpp = llama_app.lower().startswith('llama-server')
llama = "Llama.cpp" if llama_cpp else "Ollama"
llama_mismatch = "ollama" if llama_cpp else "llama.cpp"
print(f"""Notice: The executable '{llama_app}' did not match the '{llama}'
profile argument - argument updated to '{llama.lower()}'...""")
args.profile.remove(llama_mismatch) if llama_mismatch in args.profile else None
args.profile.extend([llama.lower()]) if llama_cpp else None
llama_host = "host.docker.internal"
llama_host_var = llama_host if llama_cpp else llama_host + ":${OLLAMA_PORT}"
check_llama_process(args.operation, env_vars)
else:
llama_cpp = any(p for p in args.profile if p in llamacpp_docker_profiles)
llama = "Llama.cpp" if llama_cpp else "Ollama"
llama_host_var = "0.0.0.0" if llama_cpp else "ollama:${OLLAMA_PORT}"
# Check if more than one llama CPU/GPU argument specified, use first argument
if any(p for p in args.profile if p in llama_docker_profiles):
first_argument = False
for profile_arg in llama_docker_profiles:
if not first_argument:
if any(p for p in args.profile if p == profile_arg):
print(f"""{name} will use {llama} CPU/GPU profile argument '{profile_arg}'...""")
first_argument = True
else:
args.profile.remove(profile_arg) if profile_arg in args.profile else None
# Check if any llama host profile arguments specified and remove if found
for profile_arg in llama_host_profiles:
if any(p for p in args.profile if p == profile_arg):
print(f"""A llama in host profile argument cannot be specified with
llama CPU/GPU profile argument - removing '{profile_arg}'...""")
args.profile.remove(profile_arg)
# Assemble .env updates, set respective keys in .env file and reload .env vars
llama_host_env = "LLAMA_ARG_HOST" if llama_cpp else "OLLAMA_HOST"
oai_base_url_var = "${LLAMACPP_HOST}" if llama_cpp else "${OLLAMA_HOST}"
mod_env_vars.update({llama_host_env: llama_host_var, 'OPENAI_API_BASE_URL': oai_base_url_var})
for env, var in mod_env_vars.items():
set_dotenv_var(env_file, env, var, None)
env_vars = get_dotenv_vars(env_file, True)
# Process operation argument
build = False
if args.operation:
status = None
if os.path.exists('.operation'):
with open('.operation', 'r') as f:
status = f.readline()
if args.operation == 'stop-ollama':
if args.operation == 'stop-llama':
args.operation = "stop"
if args.operation == status:
if status == 'stop':
@@ -1016,8 +1173,9 @@ def main():
args.operation = 'pull'
insert = "Installing" if install == 'install' else "Updating"
if default_profile:
print(f"""{insert} all container images including Ollama...""")
args.profile.extend(["cpu"])
print(f"""{insert} all container images including {llama}...""")
llama_arg = "cpp-cpu" if llama_cpp else "cpu"
args.profile.extend([llama_arg])
else:
print(f"""{insert} container images for {args.profile}...""")
docker_compose_include(True, True, True)
@@ -1030,7 +1188,7 @@ def main():
# Manually set default profile argument
if default_profile:
if build:
args.profile.remove('cpu') if 'cpu' in args.profile else None
args.profile.remove(llama_arg) if llama_arg in args.profile else None
else:
args.profile = ['open-webui']
@@ -1132,7 +1290,7 @@ def main():
start_ai_suite(args.profile, args.environment, build)
with open('.operation', 'w') as f:
f.write('start')
f.write('start' + ':' + llama.lower())
if __name__ == "__main__":
main()