<![CDATA[Installation de llama-swap]]>Clonage de https://github.com/mostlygeek/llama-swap

Copie des binaires :

# cp build/* /sbin/.
# cp build/* /usr/local/sbin/.

Création de :

# mkdir /var/lib/llama-swap
# mkdir  /etc/llama-swap/

Creation du fichier service : /etc/systemd/system/multi-user.target.wants/llama-swap.service

[Unit]
Description=llama-swap - Local LLM Model Swapper & Proxy
After=network.target

[Service]
Type=simple
User=root
WorkingDirectory=/home/XXXX/Documents/llama.cpp/
Environment="NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
Environment="NVM_DIR=/root/.nvm"
Environment="NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
Environnent="PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

WorkingDirectory=/var/lib/llama-swap
ExecStart=/sbin/llama-swap-linux-amd64--config /etc/llama-swap/config.yaml --listen 0.0.0.0:8080

Restart=on-failure
RestartSec=5s
LimitNOFILE=65536

ProtectSystem=full
ProtectHome=false
PrivateTmp=true

[Install]
WantedBy=multi-user.target
]]>https://lemmy.cyber-neurones.org/topic/379/installation-de-llama-swapRSS for NodeFri, 07 Aug 2026 05:09:42 GMTWed, 22 Jul 2026 07:32:11 GMT60<![CDATA[Reply to Installation de llama-swap on Wed, 22 Jul 2026 14:24:37 GMT]]>Sur l’interface :

image.png

]]>https://lemmy.cyber-neurones.org/post/1097https://lemmy.cyber-neurones.org/post/1097Wed, 22 Jul 2026 14:24:37 GMT<![CDATA[Reply to Installation de llama-swap on Wed, 22 Jul 2026 14:22:59 GMT]]>Nouvelle connfiguration : Passage sur le modèle Qwen3.6-35B-A3B-UD-Q4_K_M.gguf

healthCheckTimeout: 500
logLevel: info
logTimeFormat: ""
logToStdout: "proxy"
store:
  path: /root/llama-swap.sqlite
metricsMaxInMemory: 1000
captureBuffer: 15
ui:
  activity:
    session_id: ["X-Session-ID", "X-Litellm-Session-Id"]
performance:
  disabled: false
  every: 15s
startPort: 10001
sendLoadingState: true
includeAliasesInList: false
globalTTL: 0
unloadTimeout: 10
macros:
  "default_ctx": 40000
  "default_args": "--ctx-size ${default_ctx}"
models:
  "Best_Qwen3.6-35B-A3B-UD-Q4_K_M.gguf":
    proxy: http://127.0.0.1:8999
    name: "Best_Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"
    description: "-m /models/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf --ctx-size ${default_ctx} --temp ${temp} --top-p 0.95 --top-k 20 --min-p 0.00  --reasoning-preserve  --host 0.0.0.0 --fit off -ngl 40  --port 8999"
    cmd: /usr/local/bin/llama-server -m /models/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf --ctx-size ${default_ctx} --temp ${temp} --top-p 0.95 --top-k 20 --min-p 0.00  --reasoning-preserve  --host 0.0.0.0 --fit off -ngl 90  --port 8999 --n-cpu-moe 35
    aliases:
      - "best-Qwen3.6-35B-A3B-UD-Q4_K_MM"
      - "llama-swap"
    macros:
      "default_ctx": 140000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ttl: 60
    unloadTimeout: 30
]]>https://lemmy.cyber-neurones.org/post/1096https://lemmy.cyber-neurones.org/post/1096Wed, 22 Jul 2026 14:22:59 GMT<![CDATA[Reply to Installation de llama-swap on Wed, 22 Jul 2026 13:48:46 GMT]]>Pas plus de 3 t/s.

image.png

]]>https://lemmy.cyber-neurones.org/post/1095https://lemmy.cyber-neurones.org/post/1095Wed, 22 Jul 2026 13:48:46 GMT<![CDATA[Reply to Installation de llama-swap on Wed, 22 Jul 2026 12:26:26 GMT]]>La configuration finale :

Hermes: /root/.hermes/config.yaml

model:
  default: "llama-swap"
  provider: custom
  base_url: http://127.0.0.1:8080/v1
  api_key: "llama-swap"

Llama-swap : /etc/llama-swap/config.yaml

healthCheckTimeout: 500
logLevel: info
logTimeFormat: ""
logToStdout: "proxy"

# store: persistent storage for activity metrics state
# - optional, default: in-memory sqlite database capped by metricsMaxInMemory
# - path is a sqlite database file path
# - file-backed sqlite keeps activity logs across restarts
# store:
#   path: /path/to/file.sqlite

metricsMaxInMemory: 1000
captureBuffer: 15
ui:
  activity:
    session_id: ["X-Session-ID", "X-Litellm-Session-Id"]

performance:
  disabled: false
  every: 15s

startPort: 10001

sendLoadingState: true

includeAliasesInList: false

globalTTL: 0

unloadTimeout: 10

macros:
  "default_ctx": 40000
  "default_args": "--ctx-size ${default_ctx}"

models:
  "Best_Qwen3.6-27B-Q3_K_M.gguf_mmproj-BF16.gguf":
    proxy: http://127.0.0.1:8999
    name: "Best_Qwen3.6-27B-Q3_K_M"
    description: "-m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf --ctx-size ${default_ctx} --temp ${temp} --top-p 0.95 --top-k 20 --min-p 0.00  --reasoning-preserve  --no-mmproj-offload --host 0.0.0.0 --fit off -ngl 40  --port 8999"
    cmd: /usr/local/bin/llama-server  -m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf --ctx-size ${default_ctx} --temp ${temp} --top-p 0.95 --top-k 20 --min-p 0.00  --reasoning-preserve  --no-mmproj-offload --host 0.0.0.0 --fit off -ngl 40  --port 8999
    aliases:
      - "best-Qwen3.6-27B-Q3_K_M"
      - "llama-swap"
    macros:
      "default_ctx": 140000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ttl: 60
    unloadTimeout: 30
  "Qwen3.6-27B-Q3_K_M.gguf":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server  -m /models/Qwen3.6-27B-Q3_K_M.gguf --ctx-size ${default_ctx}  --temp 1.0  --top-p 0.95 --top-k 20 --min-p 0.00  --reasoning-preserve --host 0.0.0.0 --fit off -ngl 40  --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ttl: 60
    unloadTimeout: 10
  "Qwen3.6-27B-Q3_K_M.gguf_mmproj-BF16.gguf_CUDA0":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server   -m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf  --ctx-size ${default_ctx}  --temp ${temp}  --device CUDA0 --top-p 0.95  --top-k 20 --min-p 0.00 --host 0.0.0.0 --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ttl: 60
    unloadTimeout: 10
  "Qwen3.6-27B-Q3_K_M.gguf_mmproj-BF16.gguf_CUDA":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server   -m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf  --ctx-size ${default_ctx}  --temp ${temp}  --device CUDA1 --top-p 0.95  --top-k 20 --min-p 0.00 --host 0.0.0.0 --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
  "Qwen3.6-27B-Q3_K_M.gguf_mmproj-BF16.gguf_init":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server  -m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf --ctx-size ${default_ctx} --n-cpu-moe 35 --temp ${temp}  --top-p 0.95 --top-k 20 --min-p 0.00 --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
  "gemma-3-1b-it-q4_k_m.gguf":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server  -m /models/gemma-3-1b-it-q4_k_m.gguf --ctx-size ${default_ctx} --n-cpu-moe 35 --temp ${temp}  --top-p 0.95 --top-k 20 --min-p 0.00 --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ttl: 60
    unloadTimeout: 30

Cela fonctionne :

image.png

]]>https://lemmy.cyber-neurones.org/post/1094https://lemmy.cyber-neurones.org/post/1094Wed, 22 Jul 2026 12:26:26 GMT<![CDATA[Reply to Installation de llama-swap on Wed, 22 Jul 2026 11:21:51 GMT]]>Ma configuration :

healthCheckTimeout: 500
logLevel: info
logTimeFormat: ""
logToStdout: "proxy"

metricsMaxInMemory: 1000
captureBuffer: 15
ui:
  activity:
    session_id: ["X-Session-ID", "X-Litellm-Session-Id"]

performance:
  disabled: false
  every: 15s

startPort: 10001

sendLoadingState: true

includeAliasesInList: false

globalTTL: 0

unloadTimeout: 10

macros:
  "default_ctx": 40000
  "default_args": "--ctx-size ${default_ctx}"

models:
  "Best_Qwen3.6-27B-Q3_K_M.gguf_mmproj-BF16.gguf":
    proxy: http://127.0.0.1:8999
    name: "Best_Qwen3.6-27B-Q3_K_M"
    description: "-m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf --ctx-size ${default_ctx} --temp ${temp} --top-p 0.95 --top-k 20 --min-p 0.00  --reasoning-preserve  --no-mmproj-offload --host 0.0.0.0 --fit off -ngl 40  --port 8999"
    cmd: /usr/local/bin/llama-server  -m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf --ctx-size ${default_ctx} --temp ${temp} --top-p 0.95 --top-k 20 --min-p 0.00  --reasoning-preserve  --no-mmproj-offload --host 0.0.0.0 --fit off -ngl 40  --port 8999
    aliases:
      - "best-Qwen3.6-27B-Q3_K_M"
    macros:
      "default_ctx": 140000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ttl: 60
    unloadTimeout: 30
  "Qwen3.6-27B-Q3_K_M.gguf":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server  -m /models/Qwen3.6-27B-Q3_K_M.gguf --ctx-size ${default_ctx}  --temp 1.0  --top-p 0.95 --top-k 20 --min-p 0.00  --reasoning-preserve --host 0.0.0.0 --fit off -ngl 40  --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ttl: 60
    unloadTimeout: 10
  "Qwen3.6-27B-Q3_K_M.gguf_mmproj-BF16.gguf_CUDA0":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server   -m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf  --ctx-size ${default_ctx}  --temp ${temp}  --device CUDA0 --top-p 0.95  --top-k 20 --min-p 0.00 --host 0.0.0.0 --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ttl: 60
    unloadTimeout: 10
  "Qwen3.6-27B-Q3_K_M.gguf_mmproj-BF16.gguf_CUDA":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server   -m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf  --ctx-size ${default_ctx}  --temp ${temp}  --device CUDA1 --top-p 0.95  --top-k 20 --min-p 0.00 --host 0.0.0.0 --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
  "Qwen3.6-27B-Q3_K_M.gguf_mmproj-BF16.gguf_init":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server  -m /models/Qwen3.6-27B-Q3_K_M.gguf --mmproj /models/mmproj-BF16.gguf --ctx-size ${default_ctx} --n-cpu-moe 35 --temp ${temp}  --top-p 0.95 --top-k 20 --min-p 0.00 --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
  "gemma-3-1b-it-q4_k_m.gguf":
    proxy: http://127.0.0.1:8999
    cmd: /usr/local/bin/llama-server  -m /models/gemma-3-1b-it-q4_k_m.gguf --ctx-size ${default_ctx} --n-cpu-moe 35 --temp ${temp}  --top-p 0.95 --top-k 20 --min-p 0.00 --port 8999
    macros:
      "default_ctx": 100000
      "temp": 1.0
    metadata:
      temperature: ${temp}
      note: "The ${MODEL_ID} is running on port 8999 temp=${temp}, context=${default_ctx}"
    env:
      - "NVM_BIN=/root/.nvm/versions/node/v24.17.0/bin"
      - "NVM_DIR=/root/.nvm"
      - "NVM_INC=/root/.nvm/versions/node/v24.17.0/include/node"
      - "PATH=/root/.nvm/versions/node/v24.17.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    ttl: 60
    unloadTimeout: 30
]]>https://lemmy.cyber-neurones.org/post/1093https://lemmy.cyber-neurones.org/post/1093Wed, 22 Jul 2026 11:21:51 GMT