Begin
Building llama.cpp for rocm
From lama.cpp use .devops/rocm.Dockerfile
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
WORKDIR /app
@@ -137,7 +138,7 @@ FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
docker build --target server\
--build-arg ROCM_DOCKER_ARCH=gfx1151 \
-t llama-cpp:rocm-gfx1151 \
-f .devops/rocm.Dockerfile .
docker run --rm -it --device=/dev/kfd \
--device=/dev/dri \
-v /home/airmack/modells:/models:ro \
llama-cpp:rocm-gfx1151 \
-m /models/glm-4.5/GLM-4.5-Air-UD-Q4_K_XL-00001-of-00002.gguf \
-ngl 999 \
-fa on \
--ctx-size 32768 \
--host 0.0.0.0 --port 8080
Building llama.cpp for intel E5-2680
From lama.cpp use .devops/cpu.Dockerfile
index cb92343d6..daee9ead3 100644
--- a/.devops/cpu.Dockerfile
+++ b/.devops/cpu.Dockerfile
@@ -33,8 +33,16 @@ COPY . .
COPY --from=web /app/tools/ui/dist tools/ui/dist
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
- cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON; \
- else \
+ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=OFF -DGGML_CPU_ALL_VARIANTS=OFF \
+ -DGGML_AVX=ON \
+ -DGGML_AVX2=OFF \
+ -DGGML_FMA=OFF \
+ -DGGML_F16C=ON \
+ -DGGML_BMI2=OFF \
+ -DGGML_LTO=ON \
+ -DCMAKE_C_FLAGS="-march=ivybridge -mtune=ivybridge" \
+ -DCMAKE_CXX_FLAGS="-march=ivybridge -mtune=ivybridge"; \
+ else \
echo "Unsupported architecture"; \
exit 1; \
fi && \
@@ -104,7 +112,7 @@ ENTRYPOINT ["/app/tools.sh"]
### Light, CLI only
FROM base AS light
-COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app
+COPY --from=build /app/full/llama /app/full/llama-cli /app/full/llama-completion /app/
WORKDIR /app
@@ -115,7 +123,7 @@ FROM base AS server
ENV LLAMA_ARG_HOST=0.0.0.0
-COPY --from=build /app/full/llama /app/full/llama-server /app
+COPY --from=build /app/full/llama /app/full/llama-server /app/
WORKDIR /app
docker-compose up
Nächster schritt
Aktuell sind alle docker und auch der hauptcontainer im host-netzwerk. Das soll so nicht sein. Diese müssen umgezogen werden. Host soll auf :9292 laufen und die anderen container müssen ansonsten schmatzen. Da muss ich die KI Fragen ob es weiter geht
Alle container sind kjetzt im ollama Netz Nächster Schritt SSL zugriff für llama-swap.skynet.lan beantragt.
- NGINX muss konfiguriert werden für llama-swap.skynet.lan
Fehler ist noch immer nicht verstanden. Warum garbelt llama.cpp? Zu wenig context? (shrug)
Caveats
Modelle würgen unverständlichen nonsense aus. Warum? Keine Ahnung, eventuel Kontext voll. Beispiel
Links
- https://github.com/mostlygeek/llama-swap
