Merge pull request #279 from KonradHoeffner/pr-add-dockerfile

Add Dockerfile for CUDA

.dockerignore

@@ -0,0 +1,4 @@
+*
+!*.py
+!requirements.txt
+!SOURCE_DOCUMENTS

Dockerfile

@@ -0,0 +1,22 @@
+# syntax=docker/dockerfile:1
+# Build as `docker build . -t localgpt`, requires BuildKit.
+# Run as `docker run -it --mount src="$HOME/.cache",target=/root/.cache,type=bind --gpus=all localgpt`, requires Nvidia container toolkit.
+
+FROM nvidia/cuda:11.7.1-runtime-ubuntu22.04
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get install -y g++-11 make python3 python-is-python3 pip
+# only copy what's needed at every step to optimize layer cache
+COPY ./requirements.txt .
+# use BuildKit cache mount to drastically reduce redownloading from pip on repeated builds
+RUN --mount=type=cache,target=/root/.cache pip install --timeout 100 -r requirements.txt
+RUN --mount=type=cache,target=/root/.cache CMAKE_ARGS="-DLLAMA_CUBLAS=on" FORCE_CMAKE=1 pip install --upgrade --force-reinstall llama-cpp-python
+COPY SOURCE_DOCUMENTS ./SOURCE_DOCUMENTS
+COPY ingest.py constants.py ./
+# Docker BuildKit does not support GPU during *docker build* time right now, only during *docker run*.
+# See <https://github.com/moby/buildkit/issues/1436>.
+# If this changes in the future you can `docker build --build-arg device_type=cuda . -t localgpt` (+GPU argument to be determined).
+ARG device_type=cpu
+RUN --mount=type=cache,target=/root/.cache python ingest.py --device_type $device_type
+COPY . .
+ENV device_type=cuda
+CMD python run_localGPT.py --device_type $device_type

README.md

@@ -41,6 +41,15 @@ If you want to use BLAS or Metal with [llama-cpp](<(https://github.com/abetlen/l
 CMAKE_ARGS="-DLLAMA_CUBLAS=on" FORCE_CMAKE=1 pip install -r requirements.txt
 ```
 
+## Docker
+
+Installing the required packages for GPU inference on Nvidia GPUs, like gcc 11 and CUDA 11, may cause conflicts with other packages in your system.
+As an alternative to Conda, you can use Docker with the provided Dockerfile.
+It includes CUDA, your system just needs Docker, BuildKit, your Nvidia GPU driver and the Nvidia container toolkit.
+Build as `docker build . -t localgpt`, requires BuildKit.
+Docker BuildKit does not support GPU during *docker build* time right now, only during *docker run*.
+Run as `docker run -it --mount src="$HOME/.cache",target=/root/.cache,type=bind --gpus=all localgpt`.
+
 ## Test dataset
 
 This repo uses a [Constitution of USA ](https://constitutioncenter.org/media/files/constitution.pdf) as an example.