离线部署大模型

docker安装ollama

docker pull ollama/ollama
docker save ollama/ollama -o ollama.tar
docker load -i ollama.tar
# 运行容器
docker run -d -v /root/.ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama:0.11.4

# 使用zmy用户启动容器，如果已有大模型文件，设置环境变量让ollama找到文件
docker run -d \
  --name ollama \
  --user $(id -u zmy):$(id -g zmy) \
  -v /home/zmy/.ollama:/home/zmy/.ollama \
  -e OLLAMA_MODELS=/home/zmy/.ollama/models \
  -e OLLAMA_HOST=0.0.0.0:11434 \
  -p 11434:11434 \
  ollama/ollama:0.11.4
  
docker run -d \
  --name ollama \
  --gpus all \  # 如果有 GPU
  --restart
  --security-opt apparmor=unconfined \
  --security-opt seccomp=unconfined \
  --cap-add SYS_ADMIN \
  --cap-add MKNOD \
  --cap-add DAC_READ_SEARCH \
  -e OLLAMA_HOST=0.0.0.0:11434 \
  -v /root/.ollama:/root/.ollama \
  -p 11434:11434 \
  ollama/ollama:0.11.4

# 执行用户进入容器
docker exec -u $(id -u zmy) -it ollama ollama list

# 管理模型
docker exec -it ollama ollama run llama3.2
docker exec -it ollama ollama pull llama3
docker exec -it ollama ollama ls
docker exec -it ollama ollama ps

Ollama 提供了完整的 REST API：

列出模型
GET http://localhost:11434/api/tags
拉取模型
POST http://localhost:11434/api/pull
删除模型
DELETE http://localhost:11434/api/delete
推理
POST http://localhost:11434/api/generate

curl http://localhost:11434/api/pull -d '{
  "name": "qwen:7b"
}'

安装ollama

# Download and extract the package:
curl -LO https://ollama.com/download/ollama-linux-amd64.tgz
sudo tar -C /usr -xzf ollama-linux-amd64.tgz

# Start Ollama:
ollama serve

# In another terminal, verify that Ollama is running:
ollama -v

卸载ollama

# Remove the ollama service:
sudo systemctl stop ollama
sudo systemctl disable ollama
sudo rm /etc/systemd/system/ollama.service

# Remove the ollama binary from your bin directory (either /usr/local/bin, /usr/bin, or /bin):
sudo rm $(which ollama)

# Remove the downloaded models and Ollama service user and group:
sudo rm -r /usr/share/ollama
sudo userdel ollama
sudo groupdel ollama

# Remove installed libraries:
sudo rm -rf /usr/local/lib/ollama

# 安装为服务
sudo useradd -r -s /bin/false -U -m -d /usr/share/ollama ollama
sudo usermod -a -G ollama $(whoami)
sudo cat > /etc/systemd/system/ollama.service << 'EOF'
[Unit]
Description=Ollama Service
After=network-online.target

[Service]
ExecStart=/usr/bin/ollama serve
User=ollama
Group=ollama
Restart=always
RestartSec=3
Environment="PATH=$PATH"

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable ollama
sudo systemctl start ollama
sudo systemctl status ollama

升级GLIBC_2.27

wget http://ftp.gnu.org/gnu/glibc/glibc-2.28.tar.gz

tar xf glibc-2.28.tar.gz 
cd glibc-2.28/ && mkdir build  && cd build
../configure --prefix=/usr --disable-profile --enable-add-ons --with-headers=/usr/include --with-binutils=/usr/bin
make && make install

configure: error: 
*** These critical programs are missing or too old: compiler
*** Check the INSTALL file for required versions.

configure: error: 
*** LD_LIBRARY_PATH shouldn't contain the current directory when
*** building glibc. Please change the environment variable
*** and run configure again

安装 make-4.3

wget https://ftp.gnu.org/gnu/make/make-4.3.tar.gz
tar -xzvf make-4.3.tar.gz
cd make-4.3
./configure --prefix=/usr/local/make
make && make install

# 替换旧版本
mv /usr/bin/make /usr/bin/make.bak  # 备份旧版本
ln -sv /usr/local/make/bin/make /usr/bin/make
make --version  # 验证版本是否为 4.3

安装 gcc-8.5.0

  1. 安装GMP
wget https://gmplib.org/download/gmp/gmp-6.2.1.tar.bz2
tar -xvf gmp-6.2.1.tar.bz2
cd gmp-6.2.1
./configure --prefix=/usr/local/gmp-6.2.1
make && make install
  1. 安装MPFR
wget https://www.mpfr.org/mpfr-4.2.0/mpfr-4.2.0.tar.gz
tar -xvf mpfr-4.2.0.tar.gz
cd mpfr-4.2.0
./configure --prefix=/usr/local/mpfr-4.2.0 --with-gmp-include=/usr/local/gmp-6.2.1/include --with-gmp-lib=/usr/local/gmp-6.2.1/lib
make && make install
  1. 安装MPC
wget https://ftp.gnu.org/gnu/mpc/mpc-1.2.1.tar.gz
tar -xvf mpc-1.2.1.tar.gz
cd mpc-1.2.1
./configure --prefix=/usr/local/mpc-1.2.1 --with-gmp-include=/usr/local/gmp-6.2.1/include --with-gmp-lib=/usr/local/gmp-6.2.1/lib --with-mpfr-include=/usr/local/mpfr-4.2.0/include --with-mpfr-lib=/usr/local/mpfr-4.2.0/lib
make && make install
  1. 将GMP、MPFR和MPC的库文件路径添加到的LD_LIBRARY_PATH环境变量中。
export LD_LIBRARY_PATH=/usr/local/gmp-6.2.1/lib:/usr/local/mpfr-4.2.0/lib:/usr/local/mpc-1.2.1/lib:$LD_LIBRARY_PATH
  1. 安装gcc
wget https://ftp.gnu.org/gnu/gcc/gcc-8.5.0/gcc-8.5.0.tar.gz
tar -xvf gcc-8.5.0.tar.gz
cd gcc-8.5.0
./configure --prefix=/usr/local/gcc-8.5.0 --with-gmp-include=/usr/local/gmp-6.2.1/include --with-gmp-lib=/usr/local/gmp-6.2.1/lib --with-mpfr-include=/usr/local/mpfr-4.2.0/include --with-mpfr-lib=/usr/local/mpfr-4.2.0/lib --with-mpc-include=/usr/local/mpc-1.2.1/include --with-mpc-lib=/usr/local/mpc-1.2.1/lib --disable-multilib
make && make install
ℹ️note

加入 –disable-multilib 是因为这里没有32位开发库，否则会报错。

  1. 系统配置
# 更新gcc
echo 'export PATH=/usr/local/gcc-8.5.0/bin:$PATH' >> ~/.bashrc

# 1. 注册所有第三方库路径
sudo tee /etc/ld.so.conf.d/gcc-deps.conf << 'EOF'
/usr/local/gmp-6.2.1/lib
/usr/local/mpfr-4.2.0/lib
/usr/local/mpc-1.2.1/lib
/usr/local/gcc-8.5.0/lib64
EOF

# 2. 更新缓存
sudo ldconfig

# 3. 验证 cc1 的依赖
ldd /usr/local/gcc-8.5.0/libexec/gcc/x86_64-pc-linux-gnu/8.5.0/cc1 | grep mpfr
# 应该显示：libmpfr.so.6 => /usr/local/mpfr-4.2.0/lib/libmpfr.so.6

docker 安装 open-webui

#外网机器：拉取镜像
docker pull ghcr.io/open-webui/open-webui:latest
# 拉取包含cuda加速的镜像
docker pull ghcr.io/open-webui/open-webui:cuda

#外网机器：导出
docker save -o open-webui.tar ghcr.io/open-webui/open-webui:latest

#将 open-webui.tar 拷贝到内网机器，内网机器：
docker load -i open-webui.tar

#内网机器：运行镜像，修改 /app/backend/data 和端口
# 1.如果本机上有 Ollama，请使用以下命令：
docker run -d -p 3000:8080 --add-host=host.docker.internal:host-gateway -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:latest

# 2.如果 Ollama 位于不同的服务器上，请使用以下命令：修改 OLLAMA_BASE_URL
docker run -d -p 3000:8080 -e OLLAMA_BASE_URL=https://example.com -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:latest

# 3.要运行支持 Nvidia GPU 的 Open WebUI，请使用以下命令：
docker run -d -p 3000:8080 --gpus all --add-host=host.docker.internal:host-gateway -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:cuda

# 访问open-webui
http://localhost:8080/