1、安装gcc
#安装编译环境
yum -y install make gcc gcc-c++
2、下载显卡驱动
点击 直达连接
nvidia高级搜索下载历史版本驱动程序（下载历史版本驱动）
https://www.nvidia.cn/Download/Find.aspx?lang=cn
3、安装驱动
安装显卡驱动
./NVIDIA-Linux-x86_64-535.98.run -m=kernel-open
4、修改系统参数，更新内核，重启服务器
rm -f /etc/modprobe.d/blacklist-nvidia-nouveau.conf /etc/modprobe.d/nvidia-unsupported-gpu.conf
echo blacklist nouveau | tee /etc/modprobe.d/blacklist-nvidia-nouveau.conf && \echo options nouveau modeset=0 | tee -a /etc/modprobe.d/blacklist-nvidia-nouveau.conf && \echo options nvidia NVreg_OpenRmEnableUnsupportedGpus=1 | tee /etc/modprobe.d/nvidia-unsupported-gpu.conf && \dracut --force && \/sbin/reboot
5、检查驱动
执行nvidia-smi
Wed Aug 16 13:46:06 2023
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.98 Driver Version: 535.98 CUDA Version: 12.2 |
|-----------------------------------------+----------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+======================+======================|
| 0 NVIDIA GeForce RTX 3090 Off | 00000000:13:00.0 Off | N/A |
| 32% 21C P8 8W / 350W | 4MiB / 24576MiB | 0% Default |
| | | N/A |
+-----------------------------------------+----------------------+----------------------++---------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=======================================================================================|
| No running processes found |
+---------------------------------------------------------------------------------------+
6、安装nvidia-container-runtime
#安装源
curl -s -L https://nvidia.github.io/libnvidia-container/centos8/libnvidia-container.repo | sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo
#安装容器运行时
yum install -y nvidia-container-runtime
7、修改containerd配置文件
7.1、增加如下配置
[plugins."io.containerd.runtime.v1.linux"]no_shim = falseruntime = "nvidia-container-runtime"runtime_root = ""shim = "containerd-shim"shim_debug = false
7.2、修改container配置
修改前：runtime_type = "io.containerd.runc.v2"
修改后：runtime_type = "io.containerd.runtime.v1.linux"
7.3、完整配置文件
[root@ai-4 containerd]# pwd
/etc/containerd
[root@ai-4 containerd]# cat config.toml
version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
oom_score = 0[grpc]address = "/run/containerd/containerd.sock"uid = 0gid = 0max_recv_message_size = 16777216max_send_message_size = 16777216[debug]address = "/run/containerd/containerd-debug.sock"uid = 0gid = 0level = "warn"[timeouts]"io.containerd.timeout.shim.cleanup" = "5s""io.containerd.timeout.shim.load" = "5s""io.containerd.timeout.shim.shutdown" = "3s""io.containerd.timeout.task.state" = "2s"[plugins][plugins."io.containerd.grpc.v1.cri"]sandbox_image = "sealos.hub:5000/pause:3.2"max_container_log_line_size = -1max_concurrent_downloads = 20disable_apparmor = true[plugins."io.containerd.grpc.v1.cri".containerd]snapshotter = "overlayfs"default_runtime_name = "runc"[plugins."io.containerd.grpc.v1.cri".containerd.runtimes][plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]runtime_type = "io.containerd.runtime.v1.linux"runtime_engine = ""runtime_root = ""[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]SystemdCgroup = true[plugins."io.containerd.grpc.v1.cri".registry]config_path = "/etc/containerd/certs.d"[plugins."io.containerd.grpc.v1.cri".registry.configs][plugins."io.containerd.grpc.v1.cri".registry.configs."sealos.hub:5000".auth]username = "admin"password = "***********"[plugins."io.containerd.runtime.v1.linux"]no_shim = falseruntime = "nvidia-container-runtime"runtime_root = ""shim = "containerd-shim"shim_debug = false
8、测试containerd下显卡是否正常加载显卡
[root@ai-4 containerd]# ctr run --rm --gpus 0 docker.io/nvidia/cuda:11.0.3-base-ubuntu20.04 nvidia-smi nvidia-smi
Wed Aug 16 05:57:19 2023
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.98 Driver Version: 535.98 CUDA Version: 12.2 |
|-----------------------------------------+----------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+======================+======================|
| 0 NVIDIA GeForce RTX 3090 Off | 00000000:13:00.0 Off | N/A |
| 32% 21C P8 8W / 350W | 4MiB / 24576MiB | 0% Default |
| | | N/A |
+-----------------------------------------+----------------------+----------------------++---------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=======================================================================================|
| No running processes found |
+---------------------------------------------------------------------------------------+9、K8S部署插件支持显卡（如果没有部署可通过如下命令部署，K8S Master上执行）
kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.7.1/nvidia-device-plugin.yml
10、K8S检查对应节点是否有GPU资源
[root@k8s-master-17227100216 ~]# kubectl describe node node9 |grep gpugpu/type=nvidianvidia.com/gpu: 1nvidia.com/gpu: 1nvidia.com/gpu 0 0
11、部署GPU测试容器
apiVersion: v1
kind: Pod
metadata:name: cuda-vector-add
spec:restartPolicy: OnFailurecontainers:- name: cuda-vector-add#image: "k8s.gcr.io/cuda-vector-add:v0.1"image: "docker.io/nvidia/cuda:11.0.3-base-ubuntu20.04"command:- nvidia-smiresources:limits:nvidia.com/gpu: 1
