← Back to Index

给 openshift 的 coreos 编译内核驱动 rpm

作者有文档和项目，描述了如何编译设备的内核驱动，但是在 openshift 这里，rh-coreos用的 kernel 是高级订阅才有的，我们没办法弄一个和 rh-coreos 相同内核的 rhel 出来，也就没办法继续编译 .ko 了。

好在 openshift 发行版给了一个容器，里面有高级订阅才有的kernel版本开发包，可以帮助我们把这个 .ko 给编译出来，进而编译一个 rpm 包出来。那么我们今天就一步一步做做看。

制作一个工具镜像

openshift 发行版，自带一个 driver-toolkit 镜像，里面有 kernel 相关的开发包，满足了编译的需求，我们的目标是编译一个 rpm，那么我们就需要补充完善这个工具镜像。

OCP_VERSION=$(oc get clusterversion/version -ojsonpath={.status.desired.version})
        DRIVER_TOOLKIT_IMAGE=$(oc adm release info $OCP_VERSION --image-for=driver-toolkit)
        
        echo $OCP_VERSION
        
        # 4.11.39
        
        echo $DRIVER_TOOLKIT_IMAGE
        
        # quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfed734e35163b1ab8483568780d13b528b4c0f558f8e727538af723b7a41ed4
        
        # build a new image based on driver toolkit
        
        # on a rhel
        
        mkdir -p /data/driver
        cd /data/driver
        
        cat << EOF > docker.file
        FROM quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfed734e35163b1ab8483568780d13b528b4c0f558f8e727538af723b7a41ed4
        
        RUN dnf install -y rpm-build 
        RUN cd /root && git clone https://github.com/wangzheng422/nic-rpm-rnp
        RUN cd /root/nic-rpm-rnp && git checkout ocp-4.11.36
        RUN mv /root/nic-rpm-rnp/rpmbuild /root/
        EOF
        
        podman build --no-cache --authfile /data/pull-secret.json -t quay.io/wangzheng422/driver-toolkit:nic-rpm-rnp-v03 -f docker.file .
        
        podman push quay.io/wangzheng422/driver-toolkit:nic-rpm-rnp-v03

在 openshift 里面编译 rpm

我们有了工具镜像，就可以用特权模式运行它，然后到这个 pod 里面，去运行编译命令，完成 rpm 的编译。


        # come back to your cluster
        
        # https://master.sdk.operatorframework.io/docs/best-practices/pod-security-standards/
        
        oc create ns driver-build
        oc label --overwrite ns driver-build \
           pod-security.kubernetes.io/enforce=privileged
        
        # oc create serviceaccount -n driver-build demo-app
        
        # oc adm policy add-scc-to-user privileged -z demo-app -n driver-build
        
        cat << EOF > ~/wzh/build.yaml
        apiVersion: v1
        kind: Pod
        metadata:
          name: kmod-driver-samplepod
          annotations:
            openshift.io/scc: privileged
            # openshift.io/scc: restricted-v2
        
        spec:
          # serviceAccountName: demo-app
          containers:
          - image: quay.io/wangzheng422/driver-toolkit:nic-rpm-rnp-v03
            name: simple-kmod-driver-container
            imagePullPolicy: Always
            command: [sleep, infinity]
            securityContext:
              # privileged: true
              AllowPrivilegedContainer: true
          # nodeSelector:
          #   node-role.kubernetes.io/worker: ""
        EOF
        
        oc create --save-config -n driver-build -f ~/wzh/build.yaml
        
        # oc delete -n driver-build -f ~/wzh/build.yaml
        
        # oc get all -n driver-build
        
        # NAME                        READY   STATUS    RESTARTS   AGE
        
        # pod/kmod-driver-samplepod   1/1     Running   0          22m
        
        oc rsh -n driver-build pod/kmod-driver-samplepod
        
        bash
        cd ~/nic-rpm-rnp
        tar zvxf rnp-nic-drv-0.1.6.rc44-35c40ea.tgz
        cd rnp-nic-drv-0.1.6.rc44-35c40ea
        cd rnp
        bash do_build.sh
        
        #   MODPOST 1 modules
        
        #   CC      /root/nic-rpm-rnp/rnp-nic-drv-0.1.6.rc44-35c40ea/rnp/rnp.mod.o
        
        #   LD [M]  /root/nic-rpm-rnp/rnp-nic-drv-0.1.6.rc44-35c40ea/rnp/rnp.ko
        
        # make[1]: Leaving directory '/usr/src/kernels/4.18.0-372.52.1.el8_6.x86_64'
        
        exit
        
        # copy the rpm out to helper node
        
        mkdir -p ~/wzh/rsync
        oc project driver-build
        oc rsync kmod-driver-samplepod:/root/rpmbuild/RPMS/x86_64/ ~/wzh/rsync/
        
        scp ~/wzh/rsync/rnp-nic-drv-0.1.6.rc44_35c40ea-1.el8.x86_64.rpm core@172.29.17.61:~/

安装 rpm

我们有了驱动rpm，那么我们就直接在node上安装，看看效果吧。

ssh core@172.29.17.61
        sudo -i
        rpm-ostree install /home/core/rnp-nic-drv-0.1.6.rc44_35c40ea-1.el8.x86_64.rpm
        
        # wait 1 mins at least, then
        
        systemctl reboot
        
        rpm-ostree status
        
        # State: idle
        
        # Deployments:
        
        # ● pivot://quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9b2f4d103a9116e5fb0e5237dd7c932360dda0ef77d3d435374692eaa26dad7c
        
        #               CustomOrigin: Managed by machine-config-operator
        
        #                    Version: 411.86.202304190130-0 (2023-04-19T01:34:04Z)
        
        #              LocalPackages: rnp-nic-drv-0.1.6.rc44_35c40ea-1.el8.x86_64
        
        ################
        
        # nic driver update
        
        oc project driver-build
        
        oc cp ./rnp-0.2.0-wzh.tar.gz driver-build/kmod-driver-samplepod:/root/rnp-0.2.0-wzh.tar.gz
        
        oc rsh -n driver-build pod/kmod-driver-samplepod
        
        bash
        cd /root
        rpmbuild -tb rnp-0.2.0-wzh.tar.gz
        
        oc cp driver-build/kmod-driver-samplepod:/root/rpmbuild/RPMS/x86_64/rnp-0.2.0-1.x86_64.rpm ./rnp-0.2.0-1.x86_64.rpm
        
        scp rnp-0.2.0-1.x86_64.rpm core@172.29.17.61:~/
        
        ssh core@172.29.17.61
        sudo -i
        
        rpm-ostree install /home/core/rnp-0.2.0-1.x86_64.rpm

end


        #### 使用ethtool命令更新固件

        >新固件须重启设备后生效

        1.1拷贝固件到Linux系统的/lib/firmware路径下

        cp xxx.img.bin /lib/firmware

        1.2执行烧录命令，<ethx>需要修改为实际网口名

        ethtool -f <ethx> xxx.img.bin 0

        @注意:指定网卡上任何一个网口，执行一次更新固件动作即可

ocp crash

rpm-ostree kargs --append='crashkernel=256M slub_debug=FZPU'
        
        rpm-ostree kargs --delete='crashkernel=256M'
        
        rpm-ostree kargs --delete='slub_debug=FZPU'
        
        rpm-ostree kargs --append='slub_debug=F'