Solution: Rebootless Kernel Live Patching (kpatch) on OpenShift 4.16 CoreOS

# 1. Check RHCOS kernel version
oc debug node/<node> -- chroot /host uname -r
# Output: 5.14.0-427.93.1.el9_4.x86_64

# 2. On a RHEL 9.4 helper, enable EUS repos and download kpatch
subscription-manager repos --enable=rhel-9-for-x86_64-baseos-eus-rpms
subscription-manager release --set=9.4
dnf download kpatch-patch-5_14_0-427_100_1 --destdir=/var/tmp/kpatch/

# 3. Extract the .ko
cd /var/tmp/kpatch/
rpm2cpio kpatch-patch-*.rpm | cpio -idmv
# Output: ./usr/lib/kpatch/5.14.0-427.100.1.el9_4.x86_64/kpatch-*.ko

# 4. Serve via HTTP
cd usr/lib/kpatch/5.14.0-427.100.1.el9_4.x86_64/
python3 -m http.server 18080 &
firewall-cmd --add-port=18080/tcp

# 5. Download to RHCOS node and load
oc debug node/<node> -- nsenter -t 1 -m -u -i -n -p -- bash -c \
    "curl -sS -o /var/tmp/kpatch.ko http://<helper-ip>:18080/kpatch-*.ko && \
     chcon -t modules_object_t /var/tmp/kpatch.ko && \
     insmod /var/tmp/kpatch.ko"

# 6. Verify
oc debug node/<node> -- chroot /host lsmod | grep kpatch
oc debug node/<node> -- chroot /host dmesg | grep livepatch

# On a build host with podman
mkdir -p kpatch-image/usr/lib/kpatch/5.14.0-427.100.1.el9_4.x86_64/
cp kpatch-5_14_0-427_100_1-1-4.ko kpatch-image/usr/lib/kpatch/5.14.0-427.100.1.el9_4.x86_64/

cat > kpatch-image/Containerfile << 'EOF'
FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
COPY usr/lib/kpatch/ /usr/lib/kpatch/
EOF

cd kpatch-image
podman build -t quay.io/wangzheng422/qimgs:kpatch-ocp416-2026-05-21 .
podman push quay.io/wangzheng422/qimgs:kpatch-ocp416-2026-05-21

apiVersion: v1
kind: Namespace
metadata:
  name: kpatch-loader
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kpatch-loader
  namespace: kpatch-loader
spec:
  selector:
    matchLabels:
      app: kpatch-loader
  template:
    metadata:
      labels:
        app: kpatch-loader
    spec:
      hostPID: true
      hostNetwork: true
      tolerations:
        - operator: Exists
      initContainers:
        - name: load-kpatch
          image: quay.io/wangzheng422/qimgs:kpatch-ocp416-2026-05-21
          securityContext:
            privileged: true
            runAsUser: 0
          command:
            - /bin/sh
            - -c
            - |
              set -ex
              if chroot /host lsmod | grep -q kpatch; then
                echo "kpatch already loaded, skipping"
                exit 0
              fi
              KO_FILE=$(find /usr/lib/kpatch/ -name '*.ko' | head -1)
              if [ -z "${KO_FILE}" ]; then
                echo "ERROR: No kpatch .ko found"
                exit 1
              fi
              echo "Found: ${KO_FILE}"
              cp "${KO_FILE}" /host/var/tmp/kpatch.ko
              chroot /host chcon -t modules_object_t /var/tmp/kpatch.ko
              chroot /host insmod /var/tmp/kpatch.ko
              echo "insmod OK"
              chroot /host dmesg | tail -5 | grep -i livepatch || true
              echo "kpatch loaded successfully"
          volumeMounts:
            - name: host-root
              mountPath: /host
      containers:
        - name: pause
          image: quay.io/wangzheng422/qimgs:kpatch-ocp416-2026-05-21
          command: ["/bin/sh", "-c", "echo kpatch active && sleep infinity"]
      volumes:
        - name: host-root
          hostPath:
            path: /
            type: Directory

# 1. Apply
oc apply -f kpatch-daemonset.yaml

# 2. Grant privileged SCC (required)
oc adm policy add-scc-to-user privileged -z default -n kpatch-loader

# 3. Verify
oc get pods -n kpatch-loader -o wide
oc logs -n kpatch-loader <pod-name> -c load-kpatch

# 1. Build new image with updated .ko
podman build -t quay.io/<org>/<repo>:kpatch-v2 .
podman push quay.io/<org>/<repo>:kpatch-v2

# 2. Update the DaemonSet image — rolling update handles the rest:
#    preStop unloads old kpatch → initContainer loads new kpatch
oc set image daemonset/kpatch-loader \
    load-kpatch=quay.io/<org>/<repo>:kpatch-v2 \
    keeper=quay.io/<org>/<repo>:kpatch-v2 \
    -n kpatch-loader

apiVersion: v1
kind: Namespace
metadata:
  name: kpatch-loader
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kpatch-loader
  namespace: kpatch-loader
spec:
  selector:
    matchLabels:
      app: kpatch-loader
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        app: kpatch-loader
    spec:
      hostPID: true
      hostNetwork: true
      terminationGracePeriodSeconds: 60
      tolerations:
        - operator: Exists
      initContainers:
        - name: load-kpatch
          image: quay.io/wangzheng422/qimgs:kpatch-ocp416-2026-05-21
          securityContext:
            privileged: true
            runAsUser: 0
          command:
            - /bin/sh
            - -c
            - |
              set -ex

              # Unload any existing kpatch first (upgrade scenario)
              EXISTING=$(chroot /host lsmod | grep kpatch | awk '{print $1}' || true)
              if [ -n "${EXISTING}" ]; then
                echo "Unloading existing kpatch: ${EXISTING}"
                chroot /host bash -c "echo 0 > /sys/kernel/livepatch/${EXISTING}/enabled" 2>/dev/null || true
                for i in 1 2 3 4 5 6 7 8 9 10; do
                  T=$(chroot /host cat /sys/kernel/livepatch/${EXISTING}/transition 2>/dev/null || echo 0)
                  [ "$T" = "0" ] && break
                  echo "Waiting for transition... (round $i)"
                  sleep 3
                done
                chroot /host rmmod ${EXISTING} 2>/dev/null || true
                echo "Old kpatch unloaded"
              fi

              # Find and load new kpatch
              KO_FILE=$(find /usr/lib/kpatch/ -name '*.ko' | head -1)
              if [ -z "${KO_FILE}" ]; then
                echo "ERROR: No kpatch .ko found"
                exit 1
              fi
              echo "Found: ${KO_FILE}"
              cp "${KO_FILE}" /host/var/tmp/kpatch.ko
              chroot /host chcon -t modules_object_t /var/tmp/kpatch.ko
              chroot /host insmod /var/tmp/kpatch.ko
              echo "insmod OK"
              chroot /host dmesg | tail -5 | grep -i livepatch || true

              # Record loaded module name for preStop
              KO_NAME=$(chroot /host lsmod | grep kpatch | awk '{print $1}')
              echo "${KO_NAME}" > /host/var/tmp/kpatch-loaded-module
              echo "kpatch loaded: ${KO_NAME}"
          volumeMounts:
            - name: host-root
              mountPath: /host
      containers:
        - name: keeper
          image: quay.io/wangzheng422/qimgs:kpatch-ocp416-2026-05-21
          securityContext:
            privileged: true
            runAsUser: 0
          command:
            - /bin/sh
            - -c
            - |
              echo "kpatch active"
              trap 'echo SIGTERM received' TERM
              while true; do sleep 3600 & wait; done
          lifecycle:
            preStop:
              exec:
                command:
                  - /bin/sh
                  - -c
                  - |
                    echo "preStop: unloading kpatch..."
                    MOD=$(cat /host/var/tmp/kpatch-loaded-module 2>/dev/null || chroot /host lsmod | grep kpatch | awk '{print $1}')
                    if [ -z "${MOD}" ]; then
                      echo "No kpatch module found"
                      exit 0
                    fi
                    echo "Disabling livepatch: ${MOD}"
                    chroot /host bash -c "echo 0 > /sys/kernel/livepatch/${MOD}/enabled" 2>/dev/null || true
                    for i in 1 2 3 4 5 6 7 8 9 10; do
                      T=$(chroot /host cat /sys/kernel/livepatch/${MOD}/transition 2>/dev/null || echo 0)
                      [ "$T" = "0" ] && break
                      echo "Waiting for transition... (round $i)"
                      sleep 3
                    done
                    chroot /host rmmod ${MOD} 2>/dev/null
                    echo "kpatch ${MOD} unloaded"
                    rm -f /host/var/tmp/kpatch-loaded-module /host/var/tmp/kpatch.ko
          volumeMounts:
            - name: host-root
              mountPath: /host
      volumes:
        - name: host-root
          hostPath:
            path: /
            type: Directory

oc apply -f kpatch-daemonset-v2.yaml
oc adm policy add-scc-to-user privileged -z default -n kpatch-loader

# preStop hook automatically unloads kpatch from every node
oc delete daemonset kpatch-loader -n kpatch-loader
# or
oc delete project kpatch-loader