← Back to Index

ocp 4.3 recover from node not ready

https://access.redhat.com/solutions/4923031

cat << "EOF" > recover_kubeconfig.sh
        #!/bin/bash
        
        set -eou pipefail
        
        # context
        
        intapi=$(oc get infrastructures.config.openshift.io cluster -o "jsonpath={.status.apiServerInternalURI}")
        context="$(oc config current-context)"
        
        # cluster
        
        cluster="$(oc config view -o "jsonpath={.contexts[?(@.name==\"$context\")].context.cluster}")"
        server="$(oc config view -o "jsonpath={.clusters[?(@.name==\"$cluster\")].cluster.server}")"
        
        # token
        
        ca_crt_data="$(oc get secret -n openshift-machine-config-operator node-bootstrapper-token -o "jsonpath={.data.ca\.crt}" | base64 --decode)"
        namespace="$(oc get secret -n openshift-machine-config-operator node-bootstrapper-token  -o "jsonpath={.data.namespace}" | base64 --decode)"
        token="$(oc get secret -n openshift-machine-config-operator node-bootstrapper-token -o "jsonpath={.data.token}" | base64 --decode)"
        
        export KUBECONFIG="$(mktemp)"
        oc config set-credentials "kubelet" --token="$token" >/dev/null
        ca_crt="$(mktemp)"; echo "$ca_crt_data" > $ca_crt
        oc config set-cluster $cluster --server="$intapi" --certificate-authority="$ca_crt" --embed-certs >/dev/null
        oc config set-context kubelet --cluster="$cluster" --user="kubelet" >/dev/null
        oc config use-context kubelet >/dev/null
        cat "$KUBECONFIG"
        EOF
        
        chmod 755 recover_kubeconfig.sh
        ./recover_kubeconfig.sh > kubeconfig-bootstrap
        
        # scp kubeconfig-bootstrap to each affected nodes
        
        scp kubeconfig-bootstrap core@node.ip.address:~/
        
        # on each affected nodes
        
        systemctl stop kubelet
        mkdir -p /root/backup-certs
        cp -a /var/lib/kubelet/pki /var/lib/kubelet/kubeconfig /root/backup-certs
        rm -rf /var/lib/kubelet/pki /var/lib/kubelet/kubeconfig
        cp /home/core/kubeconfig-bootstrap /etc/kubernetes/kubeconfig
        systemctl start kubelet
        
        # on helper
        
        oc get node
        oc get csr
        oc get csr -ojson | jq -r '.items[] | select(.status == {} ) | .metadata.name' | xargs oc adm certificate approve