4.6 离线安装，介质准备

本文的安装步骤，最好是在美国的VPS上完成，然后打包传输回来。

准备离线安装源的步骤如下

准备好operator hub catalog，主要是需要里面的日期信息
运行脚本，准备离线安装源

环境准备


        # on vultr
        
        yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
        
        yum -y install htop byobu ethtool dstat
        
        rm -rf /data/ocp4
        mkdir -p /data/ocp4
        cd /data/ocp4
        
        yum -y install podman docker-distribution pigz skopeo docker buildah jq python3-pip git python36
        
        pip3 install yq
        
        # https://blog.csdn.net/ffzhihua/article/details/85237411
        
        # wget http://mirror.centos.org/centos/7/os/x86_64/Packages/python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm
        
        # rpm2cpio python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm | cpio -iv --to-stdout ./etc/rhsm/ca/redhat-uep.pem | tee /etc/rhsm/ca/redhat-uep.pem
        
        systemctl enable --now docker
        
        # systemctl start docker
        
        docker login -u ****** -p ******** registry.redhat.io
        docker login -u ****** -p ******** registry.access.redhat.com
        docker login -u ****** -p ******** registry.connect.redhat.com
        
        podman login -u ****** -p ******** registry.redhat.io
        podman login -u ****** -p ******** registry.access.redhat.com
        podman login -u ****** -p ******** registry.connect.redhat.com
        
        # to download the pull-secret.json, open following link
        
        # https://cloud.redhat.com/openshift/install/metal/user-provisioned
        
        cat << 'EOF' > /data/pull-secret.json
        {"auths":{"cloud.openshift.com":*********************
        EOF
        
        cat << EOF >>  /etc/hosts
        127.0.0.1 registry.redhat.ren
        EOF
        
        # 配置registry
        
        mkdir -p /etc/crts/ && cd /etc/crts
        
        # https://access.redhat.com/documentation/en-us/red_hat_codeready_workspaces/2.1/html/installation_guide/installing-codeready-workspaces-in-tls-mode-with-self-signed-certificates_crw
        
        openssl genrsa -out /etc/crts/redhat.ren.ca.key 4096
        openssl req -x509 \
          -new -nodes \
          -key /etc/crts/redhat.ren.ca.key \
          -sha256 \
          -days 36500 \
          -out /etc/crts/redhat.ren.ca.crt \
          -subj /CN="Local Red Hat Ren Signer" \
          -reqexts SAN \
          -extensions SAN \
          -config <(cat /etc/pki/tls/openssl.cnf \
              <(printf '[SAN]\nbasicConstraints=critical, CA:TRUE\nkeyUsage=keyCertSign, cRLSign, digitalSignature'))
        
        openssl genrsa -out /etc/crts/redhat.ren.key 2048
        
        openssl req -new -sha256 \
            -key /etc/crts/redhat.ren.key \
            -subj "/O=Local Red Hat Ren /CN=*.ocp4.redhat.ren" \
            -reqexts SAN \
            -config <(cat /etc/pki/tls/openssl.cnf \
                <(printf "\n[SAN]\nsubjectAltName=DNS:*.ocp4.redhat.ren,DNS:*.apps.ocp4.redhat.ren,DNS:*.redhat.ren\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth")) \
            -out /etc/crts/redhat.ren.csr
        
        openssl x509 \
            -req \
            -sha256 \
            -extfile <(printf "subjectAltName=DNS:*.ocp4.redhat.ren,DNS:*.apps.ocp4.redhat.ren,DNS:*.redhat.ren\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth") \
            -days 36500 \
            -in /etc/crts/redhat.ren.csr \
            -CA /etc/crts/redhat.ren.ca.crt \
            -CAkey /etc/crts/redhat.ren.ca.key \
            -CAcreateserial -out /etc/crts/redhat.ren.crt
        
        openssl x509 -in /etc/crts/redhat.ren.crt -text
        
        /bin/cp -f /etc/crts/redhat.ren.ca.crt /etc/pki/ca-trust/source/anchors/
        update-ca-trust extract
        
        cd /data/ocp4
        
        # systemctl stop docker-distribution
        
        /bin/rm -rf /data/registry
        mkdir -p /data/registry
        cat << EOF > /etc/docker-distribution/registry/config.yml
        version: 0.1
        log:
          fields:
            service: registry
        storage:
            cache:
                layerinfo: inmemory
            filesystem:
                rootdirectory: /data/registry
            delete:
                enabled: true
        http:
            addr: :5443
            tls:
               certificate: /etc/crts/redhat.ren.crt
               key: /etc/crts/redhat.ren.key
        compatibility:
          schema1:
            enabled: true
        EOF
        
        # systemctl restart docker
        
        # systemctl enable docker-distribution
        
        # systemctl restart docker-distribution
        
        # podman login registry.redhat.ren:5443 -u a -p a
        
        systemctl enable --now docker-distribution

operator hub catalog

mkdir -p /data/ocp4
        cd /data/ocp4
        
        export BUILDNUMBER=4.6.28
        
        wget -O openshift-client-linux-${BUILDNUMBER}.tar.gz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/${BUILDNUMBER}/openshift-client-linux-${BUILDNUMBER}.tar.gz
        wget -O openshift-install-linux-${BUILDNUMBER}.tar.gz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/${BUILDNUMBER}/openshift-install-linux-${BUILDNUMBER}.tar.gz
        
        tar -xzf openshift-client-linux-${BUILDNUMBER}.tar.gz -C /usr/local/sbin/
        tar -xzf openshift-install-linux-${BUILDNUMBER}.tar.gz -C /usr/local/sbin/
        
        wget -O operator.sh https://raw.githubusercontent.com/wangzheng422/docker_env/dev/redhat/ocp4/4.6/scripts/operator.sh
        
        bash operator.sh
        
        # 2021.05.07.0344

离线安装源制作

rm -rf /data/ocp4
        mkdir -p /data/ocp4
        cd /data/ocp4
        
        # wget -O build.dist.sh https://raw.githubusercontent.com/wangzheng422/docker_env/dev/redhat/ocp4/4.6/scripts/build.dist.sh
        
        # bash build.dist.sh
        
        wget -O prepare.offline.content.sh https://raw.githubusercontent.com/wangzheng422/docker_env/dev/redhat/ocp4/4.6/scripts/prepare.offline.content.sh
        
        # git clone https://github.com/wangzheng422/docker_env.git
        
        # cd docker_env
        
        # git checkout dev
        
        # cp redhat/ocp4/4.6/scripts/prepare.offline.content.sh /data/ocp4/
        
        # cd /data/ocp4
        
        # rm -rf docker_env
        
        bash prepare.offline.content.sh -v 4.6.28, -m 4.6 -h 2021.05.07.0344

output of mirror of images

Success
        Update image:  registry.redhat.ren:5443/ocp4/openshift4:4.6.5
        Mirror prefix: registry.redhat.ren:5443/ocp4/openshift4

        To use the new mirrored repository to install, add the following section to the install-config.yaml:

        imageContentSources:

        - mirrors:
          - registry.redhat.ren:5443/ocp4/openshift4
          source: quay.io/openshift-release-dev/ocp-release

        - mirrors:
          - registry.redhat.ren:5443/ocp4/openshift4
          source: quay.io/openshift-release-dev/ocp-v4.0-art-dev


        To use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:

        apiVersion: operator.openshift.io/v1alpha1
        kind: ImageContentSourcePolicy
        metadata:
          name: example
        spec:
          repositoryDigestMirrors:
          - mirrors:
            - registry.redhat.ren:5443/ocp4/openshift4
            source: quay.io/openshift-release-dev/ocp-release
          - mirrors:
            - registry.redhat.ren:5443/ocp4/openshift4
            source: quay.io/openshift-release-dev/ocp-v4.0-art-dev

        ########################################
        ##
        Success
        Update image:  openshift/release:4.3.3

        To upload local images to a registry, run:

            oc image mirror --from-dir=/data/mirror_dir file://openshift/release:4.3.3* REGISTRY/REPOSITORY

download image for components

########################################
        
        # your images
        
        cd /data/ocp4/
        export MIRROR_DIR='/data/install.image'
        /bin/rm -rf ${MIRROR_DIR}
        bash add.image.sh install.image.list ${MIRROR_DIR}
        
        export MIRROR_DIR='/data/poc.image'
        /bin/rm -rf ${MIRROR_DIR}
        bash add.image.sh poc.image.list ${MIRROR_DIR}
        
        ########################################
        
        # common function
        
        build_image_list() {
          VAR_INPUT_FILE=$1
          VAR_OUTPUT_FILE=$2
          VAR_OPERATOR=$3
        
          VAR_FINAL=`cat $VAR_INPUT_FILE | grep $VAR_OPERATOR | awk '{if ($2) print $2;}' | sort | uniq | tail -1`
        
          echo $VAR_FINAL
        
          cat $VAR_INPUT_FILE | grep $VAR_FINAL | awk '{if ($2) print $1;}' >> $VAR_OUTPUT_FILE
        }
        
        ########################################
        
        # redhat operator hub
        
        export MIRROR_DIR='/data/redhat-operator'
        
        /bin/rm -rf ${MIRROR_DIR}
        /bin/rm -f /data/ocp4/mapping-redhat.list
        wanted_operator_list=$(cat redhat-operator-image.list | awk '{if ($2) print $2;}' \
          | sed 's/\..*//g' | sort | uniq
        )
        
        while read -r line; do
            build_image_list '/data/ocp4/redhat-operator-image.list' '/data/ocp4/mapping-redhat.list' $line
        done <<< "$wanted_operator_list"
        
        bash add.image.sh mapping-redhat.list ${MIRROR_DIR}
        
        # /bin/cp -f pull.add.image.failed.list pull.add.image.failed.list.bak
        
        # bash add.image.resume.sh pull.add.image.failed.list.bak ${MIRROR_DIR}
        
        cd ${MIRROR_DIR%/*}
        tar cf - echo ${MIRROR_DIR##*/}/ | pigz -c > echo ${MIRROR_DIR##*/}.tgz 
        
        # to load image back
        
        bash add.image.load.sh '/data/redhat-operator' 'registry.redhat.ren:5443'
        
        ######################################
        
        # certified operator hub
        
        export MIRROR_DIR='/data/certified-operator'
        
        /bin/rm -rf ${MIRROR_DIR}
        /bin/rm -f /data/ocp4/mapping-certified.list
        wanted_operator_list=$(cat certified-operator-image.list | awk '{if ($2) print $2;}' \
          | sed 's/\..*//g' | sort | uniq
        )
        
        while read -r line; do
            build_image_list '/data/ocp4/certified-operator-image.list' '/data/ocp4/mapping-certified.list' $line
        done <<< "$wanted_operator_list"
        
        bash add.image.sh mapping-certified.list ${MIRROR_DIR}
        
        # /bin/cp -f pull.add.image.failed.list pull.add.image.failed.list.bak
        
        # bash add.image.resume.sh pull.add.image.failed.list.bak ${MIRROR_DIR}
        
        cd ${MIRROR_DIR%/*}
        tar cf - echo ${MIRROR_DIR##*/}/ | pigz -c > echo ${MIRROR_DIR##*/}.tgz 
        
        # bash add.image.sh mapping-certified.txt
        
        #######################################
        
        # community operator hub
        
        export MIRROR_DIR='/data/community-operator'
        
        /bin/rm -rf ${MIRROR_DIR}
        /bin/rm -f /data/ocp4/mapping-community.list
        wanted_operator_list=$(cat community-operator-image.list | awk '{if ($2) print $2;}' \
          | sed 's/\..*//g' | sort | uniq
        )
        
        while read -r line; do
            build_image_list '/data/ocp4/community-operator-image.list' '/data/ocp4/mapping-community.list' $line
        done <<< "$wanted_operator_list"
        
        bash add.image.sh mapping-community.list ${MIRROR_DIR}
        
        # /bin/cp -f pull.add.image.failed.list pull.add.image.failed.list.bak
        
        # bash add.image.resume.sh pull.add.image.failed.list.bak ${MIRROR_DIR}
        
        cd ${MIRROR_DIR%/*}
        tar cf - echo ${MIRROR_DIR##*/}/ | pigz -c > echo ${MIRROR_DIR##*/}.tgz 
        
        # bash add.image.sh mapping-community.txt
        
        # to load image back
        
        bash add.image.load.sh '/data/community-operator' 'registry.redhat.ren:5443'
        
        #####################################
        
        # samples operator
        
        export MIRROR_DIR='/data/is.samples'
        
        /bin/rm -rf ${MIRROR_DIR}
        bash add.image.sh is.openshift.list  ${MIRROR_DIR}

镜像仓库代理 / image registry proxy

准备离线镜像仓库非常麻烦，好在我们找到了一台在线的主机，那么我们可以使用nexus构造image registry proxy，在在线环境上面，做一遍PoC，然后就能通过image registry proxy得到离线镜像了

https://mtijhof.wordpress.com/2018/07/23/using-nexus-oss-as-a-proxy-cache-for-docker-images/

#####################################################
        
        # init build the nexus fs
        
        mkdir -p /data/ccn/nexus-image
        chown -R 200 /data/ccn/nexus-image
        
        # podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image -v /data/ccn/nexus-image:/nexus-data:Z docker.io/sonatype/nexus3:3.29.0
        
        podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image -v /data/ccn/nexus-image:/nexus-data:Z docker.io/wangzheng422/imgs:nexus3-3.29.0-wzh
        
        podman stop nexus-image
        podman rm nexus-image
        
        # get the admin password
        
        cat /data/ccn/nexus-image/admin.password && echo
        
        # 84091bcd-c82f-44a3-8b7b-dfc90f5b7da1
        
        # open http://nexus.ocp4.redhat.ren:8082
        
        # 开启 https
        
        # https://blog.csdn.net/s7799653/article/details/105378645
        
        # https://help.sonatype.com/repomanager3/system-configuration/configuring-ssl#ConfiguringSSL-InboundSSL-ConfiguringtoServeContentviaHTTPS
        
        mkdir -p /data/install/tmp
        cd /data/install/tmp
        
        # 将证书导出成pkcs格式
        
        # 这里需要输入密码  用 password，
        
        openssl pkcs12 -export -out keystore.pkcs12 -inkey /etc/crts/redhat.ren.key -in /etc/crts/redhat.ren.crt
        
        cat << EOF >> Dockerfile
        FROM docker.io/sonatype/nexus3:3.29.0
        USER root
        COPY keystore.pkcs12 /keystore.pkcs12
        RUN keytool -v -importkeystore -srckeystore keystore.pkcs12 -srcstoretype PKCS12 -destkeystore keystore.jks -deststoretype JKS -storepass password -srcstorepass password  &&\
            cp keystore.jks /opt/sonatype/nexus/etc/ssl/
        USER nexus
        EOF
        buildah bud --format=docker -t docker.io/wangzheng422/imgs:nexus3-3.29.0-wzh -f Dockerfile .
        buildah push docker.io/wangzheng422/imgs:nexus3-3.29.0-wzh
        
        ######################################################
        
        # go to helper, update proxy setting for ocp cluster
        
        cd /data/ocp4
        bash image.registries.conf.sh nexus.ocp4.redhat.ren:8083
        
        mkdir -p /etc/containers/registries.conf.d
        /bin/cp -f image.registries.conf /etc/containers/registries.conf.d/
        
        cd /data/ocp4
        oc apply -f ./99-worker-container-registries.yaml -n openshift-config
        oc apply -f ./99-master-container-registries.yaml -n openshift-config
        
        ######################################################
        
        # dump the nexus image fs out
        
        podman stop nexus-image
        
        var_date=$(date '+%Y-%m-%d-%H%M')
        echo $var_date
        cd /data/ccn
        
        tar cf - ./nexus-image | pigz -c > nexus-image.tgz 
        buildah from --name onbuild-container scratch
        buildah copy onbuild-container nexus-image.tgz  /
        buildah umount onbuild-container 
        buildah commit --rm --format=docker onbuild-container docker.io/wangzheng422/nexus-fs:image-$var_date
        
        # buildah rm onbuild-container
        
        # rm -f nexus-image.tgz 
        
        buildah push docker.io/wangzheng422/nexus-fs:image-$var_date
        echo "docker.io/wangzheng422/nexus-fs:image-$var_date"
        
        # 以下这个版本，可以作为初始化的image proxy，里面包含了nfs provision，以及sample operator的metadata。很高兴的发现，image stream并不会完全下载镜像，好想只是下载metadata，真正用的时候，才去下载。
        
        # docker.io/wangzheng422/nexus-fs:image-2020-12-26-1118
        
        ##################################################
        
        ## call nexus api to get image list
        
        # https://community.sonatype.com/t/how-can-i-get-a-list-of-tags-for-a-docker-image-akin-to-the-docker-hub-list/3210
        
        # https://help.sonatype.com/repomanager3/rest-and-integration-api/search-api
        
        curl -k -u admin:84091bcd-c82f-44a3-8b7b-dfc90f5b7da1 -X GET 'http://nexus.ocp4.redhat.ren:8082/service/rest/v1/search?repository=registry.redhat.io'
        
        curl -u admin:84091bcd-c82f-44a3-8b7b-dfc90f5b7da1 -X GET 'http://nexus.ocp4.redhat.ren:8082/service/rest/v1/components?repository=registry.redhat.io'
        
        podman pull docker.io/anoxis/registry-cli
        podman run --rm anoxis/registry-cli -l admin:84091bcd-c82f-44a3-8b7b-dfc90f5b7da1 -r https://nexus.ocp4.redhat.ren:8083
        
        # https://github.com/rpardini/docker-registry-proxy
        
        REPO_URL=nexus.ocp4.redhat.ren:8083
        
        curl -k -s -X GET https://$REPO_URL/v2/_catalog \
         | jq '.repositories[]' \
         | sort \
         | xargs -I _ curl -s -k -X GET https://$REPO_URL/v2/_/tags/list
        
        
        
        ##################################################
        
        ## prepare for baidu disk
        
        mkdir -p /data/ccn/baidu
        cd /data/ccn
        
        tar cf - ./nexus-image | pigz -c > /data/ccn/baidu/nexus-image.tgz 
        
        cd /data/ccn/baidu
        split -b 20000m nexus-image.tgz  nexus-image.tgz.
        rm -f nexus-image.tgz
        
        yum -y install python3-pip
        pip3 install --user bypy 
        /root/.local/bin/bypy list
        /root/.local/bin/bypy upload

upload to baidu disk

export BUILDNUMBER=4.6.28
        
        mkdir -p /data/bypy
        cd /data
        tar -cvf - ocp4/ | pigz -c > /data/bypy/ocp.$BUILDNUMBER.tgz
        tar -cvf - registry/ | pigz -c > /data/bypy/registry.$BUILDNUMBER.tgz
        
        cd /data/bypy
        
        # https://github.com/houtianze/bypy
        
        yum -y install python3-pip
        pip3 install --user bypy 
        /root/.local/bin/bypy list
        /root/.local/bin/bypy upload

4.6 离线安装， 介质准备

环境准备

operator hub catalog

离线安装源制作

download image for components

镜像仓库代理 / image registry proxy

upload to baidu disk

4.6 离线安装，介质准备