4.6 离线安装， 介质准备

本文的安装步骤，最好是在美国的VPS上完成，然后打包传输回来。

准备离线安装源的步骤如下

• 准备好operator hub catalog，主要是需要里面的日期信息
• 运行脚本，准备离线安装源

环境准备

# on vultr
yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm

yum -y install htop byobu ethtool dstat

rm -rf /data/ocp4
mkdir -p /data/ocp4
cd /data/ocp4

yum -y install podman docker-distribution pigz skopeo docker buildah jq python3-pip git python36

pip3 install yq

# https://blog.csdn.net/ffzhihua/article/details/85237411
# wget http://mirror.centos.org/centos/7/os/x86_64/Packages/python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm
# rpm2cpio python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm | cpio -iv --to-stdout ./etc/rhsm/ca/redhat-uep.pem | tee /etc/rhsm/ca/redhat-uep.pem

systemctl enable --now docker

# systemctl start docker

docker login -u ****** -p ******** registry.redhat.io
docker login -u ****** -p ******** registry.access.redhat.com
docker login -u ****** -p ******** registry.connect.redhat.com

podman login -u ****** -p ******** registry.redhat.io
podman login -u ****** -p ******** registry.access.redhat.com
podman login -u ****** -p ******** registry.connect.redhat.com

# to download the pull-secret.json, open following link
# https://cloud.redhat.com/openshift/install/metal/user-provisioned
cat << 'EOF' > /data/pull-secret.json
{"auths":{"cloud.openshift.com":*********************
EOF

cat << EOF >>  /etc/hosts
127.0.0.1 registry.redhat.ren
EOF

# 配置registry
mkdir -p /etc/crts/ && cd /etc/crts

# https://access.redhat.com/documentation/en-us/red_hat_codeready_workspaces/2.1/html/installation_guide/installing-codeready-workspaces-in-tls-mode-with-self-signed-certificates_crw
openssl genrsa -out /etc/crts/redhat.ren.ca.key 4096
openssl req -x509 \
  -new -nodes \
  -key /etc/crts/redhat.ren.ca.key \
  -sha256 \
  -days 36500 \
  -out /etc/crts/redhat.ren.ca.crt \
  -subj /CN="Local Red Hat Ren Signer" \
  -reqexts SAN \
  -extensions SAN \
  -config <(cat /etc/pki/tls/openssl.cnf \
      <(printf '[SAN]\nbasicConstraints=critical, CA:TRUE\nkeyUsage=keyCertSign, cRLSign, digitalSignature'))

openssl genrsa -out /etc/crts/redhat.ren.key 2048

openssl req -new -sha256 \
    -key /etc/crts/redhat.ren.key \
    -subj "/O=Local Red Hat Ren /CN=*.ocp4.redhat.ren" \
    -reqexts SAN \
    -config <(cat /etc/pki/tls/openssl.cnf \
        <(printf "\n[SAN]\nsubjectAltName=DNS:*.ocp4.redhat.ren,DNS:*.apps.ocp4.redhat.ren,DNS:*.redhat.ren\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth")) \
    -out /etc/crts/redhat.ren.csr

openssl x509 \
    -req \
    -sha256 \
    -extfile <(printf "subjectAltName=DNS:*.ocp4.redhat.ren,DNS:*.apps.ocp4.redhat.ren,DNS:*.redhat.ren\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth") \
    -days 36500 \
    -in /etc/crts/redhat.ren.csr \
    -CA /etc/crts/redhat.ren.ca.crt \
    -CAkey /etc/crts/redhat.ren.ca.key \
    -CAcreateserial -out /etc/crts/redhat.ren.crt

openssl x509 -in /etc/crts/redhat.ren.crt -text

/bin/cp -f /etc/crts/redhat.ren.ca.crt /etc/pki/ca-trust/source/anchors/
update-ca-trust extract

cd /data/ocp4
# systemctl stop docker-distribution

/bin/rm -rf /data/registry
mkdir -p /data/registry
cat << EOF > /etc/docker-distribution/registry/config.yml
version: 0.1
log:
  fields:
    service: registry
storage:
    cache:
        layerinfo: inmemory
    filesystem:
        rootdirectory: /data/registry
    delete:
        enabled: true
http:
    addr: :5443
    tls:
       certificate: /etc/crts/redhat.ren.crt
       key: /etc/crts/redhat.ren.key
compatibility:
  schema1:
    enabled: true
EOF
# systemctl restart docker
# systemctl enable docker-distribution

# systemctl restart docker-distribution

# podman login registry.redhat.ren:5443 -u a -p a

systemctl enable --now docker-distribution

operator hub catalog

mkdir -p /data/ocp4
cd /data/ocp4

export BUILDNUMBER=4.6.28

wget -O openshift-client-linux-${BUILDNUMBER}.tar.gz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/${BUILDNUMBER}/openshift-client-linux-${BUILDNUMBER}.tar.gz
wget -O openshift-install-linux-${BUILDNUMBER}.tar.gz https://mirror.openshift.com/pub/openshift-v4/clients/ocp/${BUILDNUMBER}/openshift-install-linux-${BUILDNUMBER}.tar.gz

tar -xzf openshift-client-linux-${BUILDNUMBER}.tar.gz -C /usr/local/sbin/
tar -xzf openshift-install-linux-${BUILDNUMBER}.tar.gz -C /usr/local/sbin/

wget -O operator.sh https://raw.githubusercontent.com/wangzheng422/docker_env/dev/redhat/ocp4/4.6/scripts/operator.sh

bash operator.sh

# 2021.05.07.0344

离线安装源制作

rm -rf /data/ocp4
mkdir -p /data/ocp4
cd /data/ocp4
# wget -O build.dist.sh https://raw.githubusercontent.com/wangzheng422/docker_env/dev/redhat/ocp4/4.6/scripts/build.dist.sh

# bash build.dist.sh

wget -O prepare.offline.content.sh https://raw.githubusercontent.com/wangzheng422/docker_env/dev/redhat/ocp4/4.6/scripts/prepare.offline.content.sh

# git clone https://github.com/wangzheng422/docker_env.git
# cd docker_env
# git checkout dev
# cp redhat/ocp4/4.6/scripts/prepare.offline.content.sh /data/ocp4/
# cd /data/ocp4
# rm -rf docker_env

bash prepare.offline.content.sh -v 4.6.28, -m 4.6 -h 2021.05.07.0344

output of mirror of images

Success
Update image:  registry.redhat.ren:5443/ocp4/openshift4:4.6.5
Mirror prefix: registry.redhat.ren:5443/ocp4/openshift4

To use the new mirrored repository to install, add the following section to the install-config.yaml:

imageContentSources:
- mirrors:
  - registry.redhat.ren:5443/ocp4/openshift4
  source: quay.io/openshift-release-dev/ocp-release
- mirrors:
  - registry.redhat.ren:5443/ocp4/openshift4
  source: quay.io/openshift-release-dev/ocp-v4.0-art-dev


To use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:

apiVersion: operator.openshift.io/v1alpha1
kind: ImageContentSourcePolicy
metadata:
  name: example
spec:
  repositoryDigestMirrors:
  - mirrors:
    - registry.redhat.ren:5443/ocp4/openshift4
    source: quay.io/openshift-release-dev/ocp-release
  - mirrors:
    - registry.redhat.ren:5443/ocp4/openshift4
    source: quay.io/openshift-release-dev/ocp-v4.0-art-dev

########################################
##
Success
Update image:  openshift/release:4.3.3

To upload local images to a registry, run:

    oc image mirror --from-dir=/data/mirror_dir file://openshift/release:4.3.3* REGISTRY/REPOSITORY

download image for components

########################################
# your images
cd /data/ocp4/
export MIRROR_DIR='/data/install.image'
/bin/rm -rf ${MIRROR_DIR}
bash add.image.sh install.image.list ${MIRROR_DIR}

export MIRROR_DIR='/data/poc.image'
/bin/rm -rf ${MIRROR_DIR}
bash add.image.sh poc.image.list ${MIRROR_DIR}

########################################
# common function
build_image_list() {
  VAR_INPUT_FILE=$1
  VAR_OUTPUT_FILE=$2
  VAR_OPERATOR=$3

  VAR_FINAL=`cat $VAR_INPUT_FILE | grep $VAR_OPERATOR | awk '{if ($2) print $2;}' | sort | uniq | tail -1`

  echo $VAR_FINAL

  cat $VAR_INPUT_FILE | grep $VAR_FINAL | awk '{if ($2) print $1;}' >> $VAR_OUTPUT_FILE
}

########################################
# redhat operator hub
export MIRROR_DIR='/data/redhat-operator'

/bin/rm -rf ${MIRROR_DIR}
/bin/rm -f /data/ocp4/mapping-redhat.list
wanted_operator_list=$(cat redhat-operator-image.list | awk '{if ($2) print $2;}' \
  | sed 's/\..*//g' | sort | uniq
)

while read -r line; do
    build_image_list '/data/ocp4/redhat-operator-image.list' '/data/ocp4/mapping-redhat.list' $line
done <<< "$wanted_operator_list"

bash add.image.sh mapping-redhat.list ${MIRROR_DIR}

# /bin/cp -f pull.add.image.failed.list pull.add.image.failed.list.bak
# bash add.image.resume.sh pull.add.image.failed.list.bak ${MIRROR_DIR}

cd ${MIRROR_DIR%/*}
tar cf - echo ${MIRROR_DIR##*/}/ | pigz -c > echo ${MIRROR_DIR##*/}.tgz 

# to load image back
bash add.image.load.sh '/data/redhat-operator' 'registry.redhat.ren:5443'

######################################
# certified operator hub
export MIRROR_DIR='/data/certified-operator'

/bin/rm -rf ${MIRROR_DIR}
/bin/rm -f /data/ocp4/mapping-certified.list
wanted_operator_list=$(cat certified-operator-image.list | awk '{if ($2) print $2;}' \
  | sed 's/\..*//g' | sort | uniq
)

while read -r line; do
    build_image_list '/data/ocp4/certified-operator-image.list' '/data/ocp4/mapping-certified.list' $line
done <<< "$wanted_operator_list"

bash add.image.sh mapping-certified.list ${MIRROR_DIR}

# /bin/cp -f pull.add.image.failed.list pull.add.image.failed.list.bak
# bash add.image.resume.sh pull.add.image.failed.list.bak ${MIRROR_DIR}

cd ${MIRROR_DIR%/*}
tar cf - echo ${MIRROR_DIR##*/}/ | pigz -c > echo ${MIRROR_DIR##*/}.tgz 

# bash add.image.sh mapping-certified.txt

#######################################
# community operator hub
export MIRROR_DIR='/data/community-operator'

/bin/rm -rf ${MIRROR_DIR}
/bin/rm -f /data/ocp4/mapping-community.list
wanted_operator_list=$(cat community-operator-image.list | awk '{if ($2) print $2;}' \
  | sed 's/\..*//g' | sort | uniq
)

while read -r line; do
    build_image_list '/data/ocp4/community-operator-image.list' '/data/ocp4/mapping-community.list' $line
done <<< "$wanted_operator_list"

bash add.image.sh mapping-community.list ${MIRROR_DIR}

# /bin/cp -f pull.add.image.failed.list pull.add.image.failed.list.bak
# bash add.image.resume.sh pull.add.image.failed.list.bak ${MIRROR_DIR}

cd ${MIRROR_DIR%/*}
tar cf - echo ${MIRROR_DIR##*/}/ | pigz -c > echo ${MIRROR_DIR##*/}.tgz 

# bash add.image.sh mapping-community.txt

# to load image back
bash add.image.load.sh '/data/community-operator' 'registry.redhat.ren:5443'

#####################################
# samples operator
export MIRROR_DIR='/data/is.samples'

/bin/rm -rf ${MIRROR_DIR}
bash add.image.sh is.openshift.list  ${MIRROR_DIR}

镜像仓库代理 / image registry proxy

准备离线镜像仓库非常麻烦，好在我们找到了一台在线的主机，那么我们可以使用nexus构造image registry proxy，在在线环境上面，做一遍PoC，然后就能通过image registry proxy得到离线镜像了

• https://mtijhof.wordpress.com/2018/07/23/using-nexus-oss-as-a-proxy-cache-for-docker-images/

#####################################################
# init build the nexus fs
mkdir -p /data/ccn/nexus-image
chown -R 200 /data/ccn/nexus-image

# podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image -v /data/ccn/nexus-image:/nexus-data:Z docker.io/sonatype/nexus3:3.29.0

podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image -v /data/ccn/nexus-image:/nexus-data:Z docker.io/wangzheng422/imgs:nexus3-3.29.0-wzh

podman stop nexus-image
podman rm nexus-image

# get the admin password
cat /data/ccn/nexus-image/admin.password && echo
# 84091bcd-c82f-44a3-8b7b-dfc90f5b7da1

# open http://nexus.ocp4.redhat.ren:8082

# 开启 https
# https://blog.csdn.net/s7799653/article/details/105378645
# https://help.sonatype.com/repomanager3/system-configuration/configuring-ssl#ConfiguringSSL-InboundSSL-ConfiguringtoServeContentviaHTTPS
mkdir -p /data/install/tmp
cd /data/install/tmp

# 将证书导出成pkcs格式
# 这里需要输入密码  用 password，
openssl pkcs12 -export -out keystore.pkcs12 -inkey /etc/crts/redhat.ren.key -in /etc/crts/redhat.ren.crt

cat << EOF >> Dockerfile
FROM docker.io/sonatype/nexus3:3.29.0
USER root
COPY keystore.pkcs12 /keystore.pkcs12
RUN keytool -v -importkeystore -srckeystore keystore.pkcs12 -srcstoretype PKCS12 -destkeystore keystore.jks -deststoretype JKS -storepass password -srcstorepass password  &&\
    cp keystore.jks /opt/sonatype/nexus/etc/ssl/
USER nexus
EOF
buildah bud --format=docker -t docker.io/wangzheng422/imgs:nexus3-3.29.0-wzh -f Dockerfile .
buildah push docker.io/wangzheng422/imgs:nexus3-3.29.0-wzh

######################################################
# go to helper, update proxy setting for ocp cluster
cd /data/ocp4
bash image.registries.conf.sh nexus.ocp4.redhat.ren:8083

mkdir -p /etc/containers/registries.conf.d
/bin/cp -f image.registries.conf /etc/containers/registries.conf.d/

cd /data/ocp4
oc apply -f ./99-worker-container-registries.yaml -n openshift-config
oc apply -f ./99-master-container-registries.yaml -n openshift-config

######################################################
# dump the nexus image fs out
podman stop nexus-image

var_date=$(date '+%Y-%m-%d-%H%M')
echo $var_date
cd /data/ccn

tar cf - ./nexus-image | pigz -c > nexus-image.tgz 
buildah from --name onbuild-container scratch
buildah copy onbuild-container nexus-image.tgz  /
buildah umount onbuild-container 
buildah commit --rm --format=docker onbuild-container docker.io/wangzheng422/nexus-fs:image-$var_date
# buildah rm onbuild-container
# rm -f nexus-image.tgz 
buildah push docker.io/wangzheng422/nexus-fs:image-$var_date
echo "docker.io/wangzheng422/nexus-fs:image-$var_date"

# 以下这个版本，可以作为初始化的image proxy，里面包含了nfs provision，以及sample operator的metadata。很高兴的发现，image stream并不会完全下载镜像，好想只是下载metadata，真正用的时候，才去下载。
# docker.io/wangzheng422/nexus-fs:image-2020-12-26-1118

##################################################
## call nexus api to get image list
# https://community.sonatype.com/t/how-can-i-get-a-list-of-tags-for-a-docker-image-akin-to-the-docker-hub-list/3210
# https://help.sonatype.com/repomanager3/rest-and-integration-api/search-api
curl -k -u admin:84091bcd-c82f-44a3-8b7b-dfc90f5b7da1 -X GET 'http://nexus.ocp4.redhat.ren:8082/service/rest/v1/search?repository=registry.redhat.io'

curl -u admin:84091bcd-c82f-44a3-8b7b-dfc90f5b7da1 -X GET 'http://nexus.ocp4.redhat.ren:8082/service/rest/v1/components?repository=registry.redhat.io'

podman pull docker.io/anoxis/registry-cli
podman run --rm anoxis/registry-cli -l admin:84091bcd-c82f-44a3-8b7b-dfc90f5b7da1 -r https://nexus.ocp4.redhat.ren:8083

# https://github.com/rpardini/docker-registry-proxy

REPO_URL=nexus.ocp4.redhat.ren:8083

curl -k -s -X GET https://$REPO_URL/v2/_catalog \
 | jq '.repositories[]' \
 | sort \
 | xargs -I _ curl -s -k -X GET https://$REPO_URL/v2/_/tags/list



##################################################
## prepare for baidu disk
mkdir -p /data/ccn/baidu
cd /data/ccn

tar cf - ./nexus-image | pigz -c > /data/ccn/baidu/nexus-image.tgz 

cd /data/ccn/baidu
split -b 20000m nexus-image.tgz  nexus-image.tgz.
rm -f nexus-image.tgz

yum -y install python3-pip
pip3 install --user bypy 
/root/.local/bin/bypy list
/root/.local/bin/bypy upload

upload to baidu disk

export BUILDNUMBER=4.6.28

mkdir -p /data/bypy
cd /data
tar -cvf - ocp4/ | pigz -c > /data/bypy/ocp.$BUILDNUMBER.tgz
tar -cvf - registry/ | pigz -c > /data/bypy/registry.$BUILDNUMBER.tgz

cd /data/bypy
# https://github.com/houtianze/bypy
yum -y install python3-pip
pip3 install --user bypy 
/root/.local/bin/bypy list
/root/.local/bin/bypy upload