← Back to Index

openshift 4 kvm+ovs install

openshift4在日常的安装场景中，有这样一个情况，就是需要在多台配置小一些的主机上，实现跨主机的集群安装，这就需要多个kvm跨主机通讯，本来使用bridge方式，搭配可直连的ip地址，是可以满足的，但是由于ip地址管理的限制，我们没有可以直连的ip地址，那么我们就需要ovs+vxlan的方式，来解决这个问题。

本文针对2台主机，讲述如何配置ovs，以及如何启动kvm。

参考资料：

https://stackoverflow.com/questions/30622680/kvm-ovs-bridged-network-how-to-configure
https://stackoverflow.com/questions/31566658/setup-private-networking-between-two-hosts-and-two-vms-with-libvirt-openvswitc
https://blog.csdn.net/wuliangtianzu/article/details/81870551
https://pinrojas.com/2017/05/03/how-to-use-virt-install-to-connect-at-openvswitch-bridges/
https://www.jianshu.com/p/658332deac99
https://developer.gnome.org/NetworkManager/stable/nm-openvswitch.html

mtu 调整：

https://www.cnblogs.com/JacZhu/p/11006738.html
https://stackoom.com/question/3gFcR/%E6%97%A0%E6%B3%95%E5%9C%A8OVS%E9%9A%A7%E9%81%93%E4%B8%AD%E6%8D%95%E8%8E%B7%E5%A4%A7%E4%BA%8EMTU-%E7%9A%84%E6%B5%81%E9%87%8F
https://serverfault.com/questions/680635/mtu-on-open-vswitch-bridge-port
https://stackoverflow.com/questions/54398827/unable-to-capture-traffic-greater-than-mtu-1500-in-ovs-tunnel

vxlan

https://blog.csdn.net/a363344923/article/details/98033856
https://prolinuxhub.com/configure-start-up-scripts-for-ovs-on-centos-and-red-hat/

nat

https://www.sdnlab.com/19842.html
https://www.sdnlab.com/19802.html
https://www.sdnlab.com/19765.html

基于本文的ocp4安装实践，见笔记： https://github.com/wangzheng422/docker_env/blob/master/redhat/prepare/cmri/lab.md

on redhat-01


        yum -y install openvswitch2.11 NetworkManager-ovs
        
        # install pkg for vnc and kvm
        
        systemctl enable --now openvswitch
        systemctl status openvswitch
        
        systemctl enable --now libvirtd
        
        cat << 'EOF' > /etc/sysconfig/network-scripts/ifcfg-br-int 
        DEVICE=br-int
        ONBOOT=yes
        DEVICETYPE=ovs
        TYPE=OVSBridge
        BOOTPROTO=static
        HOTPLUG=no
        IPADDR=192.168.7.1
        PREFIX=24
        MTU=1450
        EOF
        
        cat << 'EOF' > /etc/sysconfig/network-scripts/ifcfg-vxlan1
        DEVICE=vxlan1
        ONBOOT=yes
        DEVICETYPE=ovs
        TYPE=OVSTunnel
        OVS_BRIDGE=br-int
        OVS_TUNNEL_TYPE=vxlan
        OVS_TUNNEL_OPTIONS="options:remote_ip=172.29.159.100"
        BOOTPROTO=static
        HOTPLUG=no
        EOF
        
        systemctl restart network
        
        ovs-vsctl show
        
        # ovs-vsctl set int br-int mtu_request=1450
        
        # ovs-vsctl set int br-int mtu_request=[]
        
        mkdir -p /data/kvm
        cd /data/kvm
        
        # bridge mode
        
        cat << 'EOF' > ovsnet.xml
        <network>
          <name>br-int</name>
          <forward mode='bridge'/>
          <bridge name='br-int'/>
          <virtualport type='openvswitch'/>
        </network>
        EOF
        
        virsh net-define ovsnet.xml
        virsh net-start br-int
        virsh net-autostart br-int
        
        # restore
        
        virsh net-destroy br-int
        virsh net-undefine br-int
        /bin/rm -f /etc/sysconfig/network-scripts/ifcfg-br-int 
        /bin/rm -f /etc/sysconfig/network-scripts/ifcfg-vxlan1
        systemctl restart network

on redhat-02

 
        yum -y install openvswitch2.11 NetworkManager-ovs
        
        # install pkg for vnc and kvm
        
        systemctl enable --now openvswitch
        systemctl status openvswitch
        
        systemctl enable --now libvirtd
        
        ovs-vsctl show
        
        cat << 'EOF' > /etc/sysconfig/network-scripts/ifcfg-br-int 
        DEVICE=br-int
        ONBOOT=yes
        DEVICETYPE=ovs
        TYPE=OVSBridge
        BOOTPROTO=static
        HOTPLUG=no
        IPADDR=192.168.7.2
        PREFIX=24
        MTU=1450
        EOF
        
        cat << 'EOF' > /etc/sysconfig/network-scripts/ifcfg-vxlan1
        DEVICE=vxlan1
        ONBOOT=yes
        DEVICETYPE=ovs
        TYPE=OVSTunnel
        OVS_BRIDGE=br-int
        OVS_TUNNEL_TYPE=vxlan
        OVS_TUNNEL_OPTIONS="options:remote_ip=172.29.159.99"
        BOOTPROTO=static
        HOTPLUG=no
        EOF
        
        systemctl restart network
        
        ovs-vsctl show
        
        # ovs-vsctl set int br-int mtu_request=1450
        
        mkdir -p /data/kvm
        cd /data/kvm
        
        # bridge mode
        
        cat << 'EOF' > ovsnet.xml
        <network>
          <name>br-int</name>
          <forward mode='bridge'/>
          <bridge name='br-int'/>
          <virtualport type='openvswitch'/>
        </network>
        EOF
        
        virsh net-define ovsnet.xml
        virsh net-start br-int
        virsh net-autostart br-int
        
        # restore
        
        virsh net-destroy br-int
        virsh net-undefine br-int

创建虚拟机

虚机创建，注意调整每个虚机的mtu，关键在虚拟机里面，操作系统对网卡mtu的设置，这个其实是kernel安装的时候，启动参数的问题，请参考这里： https://www.man7.org/linux/man-pages/man7/dracut.cmdline.7.html


        mkdir -p /data/kvm
        cd /data/kvm
        
        lvremove -f datavg/helperlv
        lvcreate -y -L 230G -n helperlv datavg
        
        # 230G
        
        virt-install --name="ocp4-aHelper" --vcpus=2 --ram=4096 \
        --disk path=/dev/datavg/helperlv,device=disk,bus=virtio,format=raw \
        --os-variant centos7.0 --network network:br-int,model=virtio \
        --boot menu=on --location /data/kvm/rhel-server-7.8-x86_64-dvd.iso \
        --initrd-inject /data/kvm/helper-ks.cfg --extra-args "inst.ks=file:/helper-ks.cfg"

弯路

ovs上的虚拟机，要开启mtu调整


        sysctl -w net.ipv4.tcp_mtu_probing=1
        
        cat << 'EOF' > /etc/sysctl.d/99-sysctl-wzh.conf
        net.ipv4.tcp_mtu_probing = 1
        EOF
        
        sysctl --system
        
        ovs-vsctl add-port br-int vxlan1 -- \
          set Interface vxlan1 type=vxlan options:remote_ip=172.29.159.99
        
        ovs-vsctl set int br-int mtu_request=1450
        
        nmcli connection add type vxlan id 100 remote 172.29.159.99 ipv4.addresses 192.168.77.2/24 ipv4.method manual ifname vxlan1 connection.id vxlan1 vxlan.parent enp2s0f0 
        nmcli conn up vxlan1
        
        nmcli conn del vxlan1
        
        ovs-vsctl add-port br-int vxlan1 -- \
          set Interface vxlan1 type=vxlan options:remote_ip=172.29.159.100
        
        ovs-vsctl set int br-int mtu_request=1450
        ovs-vsctl set int br-int mtu_request=[]
        
        systemctl restart network
        
        # restore
        
        ovs-vsctl del-port br-int vxlan1
        ovs-vsctl del-br br-int
        rm -f /etc/sysconfig/network-scripts/ifcfg-br-int 
        systemctl restart network
        
        man nm-openvswitch
        
        nmcli con add type ovs-bridge \
            con-name br-private \
            ifname br-private \
            ipv4.method 'manual' \
            ipv4.address '192.168.7.1/24' 
        
        nmcli connection modify br-private ipv4.addresses 192.168.7.1/24
        nmcli connection modify eno2 ipv4.gateway 192.168.39.254
        nmcli connection modify eno2 ipv4.dns 192.168.39.129
        nmcli connection modify br-private ipv4.method manual
        nmcli connection modify br-private connection.autoconnect yes
        nmcli connection modify br-private connection.autoconnect yes
        nmcli connection reload
        
        nmcli con del br-private
        
        nmcli connection add type vxlan id 100 remote 172.29.159.100 ipv4.addresses 192.168.77.1/24 ipv4.method manual ifname vxlan1 connection.id vxlan1 vxlan.parent enp2s0f0 
        nmcli conn up vxlan1
        
        nmcli conn del vxlan1
        
        nmcli conn add type ovs-bridge conn.interface bridge0
        nmcli conn add type ovs-port conn.interface port0 master bridge0
        nmcli conn add type ovs-interface conn.interface iface0 master port0 \
                     ipv4.method manual ipv4.address 192.168.7.1/24
        
        nmcli conn del ovs-slave-iface0
        nmcli conn del ovs-slave-port0
        nmcli conn del ovs-bridge-bridge0
        
        ovs-vsctl add-br br-private
        
        ovs-dpctl show
        ovs-ofctl show br0