OVN EgressIP Limitation Workaround

# Create namespaces
oc create ns ns-blue
oc create ns ns-egress-infra

# Create ServiceAccount in the infra namespace
oc create sa apb-syncer -n ns-egress-infra

# Cleanup: Delete any old local rolebinding if it exists
oc delete rolebinding apb-syncer-binding -n ns-egress-infra --ignore-not-found


# 2. Create ClusterRole
# This role grants permissions to watch pods and manage OVN External Routes
cat <<EOF | oc apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: apb-cluster-manager
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create"]
- apiGroups: ["k8s.ovn.org"]
  resources: ["adminpolicybasedexternalroutes"]
  verbs: ["get", "list", "watch", "patch", "update"]
EOF

# 3. Create ClusterRoleBinding
# Bind the SA 'apb-syncer' in 'ns-egress-infra' to the 'apb-cluster-manager' ClusterRole
cat <<EOF | oc apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: apb-syncer-cluster-binding
subjects:
- kind: ServiceAccount
  name: apb-syncer
  namespace: ns-egress-infra
roleRef:
  kind: ClusterRole
  name: apb-cluster-manager
  apiGroup: rbac.authorization.k8s.io
EOF

# Label the node where the gateway will run
oc label node worker-01-demo egress-node=ns-blue-group

# Create NetworkAttachmentDefinition for macvlan
# Note: This MUST be in the same namespace as the Gateway Pod
cat <<EOF | oc apply -f -
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
  name: ns-blue-external-macvlan
  namespace: ns-egress-infra 
spec:
  config: '{
      "cniVersion": "0.3.1",
      "type": "macvlan",
      "master": "enp2s0",
      "mode": "bridge",
      "ipam": {
        "type": "static"
      }
    }'
EOF

# Create dedicated SA for Gateway
oc create sa gateway-sa -n ns-egress-infra

# Grant privileged SCC to the SA so it can change network settings
oc adm policy add-scc-to-user privileged -z gateway-sa -n ns-egress-infra

# Deploy the Gateway
cat <<'EOF' | oc apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ns-blue-gateway
  namespace: ns-egress-infra
spec:
  replicas: 1 
  selector:
    matchLabels:
      app: ns-blue-gateway
  template:
    metadata:
      labels:
        app: ns-blue-gateway
      annotations:
        k8s.v1.cni.cncf.io/networks: '[
          {
            "name": "ns-blue-external-macvlan", 
            "interface": "macvlan0",
            "ips": ["192.168.99.2/24"]
          }
        ]'
    spec:
      serviceAccountName: gateway-sa
      nodeSelector:
        egress-node: ns-blue-group
      containers:
      - name: gateway
        image: registry.redhat.io/openshift4/ose-egress-router:latest
        command: ["/bin/sh", "-c"]
        args:
        - |
          # 0. Enable IP Forwarding & Disable Reverse Path Filtering
          sysctl -w net.ipv4.ip_forward=1
          sysctl -w net.ipv4.conf.all.rp_filter=0
          sysctl -w net.ipv4.conf.default.rp_filter=0
          sysctl -w net.ipv4.conf.eth0.rp_filter=0
          sysctl -w net.ipv4.conf.macvlan0.rp_filter=0

          # Critical: Allow receiving packets where the destination IP is not localhost
          sysctl -w net.ipv4.conf.eth0.accept_local=1
          sysctl -w net.ipv4.conf.all.accept_local=1
          
          # Ensure Forward chain is ACCEPT
          iptables -P FORWARD ACCEPT
          
          # 1. Save the original OVN Network Gateway
          OVN_GW=$(ip route show default | awk '/default/ {print $3}')
          
          # 2. Add routes for Pod Network and Service Network
          ip route add 10.0.0.0/8 via $OVN_GW dev eth0
          ip route add 172.16.0.0/12 via $OVN_GW dev eth0
          
          # 3. Ensure macvlan0 interface is up
          ip link set macvlan0 up
          
          # 4. Switch default route to point to the physical gateway
          ip route del default
          ip route add default via 192.168.99.1 dev macvlan0
          
          # Get the OVN IP of the gateway pod
          GW_OVN_IP=$(ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}')
          
          # 5. Critical modification: Add policy routing to handle traffic from other pods
          # Use static route modification as policy routing tables were found to be ineffective in testing
          ip route del default
          ip route add default via 192.168.99.1 dev macvlan0
          
          # 6. SNAT Rules
          # Perform SNAT for egress traffic on eth0 (Internal traffic)
          iptables -t nat -A POSTROUTING -o eth0 -j SNAT --to-source $GW_OVN_IP
          
          # Perform SNAT for egress traffic on macvlan0 (External traffic)
          iptables -t nat -A POSTROUTING -o macvlan0 -j SNAT --to-source 192.168.99.2
          
          echo "Gateway started! Egress IP: 192.168.99.2, Physical GW: 192.168.99.1, OVN IP: $GW_OVN_IP";
          ip route show
          iptables -t nat -L -n -v
          
          # Prevent container exit
          sleep infinity;
        securityContext:
          privileged: true
EOF

# Build and push controller image
tee controller.dockerfile <<EOF
FROM registry.access.redhat.com/ubi9/python-312

# Install Python dependencies
RUN pip install kubernetes

# Install OpenShift CLI (oc)
USER 0
RUN curl -L https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz | \
    tar -xzf - -C /usr/local/bin/ oc && \
    chmod +x /usr/local/bin/oc

# Switch back to default user
USER 1001
EOF

podman build -f controller.dockerfile -t quay.io/wangzheng422/qimgs:apb-controller-2026.01.25-v01 .
podman push quay.io/wangzheng422/qimgs:apb-controller-2026.01.25-v01

# Create ConfigMap from the standalone controller.py file
# This assumes controller.py exists in the current directory
oc delete configmap apb-script -n ns-egress-infra --ignore-not-found
oc create configmap apb-script --from-file=controller.py -n ns-egress-infra

# Deploy Controller
cat <<EOF | oc apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: apb-controller
  namespace: ns-egress-infra
spec:
  replicas: 1
  selector:
    matchLabels:
      app: apb-controller
  template:
    metadata:
      labels:
        app: apb-controller
    spec:
      serviceAccountName: apb-syncer
      containers:
      - name: controller
        image: quay.io/wangzheng422/qimgs:apb-controller-2026.01.25-v01
        command: ["/bin/sh", "-c", "python /mnt/controller.py"]
        env:
        # Gateway Configuration
        - name: GATEWAY_NAMESPACE
          value: "ns-egress-infra"
        - name: GATEWAY_LABEL
          value: "app=ns-blue-gateway"
        # Business Pod Configuration
        - name: BUSINESS_NAMESPACE
          value: "ns-blue"
        - name: BUSINESS_LABEL
          value: "app=business-app"
        # APB Configuration
        - name: APB_NAME
          value: "ns-blue-route"
        # OVN Configuration (optional, use defaults if not set)
        - name: OVN_NAMESPACE
          value: "openshift-ovn-kubernetes"
        - name: OVN_ACL_PRIORITY
          value: "21743"
        volumeMounts:
        - name: script
          mountPath: /mnt
      volumes:
      - name: script
        configMap:
          name: apb-script
EOF

# Create the APB Route
# This policy directs traffic from the namespace match to the nextHop
cat <<EOF | oc apply -f -
apiVersion: k8s.ovn.org/v1
kind: AdminPolicyBasedExternalRoute
metadata:
  name: ns-blue-route
  namespace: ns-blue
spec:
  from:
    namespaceSelector:
      matchLabels:
        kubernetes.io/metadata.name: ns-blue
  nextHops:
    static:
    # This IP will be updated by apb-controller to match the gateway pod IP
    - ip: "192.168.99.1"
EOF


# Deploy the actual business application
cat <<EOF | oc apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: business-app
  namespace: ns-blue
spec:
  replicas: 5
  selector:
    matchLabels:
      app: business-app
  template:
    metadata:
      labels:
        app: business-app
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - business-app
              topologyKey: kubernetes.io/hostname
      containers:
      - name: app
        image: quay.io/wangzheng422/qimgs:centos9-test-2025.12.18.v01
        command: ["/bin/sh", "-c", "sleep infinity"]
EOF

# Example: Deploy controller for ns-red namespace
cat <<EOF | oc apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: apb-controller-red
  namespace: ns-egress-infra
spec:
  replicas: 1
  selector:
    matchLabels:
      app: apb-controller-red
  template:
    metadata:
      labels:
        app: apb-controller-red
    spec:
      serviceAccountName: apb-syncer
      containers:
      - name: controller
        image: quay.io/wangzheng422/qimgs:apb-controller-2026.01.25-v01
        command: ["/bin/sh", "-c", "python /mnt/controller.py"]
        env:
        # Gateway Configuration - Use the same infra namespace
        - name: GATEWAY_NAMESPACE
          value: "ns-egress-infra"
        - name: GATEWAY_LABEL
          value: "app=ns-red-gateway"  # Distinct gateway label
        # Business Pod Configuration - Distinct business namespace
        - name: BUSINESS_NAMESPACE
          value: "ns-red"
        - name: BUSINESS_LABEL
          value: "app=business-app"
        # APB Configuration - Distinct APB name
        - name: APB_NAME
          value: "ns-red-route"
        # OVN Configuration - Use different ACL priority to avoid conflict
        - name: OVN_ACL_PRIORITY
          value: "21743"  # Different priority
        volumeMounts:
        - name: script
          mountPath: /mnt
      volumes:
      - name: script
        configMap:
          name: apb-script
EOF


# Corresponding NetworkAttachmentDefinition for ns-red
cat <<EOF | oc apply -f -
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
  name: ns-red-external-macvlan
  namespace: ns-egress-infra  # Must be in the same namespace as the Gateway Pod
spec:
  config: '{
      "cniVersion": "0.3.1",
      "type": "macvlan",
      "master": "enp2s0",
      "mode": "bridge",
      "ipam": {
        "type": "static"
      }
    }'
EOF

# Corresponding Gateway Deployment for ns-red
cat <<'EOF' | oc apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ns-red-gateway
  namespace: ns-egress-infra
spec:
  replicas: 1 
  selector:
    matchLabels:
      app: ns-red-gateway
  template:
    metadata:
      labels:
        app: ns-red-gateway
      annotations:
        k8s.v1.cni.cncf.io/networks: '[
          {
            "name": "ns-red-external-macvlan", 
            "interface": "macvlan0",
            "ips": ["192.168.99.3/24"]
          }
        ]'
    spec:
      serviceAccountName: gateway-sa
      nodeSelector:
        egress-node: ns-blue-group
      containers:
      - name: gateway
        image: registry.redhat.io/openshift4/ose-egress-router:latest
        command: ["/bin/sh", "-c"]
        args:
        - |
          # 0. Enable IP Forwarding & Disable Reverse Path Filtering
          sysctl -w net.ipv4.ip_forward=1
          sysctl -w net.ipv4.conf.all.rp_filter=0
          sysctl -w net.ipv4.conf.default.rp_filter=0
          sysctl -w net.ipv4.conf.eth0.rp_filter=0
          sysctl -w net.ipv4.conf.macvlan0.rp_filter=0

          # Critical: Allow receiving packets where the destination IP is not localhost
          sysctl -w net.ipv4.conf.eth0.accept_local=1
          sysctl -w net.ipv4.conf.all.accept_local=1
          
          # Ensure Forward chain is ACCEPT
          iptables -P FORWARD ACCEPT
          
          # 1. Save the original OVN Network Gateway
          OVN_GW=$(ip route show default | awk '/default/ {print $3}')
          
          # 2. Add routes for Pod Network and Service Network
          ip route add 10.0.0.0/8 via $OVN_GW dev eth0
          ip route add 172.16.0.0/12 via $OVN_GW dev eth0
          
          # 3. Ensure macvlan0 interface is up
          ip link set macvlan0 up
          
          # 4. Switch default route to point to the physical gateway
          ip route del default
          ip route add default via 192.168.99.1 dev macvlan0
          
          # Get the OVN IP of the gateway pod
          GW_OVN_IP=$(ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}')
          
          # 5. SNAT Rules
          # Perform SNAT for egress traffic on eth0 (Internal traffic)
          iptables -t nat -A POSTROUTING -o eth0 -j SNAT --to-source $GW_OVN_IP
          
          # Perform SNAT for egress traffic on macvlan0 (External traffic)
          iptables -t nat -A POSTROUTING -o macvlan0 -j SNAT --to-source 192.168.99.3
          
          echo "Gateway started! Egress IP: 192.168.99.3, Physical GW: 192.168.99.1, OVN IP: $GW_OVN_IP";
          ip route show
          iptables -t nat -L -n -v
          
          # Prevent container exit
          sleep infinity;
        securityContext:
          privileged: true
EOF

# Corresponding APB Policy for ns-red
cat <<EOF | oc apply -f -
apiVersion: k8s.ovn.org/v1
kind: AdminPolicyBasedExternalRoute
metadata:
  name: ns-red-route
spec:
  from:
    namespaceSelector:
      matchLabels:
        kubernetes.io/metadata.name: ns-red
  nextHops:
    static:
    - ip: "192.168.99.1"  # Will be automatically updated by controller
EOF

# Corresponding Business Pod for ns-red
cat <<EOF | oc apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: business-app
  namespace: ns-red
spec:
  replicas: 5
  selector:
    matchLabels:
      app: business-app
  template:
    metadata:
      labels:
        app: business-app
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - business-app
              topologyKey: kubernetes.io/hostname
      containers:
      - name: app
        image: quay.io/wangzheng422/qimgs:centos9-test-2025.12.18.v01
        command: ["/bin/sh", "-c", "sleep infinity"]
EOF