Page MenuHomeVyOS Platform

DNAT with multiwan and policy routing, incoming connections only work on primary interface
Closed, ResolvedPublicBUG

Description

In testing a 3 WAN setup with policy routing, DNAT only works on the primary interface. I have port 80 forwarded on all public interfaces, eth0 eth1 and eth2. However, incoming connections only work on eth0.

me@gw01# run show version

Version:          VyOS 1.4-rolling-202204162001
Release train:    sagitta

Built by:         autobuild@vyos.net
Built on:         Sat 16 Apr 2022 20:01 UTC
Build UUID:       200f84d9-9805-4087-9fd9-e63473b1110c
Build commit ID:  50c10220997f6b

Architecture:     x86_64
Boot via:         installed image
System type:      bare metal

Hardware vendor:  Supermicro
Hardware model:   Super Server
Hardware S/N:     0123456789
Hardware UUID:    00000000-0000-0000-0000-3cecef01a24c

Copyright:        VyOS maintainers and contributors
me@gw01# show policy
 prefix-list LOC-Network-v4 {
     rule 10 {
         action permit
         le 32
         prefix 10.0.0.0/8
     }
 }
 prefix-list BlockIPConflicts {
     description "Prevent Conflicting Routes"
     rule 10 {
         action permit
         description "Internal IP Space"
         le 32
         prefix 172.18.60.0/24
     }
     rule 20 {
         action permit
         description "Internal IP Space"
         le 32
         prefix 10.0.0.0/16
     }
 }
 prefix-list DN42-Network-v4 {
     rule 10 {
         action permit
         le 32
         prefix 172.20.0.0/14
     }
 }
 prefix-list6 LOC-Network-v6 {
     rule 10 {
         action permit
         le 128
         prefix fc00::/56
     }
 }
 prefix-list6 BlockIPConflicts-v6 {
     description "Prevent Conflicting Routes"
     rule 10 {
         action permit
         description "Internal IP Space"
         le 128
         prefix fc00::/64
     }
 }
 prefix-list6 DN42-Network-v6 {
     rule 10 {
         action permit
         le 128
         prefix fd00::/8
     }
 }
 route LAN-Policy {
     rule 10 {
         description "IRC Through Secure VPN Tunnel"
         destination {
             group {
                 address-group Unsafe_IRC_Servers
             }
         }
         set {
             table 114
         }
     }
     rule 20 {
         description "IRC Ports Through Secure VPN Tunnel"
         destination {
             group {
                 port-group Unsafe_IRC_Ports
             }
         }
         protocol tcp
         set {
             table 114
         }
     }
     rule 30 {
         description "Local Net Hosts Through VPN Tunnel"
         set {
             table 114
         }
         source {
             group {
                 network-group InternalNetVPNOnly
             }
         }
     }
     rule 40 {
         description "Youtube & Google Through NW"
         destination {
             group {
                 network-group YouTube-Google-IPv4
             }
         }
         set {
             table 102
         }
     }
 }
 route-map LOC-Peering-Export {
     rule 10 {
         action permit
         description "Allow LOC-Network"
         match {
             ip {
                 address {
                     prefix-list LOC-Network-v4
                 }
             }
         }
     }
     rule 20 {
         action permit
         description "Allow LOC-Network"
         match {
             ipv6 {
                 address {
                     prefix-list LOC-Network-v6
                 }
             }
         }
     }
     rule 30 {
         action permit
         description "Allow DN42-Network"
         match {
             ip {
                 address {
                     prefix-list DN42-Network-v4
                 }
             }
         }
     }
     rule 40 {
         action permit
         description "Allow DN42-Network"
         match {
             ipv6 {
                 address {
                     prefix-list DN42-Network-v6
                 }
             }
         }
     }
     rule 50 {
         action deny
         match {
             rpki invalid
         }
     }
     rule 100 {
         action deny
     }
 }
 route-map LOC-Peering-Import {
     rule 10 {
         action deny
         description "Prevent IP Conflicts"
         match {
             ip {
                 address {
                     prefix-list BlockIPConflicts
                 }
             }
         }
     }
     rule 20 {
         action deny
         description "Prevent IP Conflicts"
         match {
             ipv6 {
                 address {
                     prefix-list BlockIPConflicts-v6
                 }
             }
         }
     }
     rule 30 {
         action permit
         description "Allow LOC-Network"
         match {
             ip {
                 address {
                     prefix-list LOC-Network-v4
                 }
             }
         }
     }
     rule 40 {
         action permit
         description "Allow LOC-Network"
         match {
             ipv6 {
                 address {
                     prefix-list LOC-Network-v6
                 }
             }
         }
     }
     rule 50 {
         action deny
         match {
             rpki invalid
         }
     }
     rule 100 {
         action deny
     }
 }
 route6 LAN-Policy {
     rule 10 {
         description "IRC Through Secure VPN Tunnel"
         destination {
             group {
                 address-group Unsafe_IRC_Servers
             }
         }
         set {
             table 114
         }
     }
     rule 20 {
         description "IRC Ports Through Secure VPN Tunnel"
         destination {
             group {
                 port-group Unsafe_IRC_Ports
             }
         }
         protocol tcp
         set {
             table 114
         }
     }
 }
me@gw01#  show interfaces
 ethernet eth0 {
     address dhcp
     description VZ
     firewall {
         in {
             name OUTSIDE-IN
         }
         local {
             name OUTSIDE-LOCAL
         }
         out {
             ipv6-name OUTSIDE-6-OUT
             name OUTSIDE-OUT
         }
     }
     traffic-policy {
         out VZFiOSOut
     }
 }
 ethernet eth1 {
     address dhcp
     address dhcpv6
     description SpectrumTWC
     dhcpv6-options {
         pd 0 {
             interface eth3 {
                 sla-id 0
             }
             length 56
         }
     }
     firewall {
         in {
             ipv6-name OUTSIDE-6-IN
             name OUTSIDE-IN
         }
         local {
             ipv6-name OUTSIDE-6-LOCAL
             name OUTSIDE-LOCAL
         }
         out {
             ipv6-name OUTSIDE-6-OUT
             name OUTSIDE-OUT
         }
     }
     traffic-policy {
         out SpectrumOut
     }
 }
 ethernet eth2 {
     address dhcp
     description NaturalWireless
     firewall {
         in {
             name OUTSIDE-IN
         }
         local {
             name OUTSIDE-LOCAL
         }
         out {
             ipv6-name OUTSIDE-6-OUT
             name OUTSIDE-OUT
         }
     }
     traffic-policy {
         out NWOut
     }
 }
 ethernet eth3 {
     address 172.18.50.1/24
     description NetworkTest
     ipv6 {
         address {
             autoconf
         }
     }
 }
 ethernet eth4 {
     address 10.0.0.1/16
     address fc00::1/64
     description LAN
     firewall {
         in {
             name LAN-IN
         }
         out {
         }
     }
     ipv6 {
         address {
             autoconf
         }
     }
     policy {
         route LAN-Policy
         route6 LAN-Policy
     }
 }
 ethernet eth5 {
     disable
     hw-id 40:a6:b7:20:4d:f9
 }
 loopback lo {
 }
 wireguard wg100 {
     address 192.168.10.1/24
     address fc00:0:0:1::1/64
     description "WireGuard VPN RoadWarrior"
     peer MyiPad {
         allowed-ips 192.168.10.2/32
         allowed-ips fc00:0:0:1::2/128
         persistent-keepalive 15
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     peer MyiPhone {
         allowed-ips 192.168.10.3/32
         allowed-ips fc00:0:0:1::3/128
         persistent-keepalive 15
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     port 51820
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
 wireguard wg101 {
     address 10.66.98.57/32
     address fc00:bbbb:bbbb:bb01::3:6238/128
     description "WireGuard Mullvad NYC"
     firewall {
         in {
             ipv6-name PUB-WIREGUARD-6-IN
             name PUB-WIREGUARD-IN
         }
         local {
             ipv6-name PUB-WIREGUARD-6-LOCAL
             name PUB-WIREGUARD-LOCAL
         }
     }
     peer mullvad {
         address x.x.x.x
         allowed-ips 0.0.0.0/0
         allowed-ips ::0/0
         persistent-keepalive 10
         port 51820
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
 wireguard wg102 {
     address 10.66.32.65/32
     address fc00:bbbb:bbbb:bb01::3:2040/128
     description "WireGuard Mullvad us123"
     firewall {
         in {
             ipv6-name PUB-WIREGUARD-6-IN
             name PUB-WIREGUARD-IN
         }
         local {
             ipv6-name PUB-WIREGUARD-6-LOCAL
             name PUB-WIREGUARD-LOCAL
         }
     }
     peer mullvad {
         address x.x.x.x
         allowed-ips 0.0.0.0/0
         allowed-ips ::0/0
         persistent-keepalive 10
         port 51820
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
 wireguard wg103 {
     address 10.64.215.200/32
     address fc00:bbbb:bbbb:bb01::1:d7c7/128
     description "WireGuard Mullvad us119"
     firewall {
         in {
             ipv6-name PUB-WIREGUARD-6-IN
             name PUB-WIREGUARD-IN
         }
         local {
             ipv6-name PUB-WIREGUARD-6-LOCAL
             name PUB-WIREGUARD-LOCAL
         }
     }
     peer mullvad {
         address x.x.x.x
         allowed-ips 0.0.0.0/0
         allowed-ips ::0/0
         persistent-keepalive 10
         port 51820
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
 wireguard wg104 {
     address 10.64.126.218/32
     address fc00:bbbb:bbbb:bb01::1:7ed9/128
     description "Mullvad Multihop"
     firewall {
         in {
             ipv6-name PUB-WIREGUARD-6-IN
             name PUB-WIREGUARD-IN
         }
         local {
             ipv6-name PUB-WIREGUARD-6-LOCAL
             name PUB-WIREGUARD-LOCAL
         }
     }
     peer mullvad {
         address x.x.x.x
         allowed-ips 0.0.0.0/0
         allowed-ips ::0/0
         port 3049
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
 wireguard wg110 {
     address 172.18.60.2/32
     description aws.example.com
     mtu 1420
     peer gw01.aws.example.com {
         address x.x.x.x
         allowed-ips 0.0.0.0/0
         allowed-ips ::/0
         persistent-keepalive 10
         port 51820
         preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     port 51822
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
 wireguard wg111 {
     address 172.18.60.8/32
     description colorado.example.com
     mtu 1420
     peer gw01.aws.example.com {
         address x.x.x.x
         allowed-ips 0.0.0.0/0
         allowed-ips ::/0
         persistent-keepalive 10
         port 51821
         preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     port 51824
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
 wireguard wg112 {
     address 172.18.60.4/32
     description ia.example.com
     mtu 1420
     peer location1 {
         address x.x.x.x
         allowed-ips 0.0.0.0/0
         allowed-ips ::/0
         persistent-keepalive 10
         port 51822
         preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     port 51823
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
 wireguard wg113 {
     address 172.18.60.12/32
     description zaius.example.com
     mtu 1420
     peer location1 {
         address x.x.x.x
         allowed-ips 0.0.0.0/0
         allowed-ips ::/0
         persistent-keepalive 10
         port 51822
         preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     port 51825
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
 wireguard wg120 {
     address 172.18.60.21/31
     address fc00:0000:0000:20::2/64
     description dn42-us-nj01.exabyte.network
     peer dn42-us-nj {
         address x.x.x.x
         allowed-ips 0.0.0.0/0
         allowed-ips ::0/0
         persistent-keepalive 10
         port 51820
         preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
         public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
     }
     private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 }
me@gw01# show nat
 destination {
     rule 10 {
         description "Port Forward: PLEX to 10.0.10.1"
         destination {
             port 32408
         }
         inbound-interface eth0
         protocol tcp
         translation {
             address 10.0.10.1
             port 32400
         }
     }
     rule 20 {
         description "Port Forward: PLEX to 10.0.10.1"
         destination {
             port 32408
         }
         inbound-interface eth1
         protocol tcp
         translation {
             address 10.0.10.1
             port 32400
         }
     }
     rule 30 {
         description "Port Forward: PLEX to 10.0.10.1"
         destination {
             port 32408
         }
         inbound-interface eth2
         protocol tcp
         translation {
             address 10.0.10.1
             port 32400
         }
     }
     rule 40 {
         description "Port Forward: HTTP to 10.0.10.10 for NGINX"
         destination {
             port 80
         }
         inbound-interface eth0
         protocol tcp
         translation {
             address 10.0.10.10
             port 80
         }
     }
     rule 41 {
         description "Port Forward: HTTP to 10.0.10.10 for NGINX"
         destination {
             port 80
         }
         inbound-interface eth1
         protocol tcp
         translation {
             address 10.0.10.10
             port 80
         }
     }
     rule 42 {
         description "Port Forward: HTTP to 10.0.10.10 for NGINX"
         destination {
             port 80
         }
         inbound-interface eth2
         protocol tcp
         translation {
             address 10.0.10.10
             port 80
         }
     }
     rule 50 {
         description "Port Forward: HTTPS to 10.0.10.10 for NGINX"
         destination {
             port 443
         }
         inbound-interface eth0
         protocol tcp
         translation {
             address 10.0.10.10
             port 443
         }
     }
     rule 51 {
         description "Port Forward: HTTPS to 10.0.10.10 for NGINX"
         destination {
             port 443
         }
         inbound-interface eth1
         protocol tcp
         translation {
             address 10.0.10.10
             port 443
         }
     }
     rule 52 {
         description "Port Forward: HTTPS to 10.0.10.10 for NGINX"
         destination {
             port 443
         }
         inbound-interface eth2
         protocol tcp
         translation {
             address 10.0.10.10
             port 443
         }
     }
 }
 source {
     rule 100 {
         outbound-interface eth0
         source {
             address 10.0.0.0/16
         }
         translation {
             address masquerade
         }
     }
     rule 101 {
         outbound-interface eth0
         source {
             address 172.18.50.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 102 {
         outbound-interface eth0
         source {
             address 192.168.10.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 110 {
         outbound-interface eth1
         source {
             address 10.0.0.0/16
         }
         translation {
             address masquerade
         }
     }
     rule 111 {
         outbound-interface eth1
         source {
             address 172.18.50.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 112 {
         outbound-interface eth1
         source {
             address 192.168.10.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 120 {
         outbound-interface eth2
         source {
             address 10.0.0.0/16
         }
         translation {
             address masquerade
         }
     }
     rule 121 {
         outbound-interface eth2
         source {
             address 172.18.50.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 122 {
         outbound-interface eth2
         source {
             address 192.168.10.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 130 {
         outbound-interface wg101
         source {
             address 10.0.0.0/16
         }
         translation {
             address masquerade
         }
     }
     rule 131 {
         outbound-interface wg101
         source {
             address 172.18.60.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 132 {
         outbound-interface wg101
         source {
             address 172.18.50.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 133 {
         outbound-interface wg101
         source {
             address 192.168.10.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 140 {
         outbound-interface wg102
         source {
             address 10.0.0.0/16
         }
         translation {
             address masquerade
         }
     }
     rule 141 {
         outbound-interface wg102
         source {
             address 172.18.60.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 142 {
         outbound-interface wg102
         source {
             address 172.18.50.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 143 {
         outbound-interface wg102
         source {
             address 192.168.10.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 150 {
         outbound-interface wg103
         source {
             address 10.0.0.0/16
         }
         translation {
             address masquerade
         }
     }
     rule 151 {
         outbound-interface wg103
         source {
             address 172.18.60.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 152 {
         outbound-interface wg103
         source {
             address 172.18.50.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 153 {
         outbound-interface wg103
         source {
             address 192.168.10.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 160 {
         outbound-interface wg104
         source {
             address 10.0.0.0/16
         }
         translation {
             address masquerade
         }
     }
     rule 161 {
         outbound-interface wg104
         source {
             address 172.18.60.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 162 {
         outbound-interface wg104
         source {
             address 172.18.50.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 163 {
         outbound-interface wg104
         source {
             address 192.168.10.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 170 {
         outbound-interface wg120
         source {
             address 10.0.0.0/8
         }
         translation {
             address masquerade
         }
     }
     rule 171 {
         outbound-interface wg120
         source {
             address 172.18.60.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 172 {
         outbound-interface wg120
         source {
             address 172.18.50.0/24
         }
         translation {
             address masquerade
         }
     }
     rule 173 {
         outbound-interface wg120
         source {
             address 192.168.10.0/24
         }
         translation {
             address masquerade
         }
     }
 }

Adding the below partially fixes this when I specify the IP of the outgoing interface. I can now ping and ssh directly into eth1 and eth2 which previously did not work. Port forwarding (DNAT) still does not work.

set policy local-route rule 100 source 1.2.3.4
set policy local-route rule 100 set table 100

set policy local-route rule 101 source 2.3.4.5
set policy local-route rule 101 set table 101

set policy local-route rule 102 source 3.4.5.6
set policy local-route rule 102 set table 102

Details

Version
VyOS 1.4-rolling-202204190217
Is it a breaking change?
Unspecified (possibly destroys the router)
Issue type
Bug (incorrect behavior)

Event Timeline

I didn't test it, but you need something like this or combinations..

set policy route MARK-80-eth0 rule 10 destination port '80'
set policy route MARK-80-eth0 rule 10 protocol 'tcp'
set policy route MARK-80-eth0 rule 10 set mark '100'
set policy route MARK-80-eth0 rule 10 set table '100'

set policy route MARK-80-eth1 rule 10 destination port '80'
set policy route MARK-80-eth1 rule 10 protocol 'tcp'
set policy route MARK-80-eth1 rule 10 set mark '101'
set policy route MARK-80-eth1 rule 10 set table '101'

set policy route MARK-80-eth2 rule 10 destination port '80'
set policy route MARK-80-eth2 rule 10 protocol 'tcp'
set policy route MARK-80-eth2 rule 10 set mark '102'
set policy route MARK-80-eth2 rule 10 set table '102'

set policy local-route rule 110 fwmark '100'
set policy local-route rule 110 set table '100'
set policy local-route rule 111 fwmark '101'
set policy local-route rule 111 set table '101'
set policy local-route rule 112 fwmark '102'
set policy local-route rule 112 set table '102'

set protocols static table 100 route 0.0.0.0/0  dhcp-interface eth0
set protocols static table 101 route 0.0.0.0/0  dhcp-interface eth1
set protocols static table 102 route 0.0.0.0/0  dhcp-interface eth2

set interfaces ethernet eth0 policy route MARK-80-eth0
set interfaces ethernet eth1 policy route MARK-80-eth1
set interfaces ethernet eth2 policy route MARK-80-eth2

@ajgnet Could you test it in the test environment? Or try your rules without firewall

Tested, does not work. Even with all firewall rules removed.

set policy route mark-in-eth0 rule 10 destination port '80'
set policy route mark-in-eth0 rule 10 protocol 'tcp'
set policy route mark-in-eth0 rule 10 set mark '100'
set policy route mark-in-eth0 rule 10 set table '100'

set policy route mark-in-eth1 rule 10 destination port '80'
set policy route mark-in-eth1 rule 10 protocol 'tcp'
set policy route mark-in-eth1 rule 10 set mark '101'
set policy route mark-in-eth1 rule 10 set table '101'

set policy route mark-in-eth2 rule 10 destination port '80'
set policy route mark-in-eth2 rule 10 protocol 'tcp'
set policy route mark-in-eth2 rule 10 set mark '102'
set policy route mark-in-eth2 rule 10 set table '102'

set policy local-route rule 110 fwmark '100'
set policy local-route rule 110 set table '100'
set policy local-route rule 111 fwmark '101'
set policy local-route rule 111 set table '101'
set policy local-route rule 112 fwmark '102'
set policy local-route rule 112 set table '102'

set interfaces ethernet eth0 policy route mark-in-eth0
set interfaces ethernet eth1 policy route mark-in-eth1
set interfaces ethernet eth2 policy route mark-in-eth2

Also tried the above in addition to adding:

set policy local-route rule 100 source 1.2.3.4
set policy local-route rule 100 set table 100

set policy local-route rule 101 source 2.3.4.5
set policy local-route rule 101 set table 101

set policy local-route rule 102 source 3.4.5.6
set policy local-route rule 102 set table 102

When I add the above local-routes, I am able to connect directly to the eth1 and eth2 interfaces (for example, can ssh into eth1 and eth2). However the DNAT rules do not work.

It looks like mine issue with wan load balancing - reply for dnat-ed packets from secondary interfaces was sent by vyos from "primary" https://phabricator.vyos.net/T4587 . Could you dump traffic and check that possibility

I have prepared a configuration example using one of the latest 1.4 images, where more features were introduced.
Scenario and requirements:

  • One vyos router
  • 3 Uplinks to internet (eth0, eth1 and eth2). Static IP used on three links
  • 2 VLANs
    • vif 2: + New Connections from vif-2 routed through WAN-2 + Server on vif 2 should accept ssh connections from internet, through dnat on 3 WAN interfaces (outside port 122)
    • vif 4: + NewConnections from vif-24routed through WAN-2 + Server on vif 4 should accept ssh connections from internet, through dnat on 3 WAN interfaces (outside port 222)

Configuration:

### Interfaces
vyos@ROUTER# run show config comm | grep interf
set interfaces ethernet eth0 address '198.51.100.2/30'
set interfaces ethernet eth1 address '192.0.2.2/30'
set interfaces ethernet eth2 address '203.0.113.2/30'
set interfaces ethernet eth3 vif 2 address '10.2.2.1/24'
set interfaces ethernet eth3 vif 4 address '10.4.4.1/24'

### Policies
vyos@ROUTER# run show config comm | grep policy
set policy route VIF-2 interface 'eth3.2'
set policy route VIF-2 rule 10 connection-mark '192'
set policy route VIF-2 rule 10 destination group network-group '!LOCALS'
set policy route VIF-2 rule 10 set table '192'
set policy route VIF-2 rule 10 source address '10.2.2.0/24'
set policy route VIF-2 rule 20 connection-mark '198'
set policy route VIF-2 rule 20 destination group network-group '!LOCALS'
set policy route VIF-2 rule 20 set table '198'
set policy route VIF-2 rule 30 connection-mark '113'
set policy route VIF-2 rule 30 destination group network-group '!LOCALS'
set policy route VIF-2 rule 30 set table '113'
set policy route VIF-2 rule 99 description 'New connection through WAN2'
set policy route VIF-2 rule 99 destination group network-group '!LOCALS'
set policy route VIF-2 rule 99 set table '192'
set policy route VIF-2 rule 99 source address '10.2.2.0/24'
set policy route VIF-4 interface 'eth3.4'
set policy route VIF-4 rule 10 connection-mark '192'
set policy route VIF-4 rule 10 destination group network-group '!LOCALS'
set policy route VIF-4 rule 10 set table '192'
set policy route VIF-4 rule 10 source address '10.4.4.0/24'
set policy route VIF-4 rule 20 connection-mark '198'
set policy route VIF-4 rule 20 destination group network-group '!LOCALS'
set policy route VIF-4 rule 20 set table '198'
set policy route VIF-4 rule 20 source address '10.4.4.0/24'
set policy route VIF-4 rule 30 connection-mark '113'
set policy route VIF-4 rule 30 destination group network-group '!LOCALS'
set policy route VIF-4 rule 30 set table '113'
set policy route VIF-4 rule 30 source address '10.4.4.0/24'
set policy route VIF-4 rule 99 description 'New connection through WAN3'
set policy route VIF-4 rule 99 destination group network-group '!LOCALS'
set policy route VIF-4 rule 99 set table '113'
set policy route VIF-4 rule 99 source address '10.4.4.0/24'
set policy route WAN1 interface 'eth0'
set policy route WAN1 rule 10 destination address '198.51.100.2'
set policy route WAN1 rule 10 set connection-mark '198'
set policy route WAN2 interface 'eth1'
set policy route WAN2 rule 10 destination address '192.0.2.2'
set policy route WAN2 rule 10 set connection-mark '192'
set policy route WAN3 interface 'eth2'
set policy route WAN3 rule 10 destination address '203.0.113.2'
set policy route WAN3 rule 10 set connection-mark '113'

### NAT rules
vyos@ROUTER# run show config comm | grep nat
set nat destination rule 10 description 'NAT host-2 - WAN1'
set nat destination rule 10 destination address '198.51.100.2'
set nat destination rule 10 destination port '122'
set nat destination rule 10 inbound-interface 'eth0'
set nat destination rule 10 protocol 'tcp'
set nat destination rule 10 translation address '10.2.2.100'
set nat destination rule 10 translation port '22'
set nat destination rule 20 description 'NAT host-2 - WAN2'
set nat destination rule 20 destination address '192.0.2.2'
set nat destination rule 20 destination port '122'
set nat destination rule 20 inbound-interface 'eth1'
set nat destination rule 20 protocol 'tcp'
set nat destination rule 20 translation address '10.2.2.100'
set nat destination rule 20 translation port '22'
set nat destination rule 30 description 'NAT host-2 - WAN3'
set nat destination rule 30 destination address '203.0.113.2'
set nat destination rule 30 destination port '122'
set nat destination rule 30 inbound-interface 'eth2'
set nat destination rule 30 protocol 'tcp'
set nat destination rule 30 translation address '10.2.2.100'
set nat destination rule 30 translation port '22'
set nat destination rule 110 description 'NAT host-4 - WAN1'
set nat destination rule 110 destination address '198.51.100.2'
set nat destination rule 110 destination port '222'
set nat destination rule 110 inbound-interface 'eth0'
set nat destination rule 110 protocol 'tcp'
set nat destination rule 110 translation address '10.4.4.100'
set nat destination rule 110 translation port '22'
set nat destination rule 120 description 'NAT host-4 - WAN2'
set nat destination rule 120 destination address '192.0.2.2'
set nat destination rule 120 destination port '222'
set nat destination rule 120 inbound-interface 'eth1'
set nat destination rule 120 protocol 'tcp'
set nat destination rule 120 translation address '10.4.4.100'
set nat destination rule 120 translation port '22'
set nat destination rule 130 description 'NAT host-4 - WAN3'
set nat destination rule 130 destination address '203.0.113.2'
set nat destination rule 130 destination port '222'
set nat destination rule 130 inbound-interface 'eth2'
set nat destination rule 130 protocol 'tcp'
set nat destination rule 130 translation address '10.4.4.100'
set nat destination rule 130 translation port '22'
set nat source rule 10 outbound-interface 'eth0'
set nat source rule 10 translation address 'masquerade'
set nat source rule 20 outbound-interface 'eth1'
set nat source rule 20 translation address 'masquerade'
set nat source rule 30 outbound-interface 'eth2'
set nat source rule 30 translation address 'masquerade'

Checks:
01: Check dNAT isworking for both servers using all internet connections:

## Check conntrack status for such purpose while 6 ssh connections are established
vyos@ROUTER# sudo conntrack -L | grep tcp
conntrack v1.4.6 (conntrack-tools): 6 flow entries have been shown.
tcp      6 431385 ESTABLISHED src=192.168.99.100 dst=203.0.113.2 sport=36350 dport=222 src=10.4.4.100 dst=192.168.99.100 sport=22 dport=36350 [ASSURED] mark=113 helper=tns use=1
tcp      6 431337 ESTABLISHED src=192.168.99.100 dst=192.0.2.2 sport=43672 dport=222 src=10.4.4.100 dst=192.168.99.100 sport=22 dport=43672 [ASSURED] mark=192 helper=tns use=1
tcp      6 431324 ESTABLISHED src=192.168.99.100 dst=203.0.113.2 sport=46634 dport=122 src=10.2.2.100 dst=192.168.99.100 sport=22 dport=46634 [ASSURED] mark=113 helper=tns use=1
tcp      6 431359 ESTABLISHED src=192.168.99.100 dst=198.51.100.2 sport=38532 dport=222 src=10.4.4.100 dst=192.168.99.100 sport=22 dport=38532 [ASSURED] mark=198 helper=tns use=1
tcp      6 431164 ESTABLISHED src=192.168.99.100 dst=198.51.100.2 sport=52968 dport=122 src=10.2.2.100 dst=192.168.99.100 sport=22 dport=52968 [ASSURED] mark=198 helper=tns use=1
tcp      6 431220 ESTABLISHED src=192.168.99.100 dst=192.0.2.2 sport=55086 dport=122 src=10.2.2.100 dst=192.168.99.100 sport=22 dport=55086 [ASSURED] mark=192 helper=tns use=1
[edit]
vyos@ROUTER#

Check 02: check that servers are routed as expected:

#Server on vif 2 through WAN2 (192.0.2.1)
vyos@host-2# run traceroute www.google.com
traceroute to www.google.com (142.250.79.132), 30 hops max, 60 byte packets
 1  10.2.2.1 (10.2.2.1)  0.751 ms  0.716 ms  0.709 ms
 2  192.0.2.1 (192.0.2.1)  1.308 ms  1.631 ms  1.589 ms
 3  192.168.0.1 (192.168.0.1)  2.054 ms  2.048 ms  2.119 ms
 4  192.168.100.1 (192.168.100.1)  2.654 ms  2.714 ms  2.869 ms
...


#Server on vif 4 through WAN3 (203.0.113.1)
vyos@host-4# run tracer www.google.com
traceroute to www.google.com (142.250.79.132), 30 hops max, 60 byte packets
 1  10.4.4.1 (10.4.4.1)  0.823 ms  1.352 ms  1.207 ms
 2  203.0.113.1 (203.0.113.1)  1.027 ms  2.089 ms  2.155 ms
 3  192.168.0.1 (192.168.0.1)  2.590 ms  2.614 ms  2.606 ms
 4  192.168.100.1 (192.168.100.1)  3.390 ms  3.315 ms  3.272 ms
....
  • No WLB used: just the rules attached
  • Currently working for traffic through the router. More features needs to be added if we want to achieve the same result for connectios towards the router itself
n.fort claimed this task.

Marking this bug as resolved, since in last comment it shows that DNAT is working using multiple "WANs" connections