In testing a 3 WAN setup with policy routing, DNAT only works on the primary interface. I have port 80 forwarded on all public interfaces, eth0 eth1 and eth2. However, incoming connections only work on eth0.
me@gw01# run show version Version: VyOS 1.4-rolling-202204162001 Release train: sagitta Built by: autobuild@vyos.net Built on: Sat 16 Apr 2022 20:01 UTC Build UUID: 200f84d9-9805-4087-9fd9-e63473b1110c Build commit ID: 50c10220997f6b Architecture: x86_64 Boot via: installed image System type: bare metal Hardware vendor: Supermicro Hardware model: Super Server Hardware S/N: 0123456789 Hardware UUID: 00000000-0000-0000-0000-3cecef01a24c Copyright: VyOS maintainers and contributors
me@gw01# show policy prefix-list LOC-Network-v4 { rule 10 { action permit le 32 prefix 10.0.0.0/8 } } prefix-list BlockIPConflicts { description "Prevent Conflicting Routes" rule 10 { action permit description "Internal IP Space" le 32 prefix 172.18.60.0/24 } rule 20 { action permit description "Internal IP Space" le 32 prefix 10.0.0.0/16 } } prefix-list DN42-Network-v4 { rule 10 { action permit le 32 prefix 172.20.0.0/14 } } prefix-list6 LOC-Network-v6 { rule 10 { action permit le 128 prefix fc00::/56 } } prefix-list6 BlockIPConflicts-v6 { description "Prevent Conflicting Routes" rule 10 { action permit description "Internal IP Space" le 128 prefix fc00::/64 } } prefix-list6 DN42-Network-v6 { rule 10 { action permit le 128 prefix fd00::/8 } } route LAN-Policy { rule 10 { description "IRC Through Secure VPN Tunnel" destination { group { address-group Unsafe_IRC_Servers } } set { table 114 } } rule 20 { description "IRC Ports Through Secure VPN Tunnel" destination { group { port-group Unsafe_IRC_Ports } } protocol tcp set { table 114 } } rule 30 { description "Local Net Hosts Through VPN Tunnel" set { table 114 } source { group { network-group InternalNetVPNOnly } } } rule 40 { description "Youtube & Google Through NW" destination { group { network-group YouTube-Google-IPv4 } } set { table 102 } } } route-map LOC-Peering-Export { rule 10 { action permit description "Allow LOC-Network" match { ip { address { prefix-list LOC-Network-v4 } } } } rule 20 { action permit description "Allow LOC-Network" match { ipv6 { address { prefix-list LOC-Network-v6 } } } } rule 30 { action permit description "Allow DN42-Network" match { ip { address { prefix-list DN42-Network-v4 } } } } rule 40 { action permit description "Allow DN42-Network" match { ipv6 { address { prefix-list DN42-Network-v6 } } } } rule 50 { action deny match { rpki invalid } } rule 100 { action deny } } route-map LOC-Peering-Import { rule 10 { action deny description "Prevent IP Conflicts" match { ip { address { prefix-list BlockIPConflicts } } } } rule 20 { action deny description "Prevent IP Conflicts" match { ipv6 { address { prefix-list BlockIPConflicts-v6 } } } } rule 30 { action permit description "Allow LOC-Network" match { ip { address { prefix-list LOC-Network-v4 } } } } rule 40 { action permit description "Allow LOC-Network" match { ipv6 { address { prefix-list LOC-Network-v6 } } } } rule 50 { action deny match { rpki invalid } } rule 100 { action deny } } route6 LAN-Policy { rule 10 { description "IRC Through Secure VPN Tunnel" destination { group { address-group Unsafe_IRC_Servers } } set { table 114 } } rule 20 { description "IRC Ports Through Secure VPN Tunnel" destination { group { port-group Unsafe_IRC_Ports } } protocol tcp set { table 114 } } }
me@gw01# show interfaces ethernet eth0 { address dhcp description VZ firewall { in { name OUTSIDE-IN } local { name OUTSIDE-LOCAL } out { ipv6-name OUTSIDE-6-OUT name OUTSIDE-OUT } } traffic-policy { out VZFiOSOut } } ethernet eth1 { address dhcp address dhcpv6 description SpectrumTWC dhcpv6-options { pd 0 { interface eth3 { sla-id 0 } length 56 } } firewall { in { ipv6-name OUTSIDE-6-IN name OUTSIDE-IN } local { ipv6-name OUTSIDE-6-LOCAL name OUTSIDE-LOCAL } out { ipv6-name OUTSIDE-6-OUT name OUTSIDE-OUT } } traffic-policy { out SpectrumOut } } ethernet eth2 { address dhcp description NaturalWireless firewall { in { name OUTSIDE-IN } local { name OUTSIDE-LOCAL } out { ipv6-name OUTSIDE-6-OUT name OUTSIDE-OUT } } traffic-policy { out NWOut } } ethernet eth3 { address 172.18.50.1/24 description NetworkTest ipv6 { address { autoconf } } } ethernet eth4 { address 10.0.0.1/16 address fc00::1/64 description LAN firewall { in { name LAN-IN } out { } } ipv6 { address { autoconf } } policy { route LAN-Policy route6 LAN-Policy } } ethernet eth5 { disable hw-id 40:a6:b7:20:4d:f9 } loopback lo { } wireguard wg100 { address 192.168.10.1/24 address fc00:0:0:1::1/64 description "WireGuard VPN RoadWarrior" peer MyiPad { allowed-ips 192.168.10.2/32 allowed-ips fc00:0:0:1::2/128 persistent-keepalive 15 public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } peer MyiPhone { allowed-ips 192.168.10.3/32 allowed-ips fc00:0:0:1::3/128 persistent-keepalive 15 public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } port 51820 private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } wireguard wg101 { address 10.66.98.57/32 address fc00:bbbb:bbbb:bb01::3:6238/128 description "WireGuard Mullvad NYC" firewall { in { ipv6-name PUB-WIREGUARD-6-IN name PUB-WIREGUARD-IN } local { ipv6-name PUB-WIREGUARD-6-LOCAL name PUB-WIREGUARD-LOCAL } } peer mullvad { address x.x.x.x allowed-ips 0.0.0.0/0 allowed-ips ::0/0 persistent-keepalive 10 port 51820 public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } wireguard wg102 { address 10.66.32.65/32 address fc00:bbbb:bbbb:bb01::3:2040/128 description "WireGuard Mullvad us123" firewall { in { ipv6-name PUB-WIREGUARD-6-IN name PUB-WIREGUARD-IN } local { ipv6-name PUB-WIREGUARD-6-LOCAL name PUB-WIREGUARD-LOCAL } } peer mullvad { address x.x.x.x allowed-ips 0.0.0.0/0 allowed-ips ::0/0 persistent-keepalive 10 port 51820 public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } wireguard wg103 { address 10.64.215.200/32 address fc00:bbbb:bbbb:bb01::1:d7c7/128 description "WireGuard Mullvad us119" firewall { in { ipv6-name PUB-WIREGUARD-6-IN name PUB-WIREGUARD-IN } local { ipv6-name PUB-WIREGUARD-6-LOCAL name PUB-WIREGUARD-LOCAL } } peer mullvad { address x.x.x.x allowed-ips 0.0.0.0/0 allowed-ips ::0/0 persistent-keepalive 10 port 51820 public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } wireguard wg104 { address 10.64.126.218/32 address fc00:bbbb:bbbb:bb01::1:7ed9/128 description "Mullvad Multihop" firewall { in { ipv6-name PUB-WIREGUARD-6-IN name PUB-WIREGUARD-IN } local { ipv6-name PUB-WIREGUARD-6-LOCAL name PUB-WIREGUARD-LOCAL } } peer mullvad { address x.x.x.x allowed-ips 0.0.0.0/0 allowed-ips ::0/0 port 3049 public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } wireguard wg110 { address 172.18.60.2/32 description aws.example.com mtu 1420 peer gw01.aws.example.com { address x.x.x.x allowed-ips 0.0.0.0/0 allowed-ips ::/0 persistent-keepalive 10 port 51820 preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } port 51822 private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } wireguard wg111 { address 172.18.60.8/32 description colorado.example.com mtu 1420 peer gw01.aws.example.com { address x.x.x.x allowed-ips 0.0.0.0/0 allowed-ips ::/0 persistent-keepalive 10 port 51821 preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } port 51824 private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } wireguard wg112 { address 172.18.60.4/32 description ia.example.com mtu 1420 peer location1 { address x.x.x.x allowed-ips 0.0.0.0/0 allowed-ips ::/0 persistent-keepalive 10 port 51822 preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } port 51823 private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } wireguard wg113 { address 172.18.60.12/32 description zaius.example.com mtu 1420 peer location1 { address x.x.x.x allowed-ips 0.0.0.0/0 allowed-ips ::/0 persistent-keepalive 10 port 51822 preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } port 51825 private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } wireguard wg120 { address 172.18.60.21/31 address fc00:0000:0000:20::2/64 description dn42-us-nj01.exabyte.network peer dn42-us-nj { address x.x.x.x allowed-ips 0.0.0.0/0 allowed-ips ::0/0 persistent-keepalive 10 port 51820 preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx } private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx }
me@gw01# show nat destination { rule 10 { description "Port Forward: PLEX to 10.0.10.1" destination { port 32408 } inbound-interface eth0 protocol tcp translation { address 10.0.10.1 port 32400 } } rule 20 { description "Port Forward: PLEX to 10.0.10.1" destination { port 32408 } inbound-interface eth1 protocol tcp translation { address 10.0.10.1 port 32400 } } rule 30 { description "Port Forward: PLEX to 10.0.10.1" destination { port 32408 } inbound-interface eth2 protocol tcp translation { address 10.0.10.1 port 32400 } } rule 40 { description "Port Forward: HTTP to 10.0.10.10 for NGINX" destination { port 80 } inbound-interface eth0 protocol tcp translation { address 10.0.10.10 port 80 } } rule 41 { description "Port Forward: HTTP to 10.0.10.10 for NGINX" destination { port 80 } inbound-interface eth1 protocol tcp translation { address 10.0.10.10 port 80 } } rule 42 { description "Port Forward: HTTP to 10.0.10.10 for NGINX" destination { port 80 } inbound-interface eth2 protocol tcp translation { address 10.0.10.10 port 80 } } rule 50 { description "Port Forward: HTTPS to 10.0.10.10 for NGINX" destination { port 443 } inbound-interface eth0 protocol tcp translation { address 10.0.10.10 port 443 } } rule 51 { description "Port Forward: HTTPS to 10.0.10.10 for NGINX" destination { port 443 } inbound-interface eth1 protocol tcp translation { address 10.0.10.10 port 443 } } rule 52 { description "Port Forward: HTTPS to 10.0.10.10 for NGINX" destination { port 443 } inbound-interface eth2 protocol tcp translation { address 10.0.10.10 port 443 } } } source { rule 100 { outbound-interface eth0 source { address 10.0.0.0/16 } translation { address masquerade } } rule 101 { outbound-interface eth0 source { address 172.18.50.0/24 } translation { address masquerade } } rule 102 { outbound-interface eth0 source { address 192.168.10.0/24 } translation { address masquerade } } rule 110 { outbound-interface eth1 source { address 10.0.0.0/16 } translation { address masquerade } } rule 111 { outbound-interface eth1 source { address 172.18.50.0/24 } translation { address masquerade } } rule 112 { outbound-interface eth1 source { address 192.168.10.0/24 } translation { address masquerade } } rule 120 { outbound-interface eth2 source { address 10.0.0.0/16 } translation { address masquerade } } rule 121 { outbound-interface eth2 source { address 172.18.50.0/24 } translation { address masquerade } } rule 122 { outbound-interface eth2 source { address 192.168.10.0/24 } translation { address masquerade } } rule 130 { outbound-interface wg101 source { address 10.0.0.0/16 } translation { address masquerade } } rule 131 { outbound-interface wg101 source { address 172.18.60.0/24 } translation { address masquerade } } rule 132 { outbound-interface wg101 source { address 172.18.50.0/24 } translation { address masquerade } } rule 133 { outbound-interface wg101 source { address 192.168.10.0/24 } translation { address masquerade } } rule 140 { outbound-interface wg102 source { address 10.0.0.0/16 } translation { address masquerade } } rule 141 { outbound-interface wg102 source { address 172.18.60.0/24 } translation { address masquerade } } rule 142 { outbound-interface wg102 source { address 172.18.50.0/24 } translation { address masquerade } } rule 143 { outbound-interface wg102 source { address 192.168.10.0/24 } translation { address masquerade } } rule 150 { outbound-interface wg103 source { address 10.0.0.0/16 } translation { address masquerade } } rule 151 { outbound-interface wg103 source { address 172.18.60.0/24 } translation { address masquerade } } rule 152 { outbound-interface wg103 source { address 172.18.50.0/24 } translation { address masquerade } } rule 153 { outbound-interface wg103 source { address 192.168.10.0/24 } translation { address masquerade } } rule 160 { outbound-interface wg104 source { address 10.0.0.0/16 } translation { address masquerade } } rule 161 { outbound-interface wg104 source { address 172.18.60.0/24 } translation { address masquerade } } rule 162 { outbound-interface wg104 source { address 172.18.50.0/24 } translation { address masquerade } } rule 163 { outbound-interface wg104 source { address 192.168.10.0/24 } translation { address masquerade } } rule 170 { outbound-interface wg120 source { address 10.0.0.0/8 } translation { address masquerade } } rule 171 { outbound-interface wg120 source { address 172.18.60.0/24 } translation { address masquerade } } rule 172 { outbound-interface wg120 source { address 172.18.50.0/24 } translation { address masquerade } } rule 173 { outbound-interface wg120 source { address 192.168.10.0/24 } translation { address masquerade } } }
Adding the below partially fixes this when I specify the IP of the outgoing interface. I can now ping and ssh directly into eth1 and eth2 which previously did not work. Port forwarding (DNAT) still does not work.
set policy local-route rule 100 source 1.2.3.4 set policy local-route rule 100 set table 100 set policy local-route rule 101 source 2.3.4.5 set policy local-route rule 101 set table 101 set policy local-route rule 102 source 3.4.5.6 set policy local-route rule 102 set table 102