Page MenuHomeVyOS Platform
Create Task
Maniphest T4352

wan-load balance - priority traffic rule doesn't work
Closed, ResolvedPublicBUG
Actions

Description

hi

I've been testing wan load balancing feature , I found an issues when you want to create a policy traffic rule , it doesn't work as expected .let me show.

case reference :
https://forum.vyos.io/t/multi-wan-load-balancing-configurations-have-no-effect/8738

#vyos verison : vyos-1.4-rolling-202204060217

## dual-wan configuration ,with rule policy traffic 
set interfaces ethernet eth0 address 'dhcp'
set interfaces ethernet eth0 description 'WAN1'
set interfaces ethernet eth1 address 'dhcp'
set interfaces ethernet eth1 description 'WAN2'
set interfaces ethernet eth3 address '172.16.80.1/24'
set interfaces ethernet eth3 description 'LAN'
set load-balancing wan interface-health eth0 failure-count '5'
set load-balancing wan interface-health eth0 nexthop 'dhcp'
set load-balancing wan interface-health eth0 success-count '1'
set load-balancing wan interface-health eth0 test 10 resp-time '5'
set load-balancing wan interface-health eth0 test 10 target '186.57.128.104'
set load-balancing wan interface-health eth0 test 10 ttl-limit '1'
set load-balancing wan interface-health eth0 test 10 type 'ttl'
set load-balancing wan interface-health eth0 test 20 resp-time '5'
set load-balancing wan interface-health eth0 test 20 target '1.0.0.1'
set load-balancing wan interface-health eth0 test 20 ttl-limit '1'
set load-balancing wan interface-health eth0 test 20 type 'ping'
set load-balancing wan interface-health eth1 failure-count '5'
set load-balancing wan interface-health eth1 nexthop 'dhcp'
set load-balancing wan interface-health eth1 success-count '1'
set load-balancing wan interface-health eth1 test 10 resp-time '5'
set load-balancing wan interface-health eth1 test 10 target '8.8.8.8'
set load-balancing wan interface-health eth1 test 10 ttl-limit '1'
set load-balancing wan interface-health eth1 test 10 type 'ping'
set load-balancing wan interface-health eth1 test 20 resp-time '5'
set load-balancing wan interface-health eth1 test 20 target '8.8.4.4'
set load-balancing wan interface-health eth1 test 20 ttl-limit '1'
set load-balancing wan interface-health eth1 test 20 type 'ping'
set load-balancing wan rule 100 description 'SpeedTest - Spectrum'
set load-balancing wan rule 100 destination address '24.29.97.0/24'
set load-balancing wan rule 100 inbound-interface 'eth3'
set load-balancing wan rule 100 interface eth1 weight '1'
set load-balancing wan rule 100 protocol 'all'
set load-balancing wan rule 100 source address '172.16.80.0/24'
set load-balancing wan rule 110 description 'SpeedTest - NaturalWireless'
set load-balancing wan rule 110 destination address '163.182.128.0/24'
set load-balancing wan rule 110 inbound-interface 'eth3'
set load-balancing wan rule 110 interface eth0 weight '1'
set load-balancing wan rule 110 protocol 'all'
set load-balancing wan rule 110 source address '172.16.80.0/24'
set load-balancing wan rule 1000 description 'DEFAULT FAILOVER RULE'
set load-balancing wan rule 1000 failover
set load-balancing wan rule 1000 inbound-interface 'eth3'
set load-balancing wan rule 1000 interface eth0 weight '3'
set load-balancing wan rule 1000 interface eth1 weight '2'
set load-balancing wan rule 1000 protocol 'all'
set load-balancing wan sticky-connections inbound
set protocols static route 1.0.0.1/32 next-hop 172.16.50.1
set protocols static route 1.1.1.1/32 next-hop 172.16.50.1
set protocols static route 8.8.4.4/32 next-hop 10.10.0.1
set protocols static route 8.8.8.8/32 next-hop 10.10.0.1

if we see the wan load balance status and checking sudo nft -a list ruleset

# status

vyos@test-wan:~$ show wan-load-balance status
Chain WANLOADBALANCE_PRE (1 references)
 pkts bytes target     prot opt in     out     source               destination
   22  1848 ISP_eth1   all  --  eth3   *       172.16.80.0/24       24.29.97.0/24        state NEW
    0     0 CONNMARK   all  --  eth3   *       172.16.80.0/24       24.29.97.0/24        CONNMARK restore
   60  5040 ISP_eth0   all  --  eth3   *       172.16.80.0/24       163.182.128.0/24     state NEW
    0     0 CONNMARK   all  --  eth3   *       172.16.80.0/24       163.182.128.0/24     CONNMARK restore
    0     0 ISP_eth0   all  --  eth3   *       0.0.0.0/0            0.0.0.0/0            state NEW
    0     0 CONNMARK   all  --  eth3   *       0.0.0.0/0            0.0.0.0/0            CONNMARK restore

# ne

vyos@test-wan:~$ sudo  nft -a list ruleset
table ip filter { # handle 28
	chain VYOS_FW_FORWARD { # handle 1
		type filter hook forward priority filter; policy accept;
		jump VYOS_POST_FW # handle 6
	}

	chain VYOS_FW_LOCAL { # handle 2
		type filter hook input priority filter; policy accept;
		jump VYOS_POST_FW # handle 7
	}

	chain VYOS_FW_OUTPUT { # handle 3
		type filter hook output priority filter; policy accept;
		jump VYOS_POST_FW # handle 8
	}

	chain VYOS_POST_FW { # handle 4
		return # handle 9
	}

	chain VYOS_FRAG_MARK { # handle 5
		type filter hook prerouting priority -450; policy accept;
		ip frag-off & 16383 != 0 meta mark set 0x000ffff1 return # handle 10
	}
}
table ip6 filter { # handle 29
	chain VYOS_FW6_FORWARD { # handle 1
		type filter hook forward priority filter; policy accept;
		jump VYOS_POST_FW6 # handle 6
	}

	chain VYOS_FW6_LOCAL { # handle 2
		type filter hook input priority filter; policy accept;
		jump VYOS_POST_FW6 # handle 7
	}

	chain VYOS_FW6_OUTPUT { # handle 3
		type filter hook output priority filter; policy accept;
		jump VYOS_POST_FW6 # handle 8
	}

	chain VYOS_POST_FW6 { # handle 4
		return # handle 9
	}

	chain VYOS_FRAG6_MARK { # handle 5
		type filter hook prerouting priority -450; policy accept;
		exthdr frag exists meta mark set 0x000ffff1 return # handle 10
	}
}
table ip nat { # handle 30
	chain PREROUTING { # handle 1
		type nat hook prerouting priority dstnat; policy accept;
	}

	chain POSTROUTING { # handle 2
		type nat hook postrouting priority srcnat; policy accept;
	}

	chain VYOS_PRE_DNAT_HOOK { # handle 3
	}

	chain VYOS_PRE_SNAT_HOOK { # handle 4
		counter packets 0 bytes 0 jump WANLOADBALANCE # handle 10
	}

	chain WANLOADBALANCE { # handle 9
		ct mark 0xc9 counter packets 0 bytes 0 snat to 172.16.50.16 # handle 11
		ct mark 0xca counter packets 0 bytes 0 snat to 10.10.0.2 # handle 12
	}
}
table ip6 nat { # handle 31
	chain PREROUTING { # handle 1
		type nat hook prerouting priority dstnat; policy accept;
		counter packets 0 bytes 0 jump VYOS_DNPT_HOOK # handle 5
	}

	chain POSTROUTING { # handle 2
		type nat hook postrouting priority srcnat; policy accept;
		counter packets 0 bytes 0 jump VYOS_SNPT_HOOK # handle 6
	}

	chain VYOS_DNPT_HOOK { # handle 3
		return # handle 7
	}

	chain VYOS_SNPT_HOOK { # handle 4
		return # handle 8
	}
}
table inet mangle { # handle 32
	chain FORWARD { # handle 1
		type filter hook forward priority mangle; policy accept;
	}
}
table ip raw { # handle 33
	ct helper rpc_tcp { # handle 10
		type "rpc" protocol tcp
		l3proto ip
	}

	ct helper rpc_udp { # handle 11
		type "rpc" protocol udp
		l3proto ip
	}

	ct helper tns_tcp { # handle 12
		type "tns" protocol tcp
		l3proto ip
	}

	chain VYOS_TCP_MSS { # handle 1
		type filter hook forward priority raw; policy accept;
	}

	chain PREROUTING { # handle 2
		type filter hook prerouting priority -200; policy accept;
		counter packets 1789 bytes 114256 jump VYOS_CT_IGNORE # handle 16
		counter packets 1782 bytes 112240 jump WLB_CONNTRACK # handle 63
		counter packets 7 bytes 2016 jump VYOS_CT_HELPER # handle 57
		counter packets 7 bytes 2016 jump VYOS_CT_TIMEOUT # handle 17
		counter packets 7 bytes 2016 jump VYOS_CT_PREROUTING_HOOK # handle 18
		counter packets 7 bytes 2016 jump NAT_CONNTRACK # handle 59
		counter packets 0 bytes 0 jump FW_CONNTRACK # handle 19
		notrack # handle 20
	}

	chain OUTPUT { # handle 3
		type filter hook output priority -200; policy accept;
		counter packets 1700 bytes 81608 jump VYOS_CT_IGNORE # handle 21
		counter packets 1700 bytes 81608 jump VYOS_CT_HELPER # handle 58
		counter packets 1700 bytes 81608 jump VYOS_CT_TIMEOUT # handle 22
		counter packets 1700 bytes 81608 jump VYOS_CT_OUTPUT_HOOK # handle 23
		counter packets 1700 bytes 81608 jump NAT_CONNTRACK # handle 60
		counter packets 0 bytes 0 jump FW_CONNTRACK # handle 24
		notrack # handle 25
	}

	chain VYOS_CT_HELPER { # handle 4
		ct helper set "tns_tcp" tcp dport { 1521, 1525, 1536 } return # handle 46
		ct helper set "rpc_udp" udp dport { 111 } return # handle 44
		ct helper set "rpc_tcp" tcp dport { 111 } return # handle 42
		return # handle 29
	}

	chain VYOS_CT_IGNORE { # handle 5
		return # handle 47
	}

	chain VYOS_CT_TIMEOUT { # handle 6
		return # handle 48
	}

	chain VYOS_CT_PREROUTING_HOOK { # handle 7
		return # handle 32
	}

	chain VYOS_CT_OUTPUT_HOOK { # handle 8
		return # handle 33
	}

	chain FW_CONNTRACK { # handle 9
		accept # handle 34
	}

	chain NAT_CONNTRACK { # handle 55
		counter packets 1707 bytes 83624 accept # handle 56
	}

	chain WLB_CONNTRACK { # handle 61
		counter packets 1782 bytes 112240 accept # handle 62
	}
}
table ip6 raw { # handle 34
	chain VYOS_TCP_MSS { # handle 1
		type filter hook forward priority raw; policy accept;
	}

	chain PREROUTING { # handle 2
		type filter hook prerouting priority raw; policy accept;
		counter packets 2 bytes 112 jump VYOS_CT_PREROUTING_HOOK # handle 7
		counter packets 2 bytes 112 jump FW_CONNTRACK # handle 8
		notrack # handle 9
	}

	chain OUTPUT { # handle 3
		type filter hook output priority raw; policy accept;
		counter packets 28 bytes 2912 jump VYOS_CT_OUTPUT_HOOK # handle 10
		counter packets 28 bytes 2912 jump FW_CONNTRACK # handle 11
		notrack # handle 12
	}

	chain VYOS_CT_PREROUTING_HOOK { # handle 4
		return # handle 13
	}

	chain VYOS_CT_OUTPUT_HOOK { # handle 5
		return # handle 14
	}

	chain FW_CONNTRACK { # handle 6
		accept # handle 15
	}
}
table ip mangle { # handle 35
	chain WANLOADBALANCE_PRE { # handle 1
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 24.29.97.0/24 ct state new counter packets 22 bytes 1848 jump ISP_eth1 # handle 19
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 24.29.97.0/24 counter packets 0 bytes 0 meta mark set ct mark # handle 20
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 163.182.128.0/24 ct state new counter packets 60 bytes 5040 jump ISP_eth0 # handle 21
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 163.182.128.0/24 counter packets 0 bytes 0 meta mark set ct mark # handle 22
		iifname "eth3" ct state new counter packets 0 bytes 0 jump ISP_eth0 # handle 23
		iifname "eth3" counter packets 0 bytes 0 meta mark set ct mark # handle 24
	}

	chain PREROUTING { # handle 3
		type filter hook prerouting priority mangle; policy accept;
		iifname "eth1" ct state new counter packets 0 bytes 0 jump ISP_eth1_IN # handle 18
		iifname "eth0" ct state new counter packets 0 bytes 0 jump ISP_eth0_IN # handle 11
		counter packets 1782 bytes 112240 jump WANLOADBALANCE_PRE # handle 4
	}

	chain ISP_eth0 { # handle 5
		counter packets 60 bytes 5040 ct mark set 0xc9  # handle 6
		counter packets 60 bytes 5040 meta mark set 0xc9  # handle 7
		counter packets 60 bytes 5040 accept # handle 8
	}

	chain ISP_eth0_IN { # handle 9
		counter packets 0 bytes 0 ct mark set 0xc9  # handle 10
	}

	chain ISP_eth1 { # handle 12
		counter packets 22 bytes 1848 ct mark set 0xca  # handle 13
		counter packets 22 bytes 1848 meta mark set 0xca  # handle 14
		counter packets 22 bytes 1848 accept # handle 15
	}

	chain ISP_eth1_IN { # handle 16
		counter packets 0 bytes 0 ct mark set 0xca  # handle 17
	}
}

however ,we can't see matches on ct mark/CONNMARK , this configuration is working without problems 1.3.1 . here is an example:

##output 1.3.1:  

table ip nat { # handle 8
	chain PREROUTING { # handle 1
		type nat hook prerouting priority dstnat; policy accept;
		counter packets 2 bytes 168 jump VYATTA_PRE_DNAT_HOOK # handle 7
	}

	chain INPUT { # handle 2
		type nat hook input priority 100; policy accept;
	}

	chain POSTROUTING { # handle 3
		type nat hook postrouting priority srcnat; policy accept;
		counter packets 2 bytes 168 jump VYATTA_PRE_SNAT_HOOK # handle 10
	}

	chain OUTPUT { # handle 4
		type nat hook output priority -100; policy accept;
	}

	chain VYATTA_PRE_DNAT_HOOK { # handle 5
		counter packets 2 bytes 168 return # handle 6
	}

	chain VYATTA_PRE_SNAT_HOOK { # handle 8
		counter packets 2 bytes 168 jump WANLOADBALANCE # handle 12
		counter packets 0 bytes 0 return # handle 9
	}

	chain WANLOADBALANCE { # handle 11
		ct mark 0xc9 counter packets 1 bytes 84 snat to 172.16.50.10 # handle 13
		ct mark 0xca counter packets 1 bytes 84 snat to 10.10.0.2 # handle 14
	}

table ip mangle { # handle 35
	chain WANLOADBALANCE_PRE { # handle 1
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 24.29.97.0/24 ct state new counter packets 22 bytes 1848 jump ISP_eth1 # handle 19
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 24.29.97.0/24 counter packets 0 bytes 0 meta mark set ct mark # handle 20
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 163.182.128.0/24 ct state new counter packets 60 bytes 5040 jump ISP_eth0 # handle 21
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 163.182.128.0/24 counter packets 0 bytes 0 meta mark set ct mark # handle 22
		iifname "eth3" ct state new counter packets 0 bytes 0 jump ISP_eth0 # handle 23
		iifname "eth3" counter packets 0 bytes 0 meta mark set ct mark # handle 24
	}

	chain PREROUTING { # handle 3
		type filter hook prerouting priority mangle; policy accept;
		iifname "eth1" ct state new counter packets 0 bytes 0 jump ISP_eth1_IN # handle 18
		iifname "eth0" ct state new counter packets 0 bytes 0 jump ISP_eth0_IN # handle 11
		counter packets 196 bytes 13908 jump WANLOADBALANCE_PRE # handle 4
	}

	chain ISP_eth0 { # handle 5
		counter packets 60 bytes 5040 ct mark set 0xc9  # handle 6
		counter packets 60 bytes 5040 meta mark set 0xc9  # handle 7
		counter packets 60 bytes 5040 accept # handle 8
	}

	chain ISP_eth0_IN { # handle 9
		counter packets 0 bytes 0 ct mark set 0xc9  # handle 10
	}

	chain ISP_eth1 { # handle 12
		counter packets 22 bytes 1848 ct mark set 0xca  # handle 13
		counter packets 22 bytes 1848 meta mark set 0xca  # handle 14
		counter packets 22 bytes 1848 accept # handle 15
	}

	chain ISP_eth1_IN { # handle 16
		counter packets 0 bytes 0 ct mark set 0xca  # handle 17

vyos@test2:~$ show wan-load-balance status
Chain WANLOADBALANCE_PRE (1 references)
 pkts bytes target     prot opt in     out     source               destination
    1    84 ISP_eth1   all  --  eth3   *       172.16.80.0/24       24.29.97.0/24        state NEW
    9   756 CONNMARK   all  --  eth3   *       172.16.80.0/24       24.29.97.0/24        CONNMARK restore
    1    84 ISP_eth0   all  --  eth3   *       172.16.80.0/24       163.182.128.0/24     state NEW
    9   756 CONNMARK   all  --  eth3   *       172.16.80.0/24       163.182.128.0/24     CONNMARK restore
    0     0 ISP_eth0   all  --  eth3   *       0.0.0.0/0            0.0.0.0/0            state NEW
   18  1512 CONNMARK   all  --  eth3   *       0.0.0.0/0            0.0.0.0/0            CONNMARK restore

i think it's problem with 1.4 mark packet , so it's not able work properly.

Details

Difficulty level
Unknown (require assessment)
Version
vyos-1.4-rolling-202204060217
Why the issue appeared?
Will be filled on close
Is it a breaking change?
Unspecified (possibly destroys the router)
Issue type
Bug (incorrect behavior)

Related Objects

Event Timeline

fernando created this task.Apr 11 2022, 1:07 PM
fernando added a comment.Apr 11 2022, 3:43 PM

regarding this behavior , I found a similar task where there was a bug with chain VYOS_PRE_SNAT_HOOK that jumps to WANLOADBALANCE ,although it was solved .
However , policy traffic rule seems to be affected with this issues(the main difference is that now there is a pass on this chain/ POSTROUTING) , below I'll share the task just to know where you can find the problem.

	chain POSTROUTING { # handle 2
		type nat hook postrouting priority srcnat; policy accept;
	}

https://phabricator.vyos.net/T4173

pasik added a subscriber: pasik.Apr 11 2022, 8:05 PM
n.fort mentioned this in T4365: NAT - Error on setting up tables.Apr 16 2022, 5:43 PM
masterit added a subscriber: masterit.Apr 19 2022, 2:18 PM
masterit mentioned this in T4443: Wan Load Balancing Multiple Regressions.May 25 2022, 11:27 PM
Viacheslav added a subscriber: Viacheslav.May 28 2022, 7:40 PM

@fernando Could you try to set sysctl mark?

sysctl -w net.ipv4.conf.eth0.src_valid_mark=1
sysctl -w net.ipv4.conf.eth1.src_valid_mark=1

https://www.kernel.org/doc/html/latest/networking/ip-sysctl.html

fernando closed this task as Resolved.Jun 16 2022, 10:30 PM
fernando claimed this task.

i've checked this issues, it seems to be solved . I think that it was solved for another task. I used the following vyos version :

fett@test-wan:~$ show version

Version:          VyOS 1.4-rolling-202206161834
Release train:    sagitta

Built by:         [email protected]
Built on:         Thu 16 Jun 2022 18:34 UTC
Build UUID:       773d79a0-8b4d-4fda-9b82-08f39d82e47c
Build commit ID:  180e2d33fccaff

Architecture:     x86_64
Boot via:         installed image
System type:      KVM guest

Hardware vendor:  QEMU
Hardware model:   Standard PC (i440FX + PIIX, 1996)
Hardware S/N:
Hardware UUID:    b037e2d5-9498-4344-863c-c9385c43c969

Copyright:        VyOS maintainers and contributors
`
fett@test-wan:~$  show wan status
Chain WANLOADBALANCE_PRE (1 references)
 pkts bytes target     prot opt in     out     source               destination
   17  1044 ISP_eth1   all  --  eth3   *       172.16.80.0/24       24.29.97.0/24        state NEW
    2   168 CONNMARK   all  --  eth3   *       172.16.80.0/24       24.29.97.0/24        CONNMARK restore
   31  2148 ISP_eth0   all  --  eth3   *       172.16.80.0/24       163.182.128.0/24     state NEW
    0     0 CONNMARK   all  --  eth3   *       172.16.80.0/24       163.182.128.0/24     CONNMARK restore
  252 31624 ISP_eth0   all  --  eth3   *       0.0.0.0/0            0.0.0.0/0            state NEW
    3   252 CONNMARK   all  --  eth3   *       0.0.0.0/0            0.0.0.0/0            CONNMARK restore




# mangle table and their machets :

table ip mangle { # handle 35
	chain WANLOADBALANCE_PRE { # handle 1
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 24.29.97.0/24 ct state new counter packets 17 bytes 1044 jump ISP_eth1 # handle 29
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 24.29.97.0/24 counter packets 2 bytes 168 meta mark set ct mark # handle 30
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 163.182.128.0/24 ct state new counter packets 22 bytes 1392 jump ISP_eth0 # handle 31
		iifname "eth3" ip saddr 172.16.80.0/24 ip daddr 163.182.128.0/24 counter packets 0 bytes 0 meta mark set ct mark # handle 32
		iifname "eth3" ct state new counter packets 238 bytes 30248 jump ISP_eth0 # handle 33
		iifname "eth3" counter packets 3 bytes 252 meta mark set ct mark # handle 34
	}

#basic test with vyos: 


    vyos@vyos:~$ ping 24.29.97.1
PING 24.29.97.1 (24.29.97.1) 56(84) bytes of data.
64 bytes from 24.29.97.1: icmp_seq=1 ttl=254 time=0.634 ms
64 bytes from 24.29.97.1: icmp_seq=2 ttl=254 time=1.06 ms
64 bytes from 24.29.97.1: icmp_seq=3 ttl=254 time=1.34 ms
^C
--- 24.29.97.1 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2003ms
rtt min/avg/max/mdev = 0.634/1.011/1.337/0.289 ms
vyos@vyos:~$ traceroute  24.29.97.1
traceroute to 24.29.97.1 (24.29.97.1), 30 hops max, 60 byte packets
 1  172.16.80.1 (172.16.80.1)  0.749 ms  0.682 ms  0.672 ms
 2  10.10.0.1 (10.10.0.1)  0.743 ms * *
vyos@vyos:~$
vyos@vyos:~$
vyos@vyos:~$ ping  163.182.128.50 interface eth0
PING 163.182.128.50 (163.182.128.50) from 172.16.80.10 eth0: 56(84) bytes of data.
^C
--- 163.182.128.50 ping statistics ---
3 packets transmitted, 0 received, 100% packet loss, time 2077ms

vyos@vyos:~$ traceroute 163.182.128.50
traceroute to 163.182.128.50 (163.182.128.50), 30 hops max, 60 byte packets
 1  172.16.80.1 (172.16.80.1)  0.392 ms  0.354 ms  0.315 ms
 2  172.16.50.1 (172.16.50.1)  0.647 ms  0.642 ms  0.620 ms
^C