Page MenuHomeVyOS Platform
Create Task
Maniphest T7829

VPP - ARP processing fails: No ARP responses sent, leading to incomplete entries and packet loss on external IP and internal IP
Open, HighPublicBUG
Actions

Description

Periodic packet loss on the external IP (100.200.60.11) and complete unavailability of the server from outside. ARP table shows incomplete entries for gateway (100.200.60.1) and neighbors in /28 subnet.
Tcpdump shows gateway sending "who-has" requests for server IP, but no responses from server.
Static ARP entries do not help.
VPP logs show "ARP requests out of buffer" error, suggesting buffer issues preventing ARP handling.

Hardware:

CPU: Intel(R) Xeon(R) E-2288G @ 3.70GHz, 8 physical cores (no HT).
Memory: 16 GB.
NIC: Mellanox MT27800 Family [ConnectX-5] (eth0 for external, eth1 for internal).

Current Configuration:

set firewall group network-group MANAGEMENT network '100.200.60.0/28'
set firewall group network-group MANAGEMENT network '10.31.31.0/24'
set firewall group network-group MANAGEMENT network '100.43.248.1/32'
set firewall group network-group MANAGEMENT network '100.43.248.32/27'
set firewall ipv4 input filter rule 5 action 'accept'
set firewall ipv4 input filter rule 5 description 'Allow input connection from trusted hosts'
set firewall ipv4 input filter rule 5 source group network-group 'MANAGEMENT'
set firewall ipv4 input filter rule 10 action 'jump'
set firewall ipv4 input filter rule 10 jump-target 'CONN_FILTER'
set firewall ipv4 input filter rule 20 action 'jump'
set firewall ipv4 input filter rule 20 destination port '22'
set firewall ipv4 input filter rule 20 jump-target 'VyOS_MANAGEMENT'
set firewall ipv4 input filter rule 20 protocol 'tcp'
set firewall ipv4 input filter rule 30 action 'accept'
set firewall ipv4 input filter rule 30 icmp type-name 'echo-request'
set firewall ipv4 input filter rule 30 protocol 'icmp'
set firewall ipv4 input filter rule 30 state 'new'
set firewall ipv4 input filter rule 50 action 'accept'
set firewall ipv4 input filter rule 50 source address '127.0.0.0/8'
set firewall ipv4 name CONN_FILTER default-action 'return'
set firewall ipv4 name CONN_FILTER rule 10 action 'accept'
set firewall ipv4 name CONN_FILTER rule 10 state 'established'
set firewall ipv4 name CONN_FILTER rule 10 state 'related'
set firewall ipv4 name CONN_FILTER rule 20 action 'drop'
set firewall ipv4 name CONN_FILTER rule 20 state 'invalid'
set firewall ipv4 name OUTSIDE-IN default-action 'drop'
set firewall ipv4 name VyOS_MANAGEMENT default-action 'drop'
set firewall ipv4 name VyOS_MANAGEMENT rule 20 action 'accept'
set firewall ipv4 name VyOS_MANAGEMENT rule 20 source group network-group 'MANAGEMENT'
set interfaces ethernet eth0 address '100.200.60.11/28'
set interfaces ethernet eth0 description 'CORE NETWORK'
set interfaces ethernet eth0 hw-id '98:03:9b:c4:a0:00'
set interfaces ethernet eth1 address '10.31.31.202/24'
set interfaces ethernet eth1 description 'BACKBONE-FOR-ROUTING'
set interfaces ethernet eth1 hw-id '98:03:9b:c4:a0:01'
set interfaces ethernet eth1 mtu '1500'
set interfaces loopback lo
set protocols ospf area 0.0.0.0 network '10.31.31.0/24'
set protocols ospf interface eth1
set protocols static route 0.0.0.0/0 next-hop 100.200.60.1
set protocols static route 200.120.251.0/24 blackhole
set protocols static route 200.120.251.0/24 description 'NAT-POOL'
set protocols static route 172.16.0.0/12 next-hop 10.31.31.1
set protocols static route 192.168.0.0/16 next-hop 10.31.31.1
set service monitoring zabbix-agent limits buffer-size '120'
set service monitoring zabbix-agent log debug-level 'warning'
set service monitoring zabbix-agent log size '1'
set service monitoring zabbix-agent server '100.200.60.3'
set service monitoring zabbix-agent server-active 100.200.60.3 port '10051'
set service ntp allow-client address '127.0.0.0/8'
set service ntp allow-client address '169.254.0.0/16'
set service ntp allow-client address '10.0.0.0/8'
set service ntp allow-client address '172.16.0.0/12'
set service ntp allow-client address '192.168.0.0/16'
set service ntp allow-client address '::1/128'
set service ntp allow-client address 'fe80::/10'
set service ntp allow-client address 'fc00::/7'
set service ntp listen-address '100.200.60.4'
set service ntp server 79.142.192.130
set service ntp server 91.236.251.24
set service ntp server 193.106.144.6
set service ntp server time1.vyos.net
set service ntp server time2.vyos.net
set service ntp server time3.vyos.net
set service snmp community soho2014 network '100.200.60.0/28'
set service snmp contact 'noc@isp.net.ua'
set service snmp location 'isp'
set service ssh port '22'
set system config-management commit-revisions '100'
set system console device ttyS0 speed '115200'
set system domain-name 'isp.net.ua'
set system domain-search 'isp.net.ua'
set system host-name 'nat2.isp.net.ua'
set system ip arp table-size '32768'
set system ip multipath layer4-hashing
set system ipv6 disable-forwarding
set system login timeout '600'
set system name-server '1.1.1.1'
set system name-server '8.8.8.8'
set system option kernel cpu disable-nmi-watchdog
set system option kernel cpu isolate-cpus '2-7'
set system option kernel cpu nohz-full '2-7'
set system option kernel cpu rcu-no-cbs '2-7'
set system option kernel disable-hpet
set system option kernel disable-mce
set system option kernel disable-mitigations
set system option kernel disable-power-saving
set system option kernel disable-softlockup
set system option kernel memory hugepage-size 1G hugepage-count '7'
set system option kernel memory hugepage-size 2M hugepage-count '3600'
set system option performance 'network-throughput'
set system option reboot-on-panic
set system option reboot-on-upgrade-failure '5'
set system option time-format '24-hour'
set system sysctl parameter net.core.default_qdisc value 'fq'
set system sysctl parameter net.core.optmem_max value '40960'
set system sysctl parameter net.core.rmem_default value '134217728'
set system sysctl parameter net.core.rmem_max value '536870912'
set system sysctl parameter net.core.wmem_default value '134217728'
set system sysctl parameter net.core.wmem_max value '536870912'
set system sysctl parameter net.ipv4.icmp_msgs_per_sec value '1000'
set system sysctl parameter net.ipv4.icmp_ratelimit value '1000'
set system sysctl parameter net.ipv4.icmp_ratemask value '4120'
set system sysctl parameter net.ipv4.tcp_congestion_control value 'bbr'
set system sysctl parameter net.ipv4.tcp_rmem value '65536 1048576 8388608'
set system sysctl parameter net.ipv4.tcp_wmem value '65536 1048576 8388608'
set system sysctl parameter net.netfilter.nf_conntrack_buckets value '4194304'
set system sysctl parameter net.netfilter.nf_conntrack_generic_timeout value '60'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_close value '10'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_close_wait value '20'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_established value '1800'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_fin_wait value '30'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_last_ack value '30'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_max_retrans value '300'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_syn_recv value '30'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_syn_sent value '60'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_time_wait value '60'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_unacknowledged value '300'
set system sysctl parameter net.netfilter.nf_conntrack_udp_timeout value '30'
set system sysctl parameter net.netfilter.nf_conntrack_udp_timeout_stream value '60'
set system syslog local facility all level 'info'
set system syslog local facility local7 level 'debug'
set system time-zone 'Europe/Kiev'
set vpp acl ip interface eth1 input acl-tag 10 tag-name 'DENY'
set vpp acl ip tag-name DENY description 'DENY SOME PORT'
set vpp acl ip tag-name DENY rule 10 action 'permit'
set vpp acl ip tag-name DENY rule 10 description 'ALLOW BACKBONE'
set vpp acl ip tag-name DENY rule 10 destination prefix '10.31.31.0/24'
set vpp acl ip tag-name DENY rule 110 action 'deny'
set vpp acl ip tag-name DENY rule 110 destination port '25'
set vpp acl ip tag-name DENY rule 110 protocol 'tcp'
set vpp acl ip tag-name DENY rule 120 action 'deny'
set vpp acl ip tag-name DENY rule 120 destination prefix '0.0.0.0/8'
set vpp acl ip tag-name DENY rule 120 protocol 'all'
set vpp acl ip tag-name DENY rule 130 action 'deny'
set vpp acl ip tag-name DENY rule 130 destination prefix '172.16.0.0/12'
set vpp acl ip tag-name DENY rule 140 action 'deny'
set vpp acl ip tag-name DENY rule 140 destination prefix '192.168.0.0/16'
set vpp acl ip tag-name DENY rule 150 action 'deny'
set vpp acl ip tag-name DENY rule 150 destination prefix '10.0.0.0/8'
set vpp acl ip tag-name DENY rule 160 action 'deny'
set vpp acl ip tag-name DENY rule 160 destination prefix '169.254.0.0/16'
set vpp acl ip tag-name DENY rule 170 action 'deny'
set vpp acl ip tag-name DENY rule 170 destination prefix '127.0.0.0/8'
set vpp acl ip tag-name DENY rule 180 action 'deny'
set vpp acl ip tag-name DENY rule 180 destination prefix '198.18.0.0/15'
set vpp acl ip tag-name DENY rule 190 action 'deny'
set vpp acl ip tag-name DENY rule 190 destination prefix '192.0.0.0/24'
set vpp acl ip tag-name DENY rule 200 action 'deny'
set vpp acl ip tag-name DENY rule 200 destination prefix '192.0.2.0/24'
set vpp acl ip tag-name DENY rule 210 action 'deny'
set vpp acl ip tag-name DENY rule 210 destination prefix '198.51.100.0/24'
set vpp acl ip tag-name DENY rule 220 action 'deny'
set vpp acl ip tag-name DENY rule 220 destination prefix '203.0.113.0/24'
set vpp acl ip tag-name DENY rule 230 action 'deny'
set vpp acl ip tag-name DENY rule 230 destination prefix '100.64.0.0/10'
set vpp acl ip tag-name DENY rule 240 action 'deny'
set vpp acl ip tag-name DENY rule 240 destination prefix '192.88.99.0/24'
set vpp acl ip tag-name DENY rule 250 action 'deny'
set vpp acl ip tag-name DENY rule 250 destination prefix '255.255.255.255/32'
set vpp acl ip tag-name DENY rule 1000 action 'permit'
set vpp nat44 address-pool translation address '200.120.251.0-200.120.251.255'
set vpp nat44 exclude rule 10 external-interface 'eth0'
set vpp nat44 exclude rule 10 local-port '22'
set vpp nat44 exclude rule 10 protocol 'tcp'
set vpp nat44 exclude rule 15 external-interface 'eth1'
set vpp nat44 exclude rule 15 local-port '22'
set vpp nat44 exclude rule 15 protocol 'tcp'
set vpp nat44 exclude rule 20 local-address '100.200.60.11'
set vpp nat44 exclude rule 20 protocol 'all'
set vpp nat44 exclude rule 30 local-address '10.31.31.202'
set vpp nat44 exclude rule 30 protocol 'all'
set vpp nat44 interface inside 'eth1'
set vpp nat44 interface outside 'eth0'
set vpp settings buffers page-size 'default-hugepage'
set vpp settings cpu corelist-workers '3'
set vpp settings cpu corelist-workers '4'
set vpp settings cpu corelist-workers '5'
set vpp settings cpu corelist-workers '6'
set vpp settings cpu corelist-workers '7'
set vpp settings cpu main-core '2'
set vpp settings interface eth0 driver 'dpdk'
set vpp settings interface eth0 rx-mode 'polling'
set vpp settings interface eth1 driver 'dpdk'
set vpp settings interface eth1 rx-mode 'polling'
set vpp settings lcp netlink rx-buffer-size '536870912'
set vpp settings logging default-log-level 'alert'
set vpp settings memory main-heap-page-size 'default-hugepage'
set vpp settings memory main-heap-size '4G'
set vpp settings nat44 no-forwarding
set vpp settings nat44 session-limit '30000000'
set vpp settings nat44 timeout icmp '60'
set vpp settings nat44 timeout tcp-established '1800'
set vpp settings nat44 timeout udp '30'
set vpp settings physmem max-size '12G'
set vpp settings statseg page-size 'default-hugepage'
set vpp settings statseg size '256M'
set vpp settings unix poll-sleep-usec '10

Steps to Reproduce

Configure VPP with DPDK on eth0 (external interface) and NAT44.
Set static ARP for gateway 100.200.60.1.
Monitor ARP table: arp -an — shows incomplete entries.
Use tcpdump:

sudo tcpdump -i eth0 arp -vv -n

during outage — gateway sends "who-has 100.200.60.11 tell 100.200.60.1", but no reply from server MAC (98:03:9b:c4:a0:00).
After some time (variable, e.g., 10-12 hours of operation), external IP becomes unreachable with packet loss, after some time (15-20 hours) internal IP becomes unreachable with packet loss.

Some logs:

arp -an

? (10.31.31.201) at <incomplete> on eth1
? (10.31.31.1) at <incomplete> on eth1
? (10.31.31.250) at <incomplete> on eth1
? (10.31.31.101) at <incomplete> on eth1
? (100.200.60.1) at <incomplete> on eth0
? (10.31.31.245) at <incomplete> on eth1
? (10.31.31.253) at <incomplete> on eth1
? (100.200.60.3) at <incomplete> on eth0
? (10.31.31.254) at <incomplete> on eth1

ip a

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 fe80::200:ff:fe00:0/64 scope link 
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host noprefixroute 
       valid_lft forever preferred_lft forever
2: defunct_eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1496 qdisc mq state UP group default qlen 1000
    link/ether 98:03:9b:c4:a0:00 brd ff:ff:ff:ff:ff:ff
    altname enp1s0f0np0
3: defunct_eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1496 qdisc mq state UP group default qlen 1000
    link/ether 98:03:9b:c4:a0:01 brd ff:ff:ff:ff:ff:ff
    altname enp1s0f1np1
4: pim6reg@NONE: <NOARP,UP,LOWER_UP> mtu 1452 qdisc noqueue state UNKNOWN group default qlen 1000
    link/pimreg 
5: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UNKNOWN group default qlen 1000
    link/ether 98:03:9b:c4:a0:00 brd ff:ff:ff:ff:ff:ff
    inet 100.200.60.11/28 brd 100.200.60.15 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::9a03:9bff:fec4:a000/64 scope link 
       valid_lft forever preferred_lft forever
6: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UNKNOWN group default qlen 1000
    link/ether 98:03:9b:c4:a0:01 brd ff:ff:ff:ff:ff:ff
    inet 10.31.31.202/24 brd 10.31.31.255 scope global eth1
       valid_lft forever preferred_lft forever
    inet6 fe80::9a03:9bff:fec4:a001/64 scope link 
       valid_lft forever preferred_lft forever

ip l 

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: defunct_eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1496 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 98:03:9b:c4:a0:00 brd ff:ff:ff:ff:ff:ff
    altname enp1s0f0np0
3: defunct_eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1496 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 98:03:9b:c4:a0:01 brd ff:ff:ff:ff:ff:ff
    altname enp1s0f1np1
4: pim6reg@NONE: <NOARP,UP,LOWER_UP> mtu 1452 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/pimreg 
5: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UNKNOWN mode DEFAULT group default qlen 1000
    link/ether 98:03:9b:c4:a0:00 brd ff:ff:ff:ff:ff:ff
    alias CORE NETWORK
6: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UNKNOWN mode DEFAULT group default qlen 1000
    link/ether 98:03:9b:c4:a0:01 brd ff:ff:ff:ff:ff:ff
    alias BACKBONE-FOR-ROUTING

vpp show error

   Count                  Node                              Reason               Severity 
   1998364    nat44-out2in-worker-handoff                same worker               error  
   1894518    nat44-out2in-worker-handoff                 do handoff               error  
         2      nat44-ed-out2in-slowpath            unsupported ICMP type          error  
   1501242      nat44-ed-out2in-slowpath                no translation             error  
         2          nat44-ed-out2in                 unsupported ICMP type          error  
       507             dpdk-input                          no error                error  
     70026             arp-reply                       ARP replies sent            info   
      1529             arp-reply             ARP request IP4 source address lear   info   
3220524579            virtio-input                    buffer alloc error           error  
         1          ipsec4-tun-input                  no matching tunnel           error  
         6             ip4-local                    ip4 source lookup miss         error  
         9             ip4-local                       bad tcp checksum            error  
        52           ip6-icmp-input           neighbor discovery not configured    error  
        18           ip4-icmp-error            hop limit exceeded response sent    info   
         1           ip4-icmp-error                 error message dropped          error  
     20884             llc-input                    unknown llc ssap/dsap          error  
        12           ethernet-input                      unknown vlan              error  
         1           punt-dispatch                        dispatched               error  
     42717    nat44-in2out-worker-handoff                same worker               error  
     16813    nat44-in2out-worker-handoff                 do handoff               error  
    968592      nat44-ed-out2in-slowpath                no translation             error  
     15452             dpdk-input                          no error                error  
      1410        acl-plugin-in-ip4-fa                 ACL deny packets            error  
    122604        acl-plugin-in-ip4-fa                ACL permit packets           error  
     81462        acl-plugin-in-ip4-fa                 checked packets             error  
     30250             arp-reply                       ARP replies sent            info   
     15575             arp-reply             ARP request IP4 source address lear   info   
3421305418            virtio-input                    buffer alloc error           error  
         7             ip4-glean                    ARP requests throttled         info   
         1             ip4-glean                  ARP requests out of buffer       error  
        11             ip4-glean                      ARP requests sent            info   
     10262     ip4-sv-reassembly-feature           unsupported ip protocol         error  
        58     ip4-local-full-reassembly           successful reassemblies         info   
       116     ip4-local-full-reassembly            fragments reassembled          info   
       116     ip4-local-full-reassembly              fragments received           info   
       286             ip4-local             ip4 spoofed local-address packet dr   error  
     42237           ip4-icmp-error            hop limit exceeded response sent    info   
       384           ip4-icmp-error                 error message dropped          error  
      1281             snap-input                 unknown oui/snap protocol        error  
     37287             llc-input                    unknown llc ssap/dsap          error  
    613086      nat44-ed-out2in-slowpath                no translation             error  
        30             dpdk-input                          no error                error  
4064541190            virtio-input                    buffer alloc error           error  
         1             ip4-local                       bad tcp checksum            error  
        83             ip4-local             ip4 spoofed local-address packet dr   error  
        30           ip4-icmp-error            hop limit exceeded response sent    info   
    154741      nat44-ed-out2in-slowpath                no translation             error  
        88             dpdk-input                          no error                error  
1634456713            virtio-input                    buffer alloc error           error  
         2              ip4-arp                       ARP requests sent            info   
        85           ip4-icmp-error            hop limit exceeded response sent    info   
         3           ip4-icmp-error                 error message dropped          error  
    144410      nat44-ed-out2in-slowpath                no translation             error  
        24             dpdk-input                          no error                error  
         1        acl-plugin-in-ip4-fa                ACL permit packets           error  
6539165785            virtio-input                    buffer alloc error           error  
         4             ip4-local                       bad tcp checksum            error  
        24           ip4-icmp-error            hop limit exceeded response sent    info   
         2            eth0-output                     interface is down            error

Details

Version
2025.08.28-0019-rolling
Is it a breaking change?
Unspecified (possibly destroys the router)
Issue type
Bug (incorrect behavior)

Related Objects
Search...

Event Timeline

pautiina created this task.Sep 16 2025, 9:44 AM
pasik subscribed.Sep 16 2025, 10:28 AM
Viacheslav added a parent task: T7070: VPP related bugs the root task.Sep 16 2025, 1:45 PM
bbabich subscribed.Sep 17 2025, 8:15 AM
Unknown Object (User) assigned this task to Viacheslav.Sep 17 2025, 10:57 AM
Unknown Object (User) triaged this task as High priority.
pautiina added a comment.Sep 19 2025, 1:08 PM

Updated configs and some logs:

set system option kernel cpu disable-nmi-watchdog
set system option kernel cpu isolate-cpus '2-7'
set system option kernel cpu nohz-full '2-7'
set system option kernel cpu rcu-no-cbs '2-7'
set system option kernel disable-hpet
set system option kernel disable-mce
set system option kernel disable-mitigations
set system option kernel disable-power-saving
set system option kernel disable-softlockup
set system option kernel memory hugepage-size 2M hugepage-count '5000'
set system sysctl parameter net.core.default_qdisc value 'fq'
set system sysctl parameter net.core.optmem_max value '40960'
set system sysctl parameter net.core.rmem_default value '134217728'
set system sysctl parameter net.core.rmem_max value '536870912'
set system sysctl parameter net.core.wmem_default value '134217728'
set system sysctl parameter net.core.wmem_max value '536870912'
set system sysctl parameter net.ipv4.icmp_msgs_per_sec value '1000'
set system sysctl parameter net.ipv4.icmp_ratelimit value '1000'
set system sysctl parameter net.ipv4.icmp_ratemask value '4120'
set system sysctl parameter net.ipv4.tcp_congestion_control value 'bbr'
set system sysctl parameter net.ipv4.tcp_rmem value '65536 1048576 8388608'
set system sysctl parameter net.ipv4.tcp_wmem value '65536 1048576 8388608'
set system sysctl parameter net.netfilter.nf_conntrack_buckets value '4194304'
set system sysctl parameter net.netfilter.nf_conntrack_generic_timeout value '60'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_close value '10'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_close_wait value '20'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_established value '1800'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_fin_wait value '30'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_last_ack value '30'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_max_retrans value '300'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_syn_recv value '30'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_syn_sent value '60'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_time_wait value '60'
set system sysctl parameter net.netfilter.nf_conntrack_tcp_timeout_unacknowledged value '300'
set system sysctl parameter net.netfilter.nf_conntrack_udp_timeout value '30'
set system sysctl parameter net.netfilter.nf_conntrack_udp_timeout_stream value '60'
set vpp acl ip interface eth1 input acl-tag 10 tag-name 'DENY'
set vpp acl ip tag-name DENY description 'DENY SOME PORT'
set vpp acl ip tag-name DENY rule 10 action 'permit'
set vpp acl ip tag-name DENY rule 10 description 'ALLOW BACKBONE'
set vpp acl ip tag-name DENY rule 10 destination prefix '10.31.31.0/24'
set vpp acl ip tag-name DENY rule 110 action 'deny'
set vpp acl ip tag-name DENY rule 110 destination port '25'
set vpp acl ip tag-name DENY rule 110 protocol 'tcp'
set vpp acl ip tag-name DENY rule 120 action 'deny'
set vpp acl ip tag-name DENY rule 120 destination prefix '0.0.0.0/8'
set vpp acl ip tag-name DENY rule 120 protocol 'all'
set vpp acl ip tag-name DENY rule 130 action 'deny'
set vpp acl ip tag-name DENY rule 130 destination prefix '172.16.0.0/12'
set vpp acl ip tag-name DENY rule 140 action 'deny'
set vpp acl ip tag-name DENY rule 140 destination prefix '192.168.0.0/16'
set vpp acl ip tag-name DENY rule 150 action 'deny'
set vpp acl ip tag-name DENY rule 150 destination prefix '10.0.0.0/8'
set vpp acl ip tag-name DENY rule 160 action 'deny'
set vpp acl ip tag-name DENY rule 160 destination prefix '169.254.0.0/16'
set vpp acl ip tag-name DENY rule 170 action 'deny'
set vpp acl ip tag-name DENY rule 170 destination prefix '127.0.0.0/8'
set vpp acl ip tag-name DENY rule 180 action 'deny'
set vpp acl ip tag-name DENY rule 180 destination prefix '198.18.0.0/15'
set vpp acl ip tag-name DENY rule 190 action 'deny'
set vpp acl ip tag-name DENY rule 190 destination prefix '192.0.0.0/24'
set vpp acl ip tag-name DENY rule 200 action 'deny'
set vpp acl ip tag-name DENY rule 200 destination prefix '192.0.2.0/24'
set vpp acl ip tag-name DENY rule 210 action 'deny'
set vpp acl ip tag-name DENY rule 210 destination prefix '198.51.100.0/24'
set vpp acl ip tag-name DENY rule 220 action 'deny'
set vpp acl ip tag-name DENY rule 220 destination prefix '203.0.113.0/24'
set vpp acl ip tag-name DENY rule 230 action 'deny'
set vpp acl ip tag-name DENY rule 230 destination prefix '100.64.0.0/10'
set vpp acl ip tag-name DENY rule 240 action 'deny'
set vpp acl ip tag-name DENY rule 240 destination prefix '192.88.99.0/24'
set vpp acl ip tag-name DENY rule 250 action 'deny'
set vpp acl ip tag-name DENY rule 250 destination prefix '255.255.255.255/32'
set vpp acl ip tag-name DENY rule 1000 action 'permit'
set vpp nat44 address-pool translation address '200.120.251.0-200.120.251.255'
set vpp nat44 exclude rule 10 external-interface 'eth0'
set vpp nat44 exclude rule 10 local-port '22'
set vpp nat44 exclude rule 10 protocol 'tcp'
set vpp nat44 exclude rule 15 external-interface 'eth1'
set vpp nat44 exclude rule 15 local-port '22'
set vpp nat44 exclude rule 15 protocol 'tcp'
set vpp nat44 exclude rule 20 local-address '100.200.60.11'
set vpp nat44 exclude rule 20 protocol 'all'
set vpp nat44 exclude rule 30 local-address '10.31.31.202'
set vpp nat44 exclude rule 30 protocol 'all'
set vpp nat44 interface inside 'eth1'
set vpp nat44 interface outside 'eth0'
set vpp settings buffers page-size '2M'
set vpp settings cpu corelist-workers '3'
set vpp settings cpu corelist-workers '4'
set vpp settings cpu corelist-workers '5'
set vpp settings cpu corelist-workers '6'
set vpp settings cpu corelist-workers '7'
set vpp settings cpu main-core '2'
set vpp settings interface eth0 driver 'dpdk'
set vpp settings interface eth0 rx-mode 'polling'
set vpp settings interface eth1 driver 'dpdk'
set vpp settings interface eth1 rx-mode 'polling'
set vpp settings lcp netlink rx-buffer-size '536870912'
set vpp settings logging default-log-level 'alert'
set vpp settings memory main-heap-page-size '2M'
set vpp settings memory main-heap-size '8000M'
set vpp settings nat44 no-forwarding
set vpp settings nat44 session-limit '30000000'
set vpp settings nat44 timeout icmp '60'
set vpp settings nat44 timeout tcp-established '1800'
set vpp settings nat44 timeout udp '30'
set vpp settings statseg page-size '2M'
set vpp settings statseg size '1024M'
Thread 0 vpp_main
  base 0x7f1c7c400000, size 7.81g, locked, unmap-on-destroy, name 'main heap'
    page stats: page-size 2M, total 4000, mapped 4000, not-mapped 0
      numa 0: 4000 pages, 7.81g bytes
    total: 7.81G, used: 637.92M, free: 7.19G, trimmable: 7.19G

Thread 1 vpp_wk_0
  base 0x7f1c7c400000, size 7.81g, locked, unmap-on-destroy, name 'main heap'
    page stats: page-size 2M, total 4000, mapped 4000, not-mapped 0
      numa 0: 4000 pages, 7.81g bytes
    total: 7.81G, used: 637.92M, free: 7.19G, trimmable: 7.19G

Thread 2 vpp_wk_1
  base 0x7f1c7c400000, size 7.81g, locked, unmap-on-destroy, name 'main heap'
    page stats: page-size 2M, total 4000, mapped 4000, not-mapped 0
      numa 0: 4000 pages, 7.81g bytes
    total: 7.81G, used: 637.92M, free: 7.19G, trimmable: 7.19G

Thread 3 vpp_wk_2
  base 0x7f1c7c400000, size 7.81g, locked, unmap-on-destroy, name 'main heap'
    page stats: page-size 2M, total 4000, mapped 4000, not-mapped 0
      numa 0: 4000 pages, 7.81g bytes
    total: 7.81G, used: 637.92M, free: 7.19G, trimmable: 7.19G

Thread 4 vpp_wk_3
  base 0x7f1c7c400000, size 7.81g, locked, unmap-on-destroy, name 'main heap'
    page stats: page-size 2M, total 4000, mapped 4000, not-mapped 0
      numa 0: 4000 pages, 7.81g bytes
    total: 7.81G, used: 637.92M, free: 7.19G, trimmable: 7.19G

Thread 5 vpp_wk_4
  base 0x7f1c7c400000, size 7.81g, locked, unmap-on-destroy, name 'main heap'
    page stats: page-size 2M, total 4000, mapped 4000, not-mapped 0
      numa 0: 4000 pages, 7.81g bytes
    total: 7.81G, used: 637.92M, free: 7.19G, trimmable: 7.19G
ID     Name                Type        LWP     Sched Policy (Priority)  lcore  Core   Socket State     
0      vpp_main                        2013    other (0)                2      2      0                
1      vpp_wk_0            workers     2025    other (0)                3      3      0                
2      vpp_wk_1            workers     2026    other (0)                4      4      0                
3      vpp_wk_2            workers     2027    other (0)                5      5      0                
4      vpp_wk_3            workers     2028    other (0)                6      6      0                
5      vpp_wk_4            workers     2029    other (0)                7      7      0
   Count                  Node                              Reason               Severity 
   1952388    nat44-out2in-worker-handoff                same worker               error  
    955264    nat44-out2in-worker-handoff                 do handoff               error  
         2      nat44-ed-out2in-slowpath            unsupported ICMP type          error  
   1631147      nat44-ed-out2in-slowpath                no translation             error  
       228             dpdk-input                          no error                error  
     76178             arp-reply                       ARP replies sent            info   
      1219             arp-reply             ARP request IP4 source address lear   info   
2453051957            virtio-input                    buffer alloc error           error  
         1          ipsec4-tun-input                  no matching tunnel           error  
         6             ip4-local                       bad tcp checksum            error  
        50           ip6-icmp-input           neighbor discovery not configured    error  
        27           ip4-icmp-error            hop limit exceeded response sent    info   
     20939             llc-input                    unknown llc ssap/dsap          error  
        13           ethernet-input                      unknown vlan              error  
         1           punt-dispatch                     No registrations            error  
     29476    nat44-in2out-worker-handoff                same worker               error  
     10126    nat44-in2out-worker-handoff                 do handoff               error  
    209371      nat44-ed-out2in-slowpath                no translation             error  
     10151             dpdk-input                          no error                error  
       845        acl-plugin-in-ip4-fa                 ACL deny packets            error  
     86612        acl-plugin-in-ip4-fa                ACL permit packets           error  
     58091        acl-plugin-in-ip4-fa                 checked packets             error  
     21915             arp-reply                       ARP replies sent            info   
     12962             arp-reply             ARP request IP4 source address lear   info   
6407161231            virtio-input                    buffer alloc error           error  
         2             ip4-glean                      ARP requests sent            info   
      8822     ip4-sv-reassembly-feature           unsupported ip protocol         error  
        34     ip4-local-full-reassembly           successful reassemblies         info   
        68     ip4-local-full-reassembly            fragments reassembled          info   
        68     ip4-local-full-reassembly              fragments received           info   
       108             ip4-local             ip4 spoofed local-address packet dr   error  
     28891           ip4-icmp-error            hop limit exceeded response sent    info   
       499           ip4-icmp-error                 error message dropped          error  
       845             snap-input                 unknown oui/snap protocol        error  
     24602             llc-input                    unknown llc ssap/dsap          error  
    208415      nat44-ed-out2in-slowpath                no translation             error  
        22             dpdk-input                          no error                error  
        22             ip4-local             ip4 spoofed local-address packet dr   error  
        22           ip4-icmp-error            hop limit exceeded response sent    info   
    328819      nat44-ed-out2in-slowpath                no translation             error  
        22             dpdk-input                          no error                error  
        22           ip4-icmp-error            hop limit exceeded response sent    info   
    203294      nat44-ed-out2in-slowpath                no translation             error  
        14             dpdk-input                          no error                error  
        30             ip4-local             ip4 spoofed local-address packet dr   error  
        14           ip4-icmp-error            hop limit exceeded response sent    info
               total        used        free      shared  buff/cache   available
Mem:            15Gi        11Gi       4.1Gi       4.1Mi       707Mi       4.5Gi
Swap:             0B          0B          0B
pautiina added a comment.EditedOct 28 2025, 4:52 PM

Hello.

I'm delete option

delete vpp settings nat44 no-forwarding

and 5 days all normal with arp.

Viacheslav added a comment.Oct 28 2025, 5:18 PM

@pautiina nice catch! Unexpected and not clear workaround.