VPP IPsec traffic stops working after the reboot.
IPsec is configured between `vpp-left` and `vpp-right` nodes, topology:
{F16078902}
`vpp-left` config:
```
set interfaces ethernet eth0 address '10.0.0.1/30'
set interfaces ethernet eth0 address '192.168.122.101/24'
set interfaces ethernet eth0 description 'WAN'
set interfaces ethernet eth1 address '100.64.1.1/24'
set interfaces ethernet eth1 description 'IPsec-client-left'
set system host-name 'vpp-left'
set system option kernel memory default-hugepage-size '2M'
set system option kernel memory hugepage-size 2M hugepage-count '1800'
set vpn ipsec authentication psk PSK id '10.0.0.1'
set vpn ipsec authentication psk PSK id '10.0.0.2'
set vpn ipsec authentication psk PSK secret '1234567890'
set vpn ipsec esp-group ESP-group lifetime '3600'
set vpn ipsec esp-group ESP-group mode 'tunnel'
set vpn ipsec esp-group ESP-group pfs 'enable'
set vpn ipsec esp-group ESP-group proposal 1 encryption 'aes256'
set vpn ipsec esp-group ESP-group proposal 1 hash 'sha1'
set vpn ipsec ike-group IKE-group key-exchange 'ikev2'
set vpn ipsec ike-group IKE-group lifetime '28800'
set vpn ipsec ike-group IKE-group proposal 1 dh-group '14'
set vpn ipsec ike-group IKE-group proposal 1 encryption 'aes256'
set vpn ipsec ike-group IKE-group proposal 1 hash 'sha1'
set vpn ipsec interface 'eth0'
set vpn ipsec site-to-site peer RIGHT authentication local-id '10.0.0.1'
set vpn ipsec site-to-site peer RIGHT authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer RIGHT authentication remote-id '10.0.0.2'
set vpn ipsec site-to-site peer RIGHT connection-type 'initiate'
set vpn ipsec site-to-site peer RIGHT ike-group 'IKE-group'
set vpn ipsec site-to-site peer RIGHT local-address '10.0.0.1'
set vpn ipsec site-to-site peer RIGHT remote-address '10.0.0.2'
set vpn ipsec site-to-site peer RIGHT tunnel 0 esp-group 'ESP-group'
set vpn ipsec site-to-site peer RIGHT tunnel 0 local prefix '100.64.1.0/24'
set vpn ipsec site-to-site peer RIGHT tunnel 0 remote prefix '100.64.2.0/24'
set vpp settings interface eth0 driver 'dpdk'
set vpp settings interface eth1 driver 'dpdk'
set vpp settings ipsec
set vpp settings unix poll-sleep-usec '120'
```
`vyos-right` config:
```
set interfaces ethernet eth0 address '10.0.0.2/30'
set interfaces ethernet eth0 address '192.168.122.102/24'
set interfaces ethernet eth0 description 'WAN'
set interfaces ethernet eth1 address '100.64.2.1/24'
set interfaces ethernet eth1 description 'IPsec-client-right'
set system host-name 'vpp-right'
set system option kernel memory default-hugepage-size '2M'
set system option kernel memory hugepage-size 2M hugepage-count '1800'
set vpn ipsec authentication psk PSK id '10.0.0.1'
set vpn ipsec authentication psk PSK id '10.0.0.2'
set vpn ipsec authentication psk PSK secret '1234567890'
set vpn ipsec esp-group ESP-group lifetime '3600'
set vpn ipsec esp-group ESP-group mode 'tunnel'
set vpn ipsec esp-group ESP-group pfs 'enable'
set vpn ipsec esp-group ESP-group proposal 1 encryption 'aes256'
set vpn ipsec esp-group ESP-group proposal 1 hash 'sha1'
set vpn ipsec ike-group IKE-group key-exchange 'ikev2'
set vpn ipsec ike-group IKE-group lifetime '28800'
set vpn ipsec ike-group IKE-group proposal 1 dh-group '14'
set vpn ipsec ike-group IKE-group proposal 1 encryption 'aes256'
set vpn ipsec ike-group IKE-group proposal 1 hash 'sha1'
set vpn ipsec interface 'eth0'
set vpn ipsec site-to-site peer LEFT authentication local-id '10.0.0.2'
set vpn ipsec site-to-site peer LEFT authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer LEFT authentication remote-id '10.0.0.1'
set vpn ipsec site-to-site peer LEFT connection-type 'respond'
set vpn ipsec site-to-site peer LEFT ike-group 'IKE-group'
set vpn ipsec site-to-site peer LEFT local-address '10.0.0.2'
set vpn ipsec site-to-site peer LEFT remote-address '10.0.0.1'
set vpn ipsec site-to-site peer LEFT tunnel 0 esp-group 'ESP-group'
set vpn ipsec site-to-site peer LEFT tunnel 0 local prefix '100.64.2.0/24'
set vpn ipsec site-to-site peer LEFT tunnel 0 remote prefix '100.64.1.0/24'
set vpp settings interface eth0 driver 'dpdk'
set vpp settings interface eth1 driver 'dpdk'
set vpp settings ipsec
set vpp settings unix poll-sleep-usec '120'
```
`ipsec-c-left` client configuration:
```
set interfaces ethernet eth0 address '100.64.1.10/24'
set protocols static route 0.0.0.0/0 next-hop 100.64.1.1
set system host-name 'ipsec-c-left'
```
`ipsec-c-right` client configuration:
```
set interfaces ethernet eth0 address '100.64.2.10/24'
set protocols static route 0.0.0.0/0 next-hop 100.64.2.1
set system host-name 'ipsec-c-right'
```
---
## Steps to reproduce:
After the reboot (both VPP routers), IPsec traffic dropped, though IPsec SAs were established
Check on the router `vpp-right` and we can see SA established and tunnel up
```
vyos@vpp-right:~$ show vpn ipsec sa
Connection State Uptime Bytes In/Out Packets In/Out Remote address Remote ID Proposal
------------- ------- -------- -------------- ---------------- ---------------- ----------- ------------------------
LEFT-tunnel-0 up 5m26s 0B/0B 0/0 10.0.0.1 10.0.0.1 AES_CBC_256/HMAC_SHA1_96
vyos@vpp-right:~$
vyos@vpp-right:~$ sudo vppctl show ipsec sa
[0] sa 3442356361 (0xcd2e3089) spi 3475963309 (0xcf2efdad) protocol:esp flags:[anti-replay tunnel Protect inbound ]
[1] sa 3472657337 (0xcefc8bb9) spi 3489416527 (0xcffc454f) protocol:esp flags:[tunnel Protect no-algo-no-drop ]
vyos@vpp-right:~$
```
1. On the router `ipsec-c-left`, start ping the remote host:
```
vyos@ipsec-c-left:~$ ping 100.64.2.10
PING 100.64.2.10 (100.64.2.10) 56(84) bytes of data.
```
2. Check trace on the router `vpp-right` IPsec packets dropped
```
vpp# trace add dpdk-input 5
vpp# show trace
------------------- Start of thread 0 vpp_main -------------------
Packet 1
00:10:18:347828: dpdk-input
eth0 rx queue 0
buffer 0x94b11: current data 0, length 166, buffer-pool 0, ref-count 1, trace handle 0x0
ext-hdr-valid
PKT MBUF: port 0, nb_segs 1, pkt_len 166
buf_len 2176, data_len 166, ol_flags 0x0, data_off 128, phys_addr 0xe152c4c0
packet_type 0x0 l2_len 0 l3_len 0 outer_l2_len 0 outer_l3_len 0
rss 0x0 fdir.hi 0x0 fdir.lo 0x0
IP4: 0c:c0:19:43:00:00 -> 0c:84:b5:89:00:00
IPSEC_ESP: 10.0.0.1 -> 10.0.0.2
tos 0x00, ttl 254, length 152, checksum 0xa831 dscp CS0 ecn NON_ECN
fragment id 0x0000
00:10:18:348018: ethernet-input
frame: flags 0x1, hw-if-index 1, sw-if-index 1
IP4: 0c:c0:19:43:00:00 -> 0c:84:b5:89:00:00
00:10:18:348025: ip4-input
IPSEC_ESP: 10.0.0.1 -> 10.0.0.2
tos 0x00, ttl 254, length 152, checksum 0xa831 dscp CS0 ecn NON_ECN
fragment id 0x0000
00:10:18:348030: ip4-lookup
fib 0 dpo-idx 12 flow hash: 0x00000000
IPSEC_ESP: 10.0.0.1 -> 10.0.0.2
tos 0x00, ttl 254, length 152, checksum 0xa831 dscp CS0 ecn NON_ECN
fragment id 0x0000
00:10:18:348032: ip4-receive
fib:0 adj:12 flow:0x00000000
IPSEC_ESP: 10.0.0.1 -> 10.0.0.2
tos 0x00, ttl 254, length 152, checksum 0xa831 dscp CS0 ecn NON_ECN
fragment id 0x0000
00:10:18:348035: ipsec4-tun-input
IPSec: remote:10.0.0.1 spi:3475963309 (0xcf2efdad) sa:0 tun:0 seq 219
00:10:18:348036: esp4-decrypt-tun
esp: crypto aes-cbc-256 integrity sha1-96 pkt-seq 219 sa-seq 219 pkt-seq-hi 0
00:10:18:348049: ip4-drop
fib:0 adj:32 flow:0x00000000
ICMP: 100.64.1.10 -> 100.64.2.10
tos 0x00, ttl 63, length 84, checksum 0xf590 dscp CS0 ecn NON_ECN
fragment id 0x7a84, flags DONT_FRAGMENT
ICMP echo_request checksum 0x717f id 2574
00:10:18:348050: error-drop
rx:ipsec1
00:10:18:348051: drop
esp4-decrypt-tun: no tunnel protocol
```
## To get it working
To get it working, reconfigure, for example, the DH-Group (`restart ipsec` will not return connectivity)
Do on both VPP nodes (or use another DH group)
```
delete vpn ipsec ike-group IKE-group proposal 1 dh-group
commit
```
After this, the client `ipsec-c-left` can ping the remote client
```
vyos@ipsec-c-left:~$ ping 100.64.2.10
PING 100.64.2.10 (100.64.2.10) 56(84) bytes of data.
64 bytes from 100.64.2.10: icmp_seq=1 ttl=62 time=1.50 ms
64 bytes from 100.64.2.10: icmp_seq=2 ttl=62 time=2.02 ms
^C
--- 100.64.2.10 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1002ms
rtt min/avg/max/mdev = 1.502/1.758/2.015/0.256 ms
vyos@ipsec-c-left:~$
```
Working trace on the router `vpp-right`
```
vpp# show trace
------------------- Start of thread 0 vpp_main -------------------
Packet 1
00:08:21:372609: dpdk-input
eth0 rx queue 0
buffer 0x9b345: current data 0, length 166, buffer-pool 0, ref-count 1, trace handle 0x0
ext-hdr-valid
PKT MBUF: port 0, nb_segs 1, pkt_len 166
buf_len 2176, data_len 166, ol_flags 0x0, data_off 128, phys_addr 0xe12cd1c0
packet_type 0x0 l2_len 0 l3_len 0 outer_l2_len 0 outer_l3_len 0
rss 0x0 fdir.hi 0x0 fdir.lo 0x0
IP4: 0c:c0:19:43:00:00 -> 0c:84:b5:89:00:00
IPSEC_ESP: 10.0.0.1 -> 10.0.0.2
tos 0x00, ttl 254, length 152, checksum 0xa831 dscp CS0 ecn NON_ECN
fragment id 0x0000
00:08:21:372798: ethernet-input
frame: flags 0x1, hw-if-index 1, sw-if-index 1
IP4: 0c:c0:19:43:00:00 -> 0c:84:b5:89:00:00
00:08:21:372804: ip4-input
IPSEC_ESP: 10.0.0.1 -> 10.0.0.2
tos 0x00, ttl 254, length 152, checksum 0xa831 dscp CS0 ecn NON_ECN
fragment id 0x0000
00:08:21:372808: ip4-lookup
fib 0 dpo-idx 12 flow hash: 0x00000000
IPSEC_ESP: 10.0.0.1 -> 10.0.0.2
tos 0x00, ttl 254, length 152, checksum 0xa831 dscp CS0 ecn NON_ECN
fragment id 0x0000
00:08:21:372810: ip4-receive
fib:0 adj:12 flow:0x00000000
IPSEC_ESP: 10.0.0.1 -> 10.0.0.2
tos 0x00, ttl 254, length 152, checksum 0xa831 dscp CS0 ecn NON_ECN
fragment id 0x0000
00:08:21:372812: ipsec4-tun-input
IPSec: remote:10.0.0.1 spi:3224556932 (0xc032d584) sa:2 tun:0 seq 108
00:08:21:372813: esp4-decrypt-tun
esp: crypto aes-cbc-256 integrity sha1-96 pkt-seq 108 sa-seq 108 pkt-seq-hi 0
00:08:21:372824: ip4-input-no-checksum
ICMP: 100.64.1.10 -> 100.64.2.10
tos 0x00, ttl 63, length 84, checksum 0x7e72 dscp CS0 ecn NON_ECN
fragment id 0xf1a2, flags DONT_FRAGMENT
ICMP echo_request checksum 0xa243 id 2670
00:08:21:372825: ip4-lookup
fib 0 dpo-idx 7 flow hash: 0x00000000
ICMP: 100.64.1.10 -> 100.64.2.10
tos 0x00, ttl 63, length 84, checksum 0x7e72 dscp CS0 ecn NON_ECN
fragment id 0xf1a2, flags DONT_FRAGMENT
ICMP echo_request checksum 0xa243 id 2670
00:08:21:372826: ip4-rewrite
tx_sw_if_index 2 dpo-idx 7 : ipv4 via 100.64.2.10 eth1: mtu:1500 next:6 flags:[] 0cb6a38200000c84b58900010800 flow hash: 0x00000000
00000000: 0cb6a38200000c84b5890001080045000054f1a240003e017f726440010a6440
00000020: 020a0800a2430a6e0010c43a9368000000002ec80600000000001011
00:08:21:372827: eth1-output
eth1 flags 0x0018000d
IP4: 0c:84:b5:89:00:01 -> 0c:b6:a3:82:00:00
ICMP: 100.64.1.10 -> 100.64.2.10
tos 0x00, ttl 62, length 84, checksum 0x7f72 dscp CS0 ecn NON_ECN
fragment id 0xf1a2, flags DONT_FRAGMENT
ICMP echo_request checksum 0xa243 id 2670
00:08:21:372829: eth1-tx
eth1 tx queue 0
buffer 0x9b345: current data 44, length 98, buffer-pool 0, ref-count 1, trace handle 0x0
ext-hdr-valid
l2-hdr-offset 0 l3-hdr-offset 14
PKT MBUF: port 0, nb_segs 1, pkt_len 98
buf_len 2176, data_len 98, ol_flags 0x0, data_off 172, phys_addr 0xe12cd1c0
packet_type 0x0 l2_len 0 l3_len 0 outer_l2_len 0 outer_l3_len 0
rss 0x0 fdir.hi 0x0 fdir.lo 0x0
IP4: 0c:84:b5:89:00:01 -> 0c:b6:a3:82:00:00
ICMP: 100.64.1.10 -> 100.64.2.10
tos 0x00, ttl 62, length 84, checksum 0x7f72 dscp CS0 ecn NON_ECN
fragment id 0xf1a2, flags DONT_FRAGMENT
ICMP echo_request checksum 0xa243 id 2670
```