Page MenuHomeVyOS Platform

MTR/Traceroute broken in 1.3-beta
Closed, ResolvedPublicBUG

Description

Since at least beta/rolling 202101 MTR/Traceroute has failed for all hops except the last.

Rollback to 1.3-rolling-202008240118 resolves this issue.

1.3-beta-202103170443 seems to skip all BGP routes but resolves other routes (I'm only guessing here since I can only see up to where my network ends).

Details

Difficulty level
Unknown (require assessment)
Version
202103170443
Why the issue appeared?
Will be filled on close
Is it a breaking change?
Unspecified (possibly destroys the router)

Event Timeline

@Viacheslav The issue persists in vyos-1.3.0-rc2-amd64.iso

This affects downstream clients as well

Can you share more examples/configs?
How can we reproduce it?

I'm not really sure what the issue is. I can narrow it down to this:

  • 1.3-rolling-202008240118 does not have the issue
  • Previous rolling releases would not show ANY hops between local device and destination IP (both MTR/Traceroute on TCP or UDP) but pinging worked fine for every device in the path
  • Current rolling release and rc2 don't show hops that are defined by BGP

For the specific config I can provide more targeted config values as needed but since this affects multiple devices the best I can give you is the network layout.

All devices run same version of Vyos
Network is in a DMVPN 1.1.1.0/28
The DMVPN has two hubs – two core routers
The network is connected to one ISP on two gateways
Each gateway announces 1.1.1.0/24 to ISP
Each gateway is connected to each core router
Both gateway announces the same /32 IPv4 Address via iBGP to core routers
Both core routers announce the same /32 IPv4 Address via iBGP to all DMVPN devices
Both core routers are DMVPN hubs
Campus router announces specific routers for local network to core routers
All traffic next-hops to the /32 BGP address and uses BGP multi path to load share

Screen Shot 2021-03-23 at 2.15.00 AM.png (660×627 px, 54 KB)

Any thoughts here @Viacheslav ? This issue persists in 1.3-rc4

@francis We don't know anything about this issue.
And it difficult to say without the current configuration.

I can try to sanitize my configs and post here – FWIW, this issue is not in 1.2.7 LTS (self built)

Related to the image here are sanitized configs (I've removed firewall entries since I've tested without a firewall config and the issue persists. I've replaced all IPs with dummy ones

BB-CR Config

interfaces {
    dummy dum9 {
        address "10.100.100.9/32"
        description "BGP-Loopback"
    }
    ethernet eth0 {
        address "1.2.7.8/24"
        description "Internet"
    }
    loopback "lo"
    tunnel tun0 {
        address "10.100.100.1/27"
        description "BB-CR-DMVPN"
        encapsulation "gre"
        local-ip "1.2.7.8"
        mtu "1460"
        multicast "enable"
        parameters {
            ip {
                key "1000"
            }
        }
        policy {
            route "change-mss"
        }
    }
}
policy {
    prefix-list noIPv4 {
        rule 1 {
            action "permit"
            ge "1"
            prefix "0.0.0.0/0"
        }
    }
    route change-mss {
        rule 1 {
            protocol "tcp"
            set {
                tcp-mss "1420"
            }
            tcp {
                flags "SYN"
            }
        }
    }
    route-map noIPv4 {
        rule 1 {
            action "deny"
            match {
                ip {
                    address {
                        prefix-list "noIPv4"
                    }
                }
            }
        }
    }
}
protocols {
    bgp 30707 {
        address-family {
            ipv4-unicast {
                network "10.100.100.1/32"
                network "10.100.100.9/32"
            }
        }
        neighbor 10.100.100.11 {
            address-family {
                ipv4-unicast {
                    weight "100"
                }
            }
            remote-as "30707"
            timers {
                connect "5"
                holdtime "10"
                keepalive "10"
            }
        }
        neighbor 10.100.100.12 {
            address-family {
                ipv4-unicast {
                    weight "100"
                }
            }
            remote-as "30707"
            timers {
                connect "5"
                holdtime "10"
                keepalive "10"
            }
        }
        neighbor 10.100.100.21 {
            address-family {
                ipv4-unicast {
                    weight "100"
                }
            }
            remote-as "30707"
        }
        parameters {
            bestpath {
                as-path {
                    multipath-relax {
                    }
                }
            }
            router-id "10.100.100.1"
        }
        timers {
            holdtime "10"
            keepalive "5"
        }
    }
    nhrp {
        tunnel tun0 {
            holding-time "30"
            map 10.100.100.2/27 {
                nbma-address "4.2.2.209"
            }
            multicast "dynamic"
        }
    }
    static {
        route 0.0.0.0/0 {
            next-hop "10.100.100.10"
        }
        route 10.100.100.5/32 {
            next-hop "10.100.100.21"
        }
        route 7.4.8.0/24 {
            next-hop "14.22.7.1"
        }
        route 144.202.70.92/32 {
            next-hop "14.22.7.1"
        }
        route 14.22.3.5/32 {
            next-hop "14.22.7.1"
        }
        route 4.2.2.209/32 {
            next-hop "14.22.7.1"
        }
    }
}
service {
    snmp {
        listen-address "10.100.100.1"
        location "USDFW1-BB-CR1"
        v3 {
            engineid "000000000000000000000002"
            group default {
                mode "ro"
                view "default"
            }
            user public {
                auth {
                    encrypted-password "password"
                    type "md5"
                }
                group "default"
                mode "ro"
                privacy {
                    encrypted-password "password"
                    type "aes"
                }
            }
            user vyos {
                auth {
                    encrypted-password "password"
                    type "sha"
                }
                group "default"
                privacy {
                    encrypted-password ""
                    type "aes"
                }
            }
            view default {
                oid "0"
                oid "1"
                oid "2"
                oid "3"
            }
        }
    }
    ssh {
        port "16922"
    }
}
system {
    config-management {
        commit-revisions "100"
    }
    console {
        device ttyS0 {
            speed "115200"
            speed 9600
        }
    }
    host-name "usdfw1-bb-cr1"
    ip {
        multipath {
            layer4-hashing {
            }
        }
    }
    login {
        user vyos {
            authentication {
                encrypted-password "password"
                plaintext-password ""
            }
        }
    }
    name-server "1.1.1.1"
    name-server "1.0.0.1"
    ntp {
        server "0.pool.ntp.org"
        server "1.pool.ntp.org"
        server "2.pool.ntp.org"
    }
    syslog {
        global {
            facility all {
                level "info"
            }
            facility protocols {
                level "debug"
            }
        }
    }
}
/* Warning: Do not remove the following line. */
/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@1:conntrack-sync@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */
/* Release version: 1.2.7-epa1-4.19.178 */

BB-GW Config

interfaces {
    dummy dum10 {
        address "10.100.100.10/32"
        description "BGP-Loopback"
    }
    ethernet eth0 {
        address "14.22.3.5/23"
        description "Internet"
        firewall {
            in {
                ipv6-name "TO-SECURENET-IPV6"
                name "TO-SECURENET-IPV4"
            }
        }
    }
    loopback "lo"
    tunnel tun0 {
        address "10.100.100.12/27"
        description "BB-DMVPN"
        encapsulation "gre"
        local-ip "14.22.3.5"
        mtu "1460"
        parameters {
            ip {
                key "1000"
            }
        }
        policy {
            route "chnage-mss"
        }
    }
}
policy {
    prefix-list loopback-routes {
        rule 10 {
            action "permit"
            ge "32"
            prefix "0.0.0.0/0"
        }
        rule 100 {
            action "deny"
            le "32"
            prefix "0.0.0.0/0"
        }
    }
    prefix-list noIPv4 {
        rule 1 {
            action "permit"
            ge "1"
            prefix "0.0.0.0/0"
        }
    }
    prefix-list public-routes {
        rule 10 {
            action "permit"
            le "24"
            prefix "0.0.0.0/0"
        }
        rule 11 {
            action "deny"
            le "32"
            prefix "0.0.0.0/0"
        }
    }
    route change-mss {
        rule 1 {
            protocol "tcp"
            set {
                tcp-mss "1420"
            }
            tcp {
                flags "SYN"
            }
        }
    }
    route chnage-mss {
        rule 1 {
            protocol "tcp"
            set {
                tcp-mss "1420"
            }
            tcp {
                flags "SYN"
            }
        }
    }
    route-map noIPv4 {
        rule 1 {
            action "deny"
            match {
                ip {
                    address {
                        prefix-list "noIPv4"
                    }
                }
            }
        }
    }
    route-map self-hop {
        rule 1 {
            action "permit"
            set {
                as-path-prepend "30707 30707 30707 30707"
            }
        }
    }
}
protocols {
    bgp 30707 {
        address-family {
            ipv4-unicast {
                network "10.100.100.0/24"
                network "10.100.100.10/32"
                network "10.100.100.12/32"
                network "10.200.200.0/24"
            }
        }
        neighbor 10.100.100.1 {
            address-family {
                ipv4-unicast {
                    prefix-list {
                        export "loopback-routes"
                    }
                    weight "100"
                }
            }
            remote-as "30707"
        }
        neighbor 10.100.100.2 {
            address-family {
                ipv4-unicast {
                    prefix-list {
                        export "loopback-routes"
                    }
                    weight "100"
                }
            }
            remote-as "30707"
        }
        neighbor 169.254.169.254 {
            address-family {
                ipv4-unicast {
                    prefix-list {
                        export "public-routes"
                    }
                }
            }
            ebgp-multihop "2"
            password "password"
            remote-as "64515"
        }
        parameters {
            bestpath {
                as-path {
                    multipath-relax {
                    }
                }
            }
            router-id "10.100.100.12"
        }
        timers {
            holdtime "10"
            keepalive "5"
        }
    }
    nhrp {
        tunnel tun0 {
            holding-time "30"
            map 10.100.100.1/27 {
                nbma-address "1.2.7.8"
                register {
                }
            }
            map 10.100.100.2/27 {
                nbma-address "4.2.2.209"
                register {
                }
            }
            multicast "nhs"
        }
    }
    static {
        route 0.0.0.0/0 {
            next-hop "14.2.7.1"
        }
        route 10.100.100.0/24 {
            next-hop "10.100.100.9"
        }
        route 10.100.100.1/32 {
            next-hop "1.2.7.8"
        }
        route 10.100.100.2/32 {
            next-hop "4.2.2.209"
        }
        route 10.100.100.5/32 {
            next-hop "10.100.100.9"
        }
        route 10.200.200.0/24 {
            next-hop "10.100.100.9"
        }
    }
}
service {
    snmp {
        listen-address "10.100.100.12"
        location "USDFW1-BB-GW2"
        v3 {
            engineid "000000000000000000000012"
            group default {
                mode "ro"
                view "default"
            }
            user cacvyos {
                auth {
                    encrypted-password "password"
                    type "md5"
                }
                group "default"
                mode "ro"
                privacy {
                    encrypted-password "password"
                    type "aes"
                }
            }
            view default {
                oid "1"
                oid "2"
                oid "3"
            }
        }
    }
    ssh {
        port "16922"
    }
}
system {
    config-management {
        commit-revisions "100"
    }
    console {
        device ttyS0 {
            speed "115200"
            speed 9600
        }
    }
    host-name "usdfw1-bb-gw2"
    ip {
        multipath {
            layer4-hashing {
            }
        }
    }
    login {
        user vyos {
            authentication {
                encrypted-password "password"
                plaintext-password ""
            }
        }
    }
    name-server "1.1.1.1"
    name-server "1.0.0.1"
    ntp {
        server "0.pool.ntp.org"
        server "1.pool.ntp.org"
        server "2.pool.ntp.org"
    }
    syslog {
        global {
            facility all {
                level "info"
            }
            facility protocols {
                level "debug"
            }
        }
    }
    conntrack {
    }
}
/* Warning: Do not remove the following line. */
/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@1:conntrack-sync@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */
/* Release version: 1.2.7-epa1-4.19.178 */

Confirmed fixed and working in 1.3.0-rc6

Viacheslav claimed this task.