Monday, July 24, 2017

Architect of OpenStack L3 Router HA

l3routerha

在两个netowrk node中,我们分别看到了virtual-router qrouter-f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec

[root@openstackcontroller13 ~]# ip netns list|grep f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec
qrouter-f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec
[root@openstackcontroller13 ~]# ip netns exec qrouter-f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec bash
[root@openstackcontroller13 ~]# ifconfig
ha-880fa0e2-8d: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 169.254.192.4  netmask 255.255.192.0  broadcast 169.254.255.255
        inet6 fe80::f816:3eff:fe90:adf0  prefixlen 64  scopeid 0x20<link>
        ether fa:16:3e:90:ad:f0  txqueuelen 0  (Ethernet)
        RX packets 609493  bytes 32927812 (31.4 MiB)
        RX errors 0  dropped 43  overruns 0  frame 0
        TX packets 304608  bytes 16449072 (15.6 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qg-3674d949-4c: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 10.89.151.168  netmask 255.255.0.0  broadcast 0.0.0.0
        inet6 fe80::f816:3eff:fe8e:b815  prefixlen 64  scopeid 0x20<link>
        ether fa:16:3e:8e:b8:15  txqueuelen 0  (Ethernet)
        RX packets 68441559  bytes 19571436387 (18.2 GiB)
        RX errors 0  dropped 2251  overruns 0  frame 0
        TX packets 55319  bytes 5194356 (4.9 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qr-352424b9-3e: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 192.168.20.1  netmask 255.255.255.0  broadcast 0.0.0.0
        inet6 fe80::f816:3eff:fe12:5526  prefixlen 64  scopeid 0x20<link>
        ether fa:16:3e:12:55:26  txqueuelen 0  (Ethernet)
        RX packets 3675  bytes 366823 (358.2 KiB)
        RX errors 0  dropped 13  overruns 0  frame 0
        TX packets 1394  bytes 132232 (129.1 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@openstackcontroller12 ~]# ip netns exec qrouter-f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec bash
[root@openstackcontroller12 ~]# ifconfig
ha-71d6264d-9d: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 169.254.192.3  netmask 255.255.192.0  broadcast 169.254.255.255
        inet6 fe80::f816:3eff:fee7:4c03  prefixlen 64  scopeid 0x20<link>
        ether fa:16:3e:e7:4c:03  txqueuelen 0  (Ethernet)
        RX packets 800714  bytes 43265351 (41.2 MiB)
        RX errors 0  dropped 31  overruns 0  frame 0
        TX packets 12  bytes 1008 (1008.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qg-3674d949-4c: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        ether fa:16:3e:8e:b8:15  txqueuelen 0  (Ethernet)
        RX packets 58884872  bytes 18031883270 (16.7 GiB)
        RX errors 0  dropped 2002  overruns 0  frame 0
        TX packets 1  bytes 110 (110.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qr-352424b9-3e: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        ether fa:16:3e:12:55:26  txqueuelen 0  (Ethernet)
        RX packets 500  bytes 57320 (55.9 KiB)
        RX errors 0  dropped 17  overruns 0  frame 0
        TX packets 1  bytes 110 (110.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

我们查一下keepalived

ps aux|grep keepalived |grep f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec

root     39958  0.0  0.0 111636  1364 ?        Ss   Oct26   0:23 keepalived -P -f /var/lib/neutron/ha_confs/f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec/keepalived.conf -p /var/lib/neutron/ha_confs/f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec.pid -r /var/lib/neutron/ha_confs/f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec.pid-vrrp

上述进程是透过network namespace下执行的。然而,network namespace并没有隔离进程,因此,在任何地方均可以看到全部进程。 顺便refer我之前对network namespace的研究

http://gogosatellite.blogspot.tw/2016/06/playing-openvswitch-and-namespace-veth.html

/var/lib/neutron/ha_confs/f8d376d2-6d8a-4ec1-9939-4c6ec25f8cec/keepalived.conf

vrrp_instance VR_2 {
    state BACKUP
    interface ha-71d6264d-9d
    virtual_router_id 2
    priority 50
    garp_master_delay 60
    nopreempt
    advert_int 2
    track_interface {
        ha-71d6264d-9d
    }
    virtual_ipaddress {
        169.254.0.2/24 dev ha-71d6264d-9d
    }
    virtual_ipaddress_excluded {
        10.89.151.168/16 dev qg-3674d949-4c
        192.168.20.1/24 dev qr-352424b9-3e
        fe80::f816:3eff:fe12:5526/64 dev qr-352424b9-3e scope link
        fe80::f816:3eff:fe8e:b815/64 dev qg-3674d949-4c scope link
    }
    virtual_routes {
        0.0.0.0/0 via 10.89.1.254 dev qg-3674d949-4c
    }
}

官网的解释

https://wiki.openstack.org/wiki/Neutron/L3HighAvailability_VRRP

global_defs {
    router_id ${VR_ID}
}
vrrp_sync_group VG${VR_GROUP_ID} {
    group {
        VI_HA
    }
    % if NOTIFY_SCRIPT:
    notify_master ${NOTIFY_SCRIPT}
    % endif
}

vrrp_instance VI_HA {
    % if TYPE == 'MASTER':
    state MASTER
    % else:
    state SLAVE
    % endif
    interface ${L3_AGENT.get_ha_device_name(TRACK_PORT_ID)}
    virtual_router_id ${VR_ID}
    priority ${PRIORITY}
    track_interface {
        ${L3_AGENT.get_ha_device_name(TRACK_PORT_ID)}
    }
    virtual_ipaddress {
        % if EXTERNAL_PORT:
        ${EXTERNAL_PORT['ip_cidr']} dev ${L3_AGENT.get_external_device_name(EXTERNAL_PORT['id'])}
        % if FLOATING_IPS:
        ${FLOATING_IPS[0]['floating_ip_address']}/32 dev ${L3_AGENT.get_external_device_name(EXTERNAL_PORT['id'])}
        % endif
        % endif

        % if INTERNAL_PORTS:
        ${INTERNAL_PORTS[0]['ip_cidr']} dev ${L3_AGENT.get_internal_device_name(INTERNAL_PORTS[0]['id'])}
        % endif
    }
    virtual_ipaddress_excluded {
        % if EXTERNAL_PORT:
        % for FLOATING_IP in FLOATING_IPS[1:]:
        ${FLOATING_IP['floating_ip_address']}/32 dev ${L3_AGENT.get_external_device_name(EXTERNAL_PORT['id'])}
        % endfor
        % endif

        % for INTERNAL_PORT in INTERNAL_PORTS[1:]:
        ${INTERNAL_PORT['ip_cidr']} dev ${L3_AGENT.get_internal_device_name(INTERNAL_PORT['id'])}
        % endfor
    }

    % if EXTERNAL_PORT:
    virtual_routes {
        0.0.0.0/0 via ${EXTERNAL_PORT['ip_cidr'].split('/')[0]} dev ${L3_AGENT.get_external_device_name(EXTERNAL_PORT['id'])}
    }
    % endif
}

virtual_ipaddress为VIP的设定,virtual_ipaddress_excluded为namespace内network device的IP的设定。 standby并未设定任何值,直到fail over产生。此网路设定还包含Mac Address的设定,两台相同。

No comments:

Post a Comment