Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

Floating IP Problem Under JUNO DVR

Floating IP Problem Under JUNO DVR

Type: Isolated issue for now

Presence: Juno 2014.2-1 RDO

Fix: Isolated issue for now

Description:

Whenever create FIP on a VM, it adds the FIP to ALL other compute nodes, a routing prefix in the FIP namespace, and IP interface alias on the qrouter. However, the iptables gets updated normally with only the DNAT for the particular IP of the VM on that compute node This causes the FIP proxy arp to answer ARP requests for ALL VM's on ALL compute nodes which results in compute nodes answering ARPs where they do not have the VM effectively blackholing traffic to that ip.

Here is a demonstration of the problem:

Before adding a vm+fip on compute4

[root@compute2 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-6ede0596-3a
169.254.31.28/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.29
173.209.44.0/24 dev fg-6ede0596-3a  proto kernel  scope link  src 173.209.44.6
173.209.44.4 via 169.254.31.28 dev fpr-3a90aae6-3



[root@compute3 neutron]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-26bef858-6b
169.254.31.238/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.239
173.209.44.0/24 dev fg-26bef858-6b  proto kernel  scope link  src 173.209.44.5
173.209.44.3 via 169.254.31.238 dev fpr-3a90aae6-3



[root@compute4 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-2919b6be-f4
173.209.44.0/24 dev fg-2919b6be-f4  proto kernel  scope link  src 173.209.44.8

after creating a new vm on compute4 and attaching a floating IP to it, we get this result. of course at this point, only the vm on compute4 is able to ping the public network

[root@compute2 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-6ede0596-3a
169.254.31.28/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.29
173.209.44.0/24 dev fg-6ede0596-3a  proto kernel  scope link  src 173.209.44.6
173.209.44.4 via 169.254.31.28 dev fpr-3a90aae6-3
173.209.44.7 via 169.254.31.28 dev fpr-3a90aae6-3



[root@compute3 neutron]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-26bef858-6b
169.254.31.238/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.239
173.209.44.0/24 dev fg-26bef858-6b  proto kernel  scope link  src 173.209.44.5
173.209.44.3 via 169.254.31.238 dev fpr-3a90aae6-3
173.209.44.7 via 169.254.31.238 dev fpr-3a90aae6-3



[root@compute4 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-2919b6be-f4
169.254.30.20/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.30.21
173.209.44.0/24 dev fg-2919b6be-f4  proto kernel  scope link  src 173.209.44.8
173.209.44.3 via 169.254.30.20 dev fpr-3a90aae6-3
173.209.44.4 via 169.254.30.20 dev fpr-3a90aae6-3
173.209.44.7 via 169.254.30.20 dev fpr-3a90aae6-3

when we deleted the extra FIP from each Compute Nodes Namespace, everything starts to work just fine

Following are the router, floating IP information and config files :

+-----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Field                 | Value                                                                                                                                                                                    |
+-----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| admin_state_up        | True                                                                                                                                                                                     |
| distributed           | True                                                                                                                                                                                     |
| external_gateway_info | {"network_id": "616a6213-c339-4164-9dff-344ae9e04929", "enable_snat": true, "external_fixed_ips": [{"subnet_id": "0077e2d5-3c3d-4cd2-b55c-ee380fba7867", "ip_address": "173.209.44.2"}]} |
| ha                    | False                                                                                                                                                                                    |
| id                    | 3a90aae6-3107-49e4-a190-92ed37a43b1a                                                                                                                                                     |
| name                  | admin-router                                                                                                                                                                             |
| routes                |                                                                                                                                                                                          |
| status                | ACTIVE                                                                                                                                                                                   |
| tenant_id             | 132a585092284807a115f61cd1e3f688                                                                                                                                                         |
+-----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

[root@controller1 ~]# neutron floatingip-show 9919c836-532b-44d8-ba9e-8600c59ec1ec

+---------------------+--------------------------------------+
| Field               | Value                                |
    +---------------------+--------------------------------------+
    | fixed_ip_address    | 10.0.0.11                            |
    | floating_ip_address | 173.209.44.3                         |
    | floating_network_id | 616a6213-c339-4164-9dff-344ae9e04929 |
    | id                  | 9919c836-532b-44d8-ba9e-8600c59ec1ec |
    | port_id             | 8b875248-0149-4e4f-805e-361b060ac1e4 |
    | router_id           | 3a90aae6-3107-49e4-a190-92ed37a43b1a |
    | status              | ACTIVE                               |
    | tenant_id           | 132a585092284807a115f61cd1e3f688     |
    +---------------------+--------------------------------------+

[root@controller1 ~]# neutron floatingip-show ab73e133-ae75-4aea-9b5e-a4152bd922e2

+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| fixed_ip_address    | 10.0.0.9                             |
| floating_ip_address | 173.209.44.4                         |
| floating_network_id | 616a6213-c339-4164-9dff-344ae9e04929 |
| id                  | ab73e133-ae75-4aea-9b5e-a4152bd922e2 |
| port_id             | 3273aa63-4928-4880-86f7-634139772e36 |
| router_id           | 3a90aae6-3107-49e4-a190-92ed37a43b1a |
| status              | ACTIVE                               |
| tenant_id           | 132a585092284807a115f61cd1e3f688     |
+---------------------+--------------------------------------+

[root@controller1 ~]# neutron floatingip-show bf456993-d20a-48b5-b62d-a1e397acfd1d

+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| fixed_ip_address    | 10.0.0.12                            |
| floating_ip_address | 173.209.44.7                         |
| floating_network_id | 616a6213-c339-4164-9dff-344ae9e04929 |
| id                  | bf456993-d20a-48b5-b62d-a1e397acfd1d |
| port_id             | 7b3ec99d-6a21-4446-b305-83a7d9bb6534 |
| router_id           | 3a90aae6-3107-49e4-a190-92ed37a43b1a |
| status              | ACTIVE                               |
| tenant_id           | 132a585092284807a115f61cd1e3f688     |
+---------------------+--------------------------------------+





[root@net1 neutron]# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose = True
router_distributed = True
debug = True
use_syslog = True
core_plugin = ml2
service_plugins = router,lbaas
auth_strategy = keystone
allow_overlapping_ips = True
allow_automatic_l3agent_failover = True
dhcp_agents_per_network = 2
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://nova:8774/v2
nova_admin_auth_url = http://keystone:35357/v2.0
nova_region_name = regionOne
nova_admin_username = nova
nova_admin_tenant_id = d7e8412b252247eea6474fdad45442c6
nova_admin_password = secret
rabbit_port = 5672
rabbit_password = guest
rabbit_hosts = queue1:5672, queue2:5672
rabbit_userid = guest
rabbit_virtual_host = /
rabbit_ha_queues = True
rpc_backend=rabbit
[matchmaker_redis]
[matchmaker_ring]
[quotas]
[agent]
[keystone_authtoken]
auth_uri = http://keystone:5000/v2.0
identity_uri = http://keystone:35357
admin_tenant_name = service
admin_user = neutron
admin_password = secret
[database]
connection = mysql://neutron:secret@db/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default

[root@net1 neutron]# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = public
verbose=True
agent_mode = dvr_snat



[root@compute1 neutron]# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose = True
router_distributed = True
debug = True
use_syslog = True
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
base_mac = fa:16:3e:01:00:00
dvr_base_mac = fa:16:3f:01:00:00
allow_overlapping_ips = True
rabbit_port = 5672
rabbit_password = guest
rabbit_hosts = queue1:5672, queue2:5672
rabbit_userid = guest
rabbit_virtual_host = /
rabbit_ha_queues = True
rpc_backend=rabbit
[matchmaker_redis]
[matchmaker_ring]
[quotas]
[agent]
[keystone_authtoken]
auth_uri = http://keystone:5000/v2.0
identity_uri = http://keystone:35357
admin_tenant_name = service
admin_user = neutron
admin_password = secret
[database]
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default



[root@compute1 neutron]# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = public
verbose=True
agent_mode = dvr

Floating IP Problem Under JUNO DVR

Floating IP Problem Under JUNO DVR

Type: Isolated issue for now

Presence: Juno 2014.2-1 RDO

Fix: Isolated issue for now

Description:

Whenever create FIP on a VM, it adds the FIP to ALL other compute nodes, a routing prefix in the FIP namespace, and IP interface alias on the qrouter. However, the iptables gets updated normally with only the DNAT for the particular IP of the VM on that compute node This causes the FIP proxy arp to answer ARP requests for ALL VM's on ALL compute nodes which results in compute nodes answering ARPs where they do not have the VM effectively blackholing traffic to that ip.

Here is a demonstration of the problem:

Before adding a vm+fip on compute4

[root@compute2 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-6ede0596-3a
169.254.31.28/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.29
173.209.44.0/24 dev fg-6ede0596-3a  proto kernel  scope link  src 173.209.44.6
173.209.44.4 via 169.254.31.28 dev fpr-3a90aae6-3



[root@compute3 neutron]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-26bef858-6b
169.254.31.238/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.239
173.209.44.0/24 dev fg-26bef858-6b  proto kernel  scope link  src 173.209.44.5
173.209.44.3 via 169.254.31.238 dev fpr-3a90aae6-3



[root@compute4 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-2919b6be-f4
173.209.44.0/24 dev fg-2919b6be-f4  proto kernel  scope link  src 173.209.44.8

after creating a new vm on compute4 and attaching a floating IP to it, we get this result. of course at this point, only the vm on compute4 is able to ping the public network

[root@compute2 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-6ede0596-3a
169.254.31.28/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.29
173.209.44.0/24 dev fg-6ede0596-3a  proto kernel  scope link  src 173.209.44.6
173.209.44.4 via 169.254.31.28 dev fpr-3a90aae6-3
173.209.44.7 via 169.254.31.28 dev fpr-3a90aae6-3



[root@compute3 neutron]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-26bef858-6b
169.254.31.238/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.239
173.209.44.0/24 dev fg-26bef858-6b  proto kernel  scope link  src 173.209.44.5
173.209.44.3 via 169.254.31.238 dev fpr-3a90aae6-3
173.209.44.7 via 169.254.31.238 dev fpr-3a90aae6-3



[root@compute4 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-2919b6be-f4
169.254.30.20/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.30.21
173.209.44.0/24 dev fg-2919b6be-f4  proto kernel  scope link  src 173.209.44.8
173.209.44.3 via 169.254.30.20 dev fpr-3a90aae6-3
173.209.44.4 via 169.254.30.20 dev fpr-3a90aae6-3
173.209.44.7 via 169.254.30.20 dev fpr-3a90aae6-3

when we deleted the extra FIP from each Compute Nodes Namespace, everything starts to work just fine

Following are the router, floating IP information and config files :

+-----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Field                 | Value                                                                                                                                                                                    |
+-----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| admin_state_up        | True                                                                                                                                                                                     |
| distributed           | True                                                                                                                                                                                     |
| external_gateway_info | {"network_id": "616a6213-c339-4164-9dff-344ae9e04929", "enable_snat": true, "external_fixed_ips": [{"subnet_id": "0077e2d5-3c3d-4cd2-b55c-ee380fba7867", "ip_address": "173.209.44.2"}]} |
| ha                    | False                                                                                                                                                                                    |
| id                    | 3a90aae6-3107-49e4-a190-92ed37a43b1a                                                                                                                                                     |
| name                  | admin-router                                                                                                                                                                             |
| routes                |                                                                                                                                                                                          |
| status                | ACTIVE                                                                                                                                                                                   |
| tenant_id             | 132a585092284807a115f61cd1e3f688                                                                                                                                                         |
+-----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

[root@controller1 ~]# neutron floatingip-show 9919c836-532b-44d8-ba9e-8600c59ec1ec

+---------------------+--------------------------------------+
| Field               | Value                                |
    +---------------------+--------------------------------------+
    | fixed_ip_address    | 10.0.0.11                            |
    | floating_ip_address | 173.209.44.3                         |
    | floating_network_id | 616a6213-c339-4164-9dff-344ae9e04929 |
    | id                  | 9919c836-532b-44d8-ba9e-8600c59ec1ec |
    | port_id             | 8b875248-0149-4e4f-805e-361b060ac1e4 |
    | router_id           | 3a90aae6-3107-49e4-a190-92ed37a43b1a |
    | status              | ACTIVE                               |
    | tenant_id           | 132a585092284807a115f61cd1e3f688     |
    +---------------------+--------------------------------------+

[root@controller1 ~]# neutron floatingip-show ab73e133-ae75-4aea-9b5e-a4152bd922e2

+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| fixed_ip_address    | 10.0.0.9                             |
| floating_ip_address | 173.209.44.4                         |
| floating_network_id | 616a6213-c339-4164-9dff-344ae9e04929 |
| id                  | ab73e133-ae75-4aea-9b5e-a4152bd922e2 |
| port_id             | 3273aa63-4928-4880-86f7-634139772e36 |
| router_id           | 3a90aae6-3107-49e4-a190-92ed37a43b1a |
| status              | ACTIVE                               |
| tenant_id           | 132a585092284807a115f61cd1e3f688     |
+---------------------+--------------------------------------+

[root@controller1 ~]# neutron floatingip-show bf456993-d20a-48b5-b62d-a1e397acfd1d

+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| fixed_ip_address    | 10.0.0.12                            |
| floating_ip_address | 173.209.44.7                         |
| floating_network_id | 616a6213-c339-4164-9dff-344ae9e04929 |
| id                  | bf456993-d20a-48b5-b62d-a1e397acfd1d |
| port_id             | 7b3ec99d-6a21-4446-b305-83a7d9bb6534 |
| router_id           | 3a90aae6-3107-49e4-a190-92ed37a43b1a |
| status              | ACTIVE                               |
| tenant_id           | 132a585092284807a115f61cd1e3f688     |
+---------------------+--------------------------------------+





[root@net1 neutron]# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose = True
router_distributed = True
debug = True
use_syslog = True
core_plugin = ml2
service_plugins = router,lbaas
auth_strategy = keystone
allow_overlapping_ips = True
allow_automatic_l3agent_failover = True
dhcp_agents_per_network = 2
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://nova:8774/v2
nova_admin_auth_url = http://keystone:35357/v2.0
nova_region_name = regionOne
nova_admin_username = nova
nova_admin_tenant_id = d7e8412b252247eea6474fdad45442c6
nova_admin_password = secret
rabbit_port = 5672
rabbit_password = guest
rabbit_hosts = queue1:5672, queue2:5672
rabbit_userid = guest
rabbit_virtual_host = /
rabbit_ha_queues = True
rpc_backend=rabbit
[matchmaker_redis]
[matchmaker_ring]
[quotas]
[agent]
[keystone_authtoken]
auth_uri = http://keystone:5000/v2.0
identity_uri = http://keystone:35357
admin_tenant_name = service
admin_user = neutron
admin_password = secret
[database]
connection = mysql://neutron:secret@db/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default

[root@net1 neutron]# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = public
verbose=True
agent_mode = dvr_snat



[root@compute1 neutron]# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose = True
router_distributed = True
debug = True
use_syslog = True
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
base_mac = fa:16:3e:01:00:00
dvr_base_mac = fa:16:3f:01:00:00
allow_overlapping_ips = True
rabbit_port = 5672
rabbit_password = guest
rabbit_hosts = queue1:5672, queue2:5672
rabbit_userid = guest
rabbit_virtual_host = /
rabbit_ha_queues = True
rpc_backend=rabbit
[matchmaker_redis]
[matchmaker_ring]
[quotas]
[agent]
[keystone_authtoken]
auth_uri = http://keystone:5000/v2.0
identity_uri = http://keystone:35357
admin_tenant_name = service
admin_user = neutron
admin_password = secret
[database]
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default



[root@compute1 neutron]# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = public
verbose=True
agent_mode = dvr


[root@net1 neutron]# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#
[ml2]
type_drivers = vxlan,vlan,flat
tenant_network_types = vxlan
mechanism_drivers = openvswitch,l2population
[ml2_type_flat]
flat_networks = public
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges = 10000:100000
[securitygroup]
enable_security_group = True
enable_ipset = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[agent]
l2_population=True
polling_interval=2
arp_responder=True
tunnel_types=vxlan
enable_distributed_routing = True
[ovs]
enable_tunneling=True
integration_bridge=br-int
local_ip=10.60.0.3
tunnel_bridge=br-tun
bridge_mappings=public:public

Floating IP Problem Under JUNO DVR

Floating IP Problem Under JUNO DVR

NOTE: This was converted to a bug report : https://bugs.launchpad.net/neutron/+bug/1389880

Type: Isolated issue for now

Presence: Juno 2014.2-1 RDO

Fix: Isolated issue for now

Description:

Whenever create FIP on a VM, it adds the FIP to ALL other compute nodes, a routing prefix in the FIP namespace, and IP interface alias on the qrouter. However, the iptables gets updated normally with only the DNAT for the particular IP of the VM on that compute node This causes the FIP proxy arp to answer ARP requests for ALL VM's on ALL compute nodes which results in compute nodes answering ARPs where they do not have the VM effectively blackholing traffic to that ip.

Here is a demonstration of the problem:

Before adding a vm+fip on compute4

[root@compute2 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-6ede0596-3a
169.254.31.28/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.29
173.209.44.0/24 dev fg-6ede0596-3a  proto kernel  scope link  src 173.209.44.6
173.209.44.4 via 169.254.31.28 dev fpr-3a90aae6-3



[root@compute3 neutron]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-26bef858-6b
169.254.31.238/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.239
173.209.44.0/24 dev fg-26bef858-6b  proto kernel  scope link  src 173.209.44.5
173.209.44.3 via 169.254.31.238 dev fpr-3a90aae6-3



[root@compute4 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-2919b6be-f4
173.209.44.0/24 dev fg-2919b6be-f4  proto kernel  scope link  src 173.209.44.8

after creating a new vm on compute4 and attaching a floating IP to it, we get this result. of course at this point, only the vm on compute4 is able to ping the public network

[root@compute2 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-6ede0596-3a
169.254.31.28/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.29
173.209.44.0/24 dev fg-6ede0596-3a  proto kernel  scope link  src 173.209.44.6
173.209.44.4 via 169.254.31.28 dev fpr-3a90aae6-3
173.209.44.7 via 169.254.31.28 dev fpr-3a90aae6-3



[root@compute3 neutron]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-26bef858-6b
169.254.31.238/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.31.239
173.209.44.0/24 dev fg-26bef858-6b  proto kernel  scope link  src 173.209.44.5
173.209.44.3 via 169.254.31.238 dev fpr-3a90aae6-3
173.209.44.7 via 169.254.31.238 dev fpr-3a90aae6-3



[root@compute4 ~]# ip netns exec fip-616a6213-c339-4164-9dff-344ae9e04929 ip route show
default via 173.209.44.1 dev fg-2919b6be-f4
169.254.30.20/31 dev fpr-3a90aae6-3  proto kernel  scope link  src 169.254.30.21
173.209.44.0/24 dev fg-2919b6be-f4  proto kernel  scope link  src 173.209.44.8
173.209.44.3 via 169.254.30.20 dev fpr-3a90aae6-3
173.209.44.4 via 169.254.30.20 dev fpr-3a90aae6-3
173.209.44.7 via 169.254.30.20 dev fpr-3a90aae6-3

when we deleted the extra FIP from each Compute Nodes Namespace, everything starts to work just fine

Following are the router, floating IP information and config files :

+-----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Field                 | Value                                                                                                                                                                                    |
+-----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| admin_state_up        | True                                                                                                                                                                                     |
| distributed           | True                                                                                                                                                                                     |
| external_gateway_info | {"network_id": "616a6213-c339-4164-9dff-344ae9e04929", "enable_snat": true, "external_fixed_ips": [{"subnet_id": "0077e2d5-3c3d-4cd2-b55c-ee380fba7867", "ip_address": "173.209.44.2"}]} |
| ha                    | False                                                                                                                                                                                    |
| id                    | 3a90aae6-3107-49e4-a190-92ed37a43b1a                                                                                                                                                     |
| name                  | admin-router                                                                                                                                                                             |
| routes                |                                                                                                                                                                                          |
| status                | ACTIVE                                                                                                                                                                                   |
| tenant_id             | 132a585092284807a115f61cd1e3f688                                                                                                                                                         |
+-----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

[root@controller1 ~]# neutron floatingip-show 9919c836-532b-44d8-ba9e-8600c59ec1ec

+---------------------+--------------------------------------+
| Field               | Value                                |
    +---------------------+--------------------------------------+
    | fixed_ip_address    | 10.0.0.11                            |
    | floating_ip_address | 173.209.44.3                         |
    | floating_network_id | 616a6213-c339-4164-9dff-344ae9e04929 |
    | id                  | 9919c836-532b-44d8-ba9e-8600c59ec1ec |
    | port_id             | 8b875248-0149-4e4f-805e-361b060ac1e4 |
    | router_id           | 3a90aae6-3107-49e4-a190-92ed37a43b1a |
    | status              | ACTIVE                               |
    | tenant_id           | 132a585092284807a115f61cd1e3f688     |
    +---------------------+--------------------------------------+

[root@controller1 ~]# neutron floatingip-show ab73e133-ae75-4aea-9b5e-a4152bd922e2

+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| fixed_ip_address    | 10.0.0.9                             |
| floating_ip_address | 173.209.44.4                         |
| floating_network_id | 616a6213-c339-4164-9dff-344ae9e04929 |
| id                  | ab73e133-ae75-4aea-9b5e-a4152bd922e2 |
| port_id             | 3273aa63-4928-4880-86f7-634139772e36 |
| router_id           | 3a90aae6-3107-49e4-a190-92ed37a43b1a |
| status              | ACTIVE                               |
| tenant_id           | 132a585092284807a115f61cd1e3f688     |
+---------------------+--------------------------------------+

[root@controller1 ~]# neutron floatingip-show bf456993-d20a-48b5-b62d-a1e397acfd1d

+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| fixed_ip_address    | 10.0.0.12                            |
| floating_ip_address | 173.209.44.7                         |
| floating_network_id | 616a6213-c339-4164-9dff-344ae9e04929 |
| id                  | bf456993-d20a-48b5-b62d-a1e397acfd1d |
| port_id             | 7b3ec99d-6a21-4446-b305-83a7d9bb6534 |
| router_id           | 3a90aae6-3107-49e4-a190-92ed37a43b1a |
| status              | ACTIVE                               |
| tenant_id           | 132a585092284807a115f61cd1e3f688     |
+---------------------+--------------------------------------+





[root@net1 neutron]# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose = True
router_distributed = True
debug = True
use_syslog = True
core_plugin = ml2
service_plugins = router,lbaas
auth_strategy = keystone
allow_overlapping_ips = True
allow_automatic_l3agent_failover = True
dhcp_agents_per_network = 2
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://nova:8774/v2
nova_admin_auth_url = http://keystone:35357/v2.0
nova_region_name = regionOne
nova_admin_username = nova
nova_admin_tenant_id = d7e8412b252247eea6474fdad45442c6
nova_admin_password = secret
rabbit_port = 5672
rabbit_password = guest
rabbit_hosts = queue1:5672, queue2:5672
rabbit_userid = guest
rabbit_virtual_host = /
rabbit_ha_queues = True
rpc_backend=rabbit
[matchmaker_redis]
[matchmaker_ring]
[quotas]
[agent]
[keystone_authtoken]
auth_uri = http://keystone:5000/v2.0
identity_uri = http://keystone:35357
admin_tenant_name = service
admin_user = neutron
admin_password = secret
[database]
connection = mysql://neutron:secret@db/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default

[root@net1 neutron]# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = public
verbose=True
agent_mode = dvr_snat



[root@compute1 neutron]# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose = True
router_distributed = True
debug = True
use_syslog = True
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
base_mac = fa:16:3e:01:00:00
dvr_base_mac = fa:16:3f:01:00:00
allow_overlapping_ips = True
rabbit_port = 5672
rabbit_password = guest
rabbit_hosts = queue1:5672, queue2:5672
rabbit_userid = guest
rabbit_virtual_host = /
rabbit_ha_queues = True
rpc_backend=rabbit
[matchmaker_redis]
[matchmaker_ring]
[quotas]
[agent]
[keystone_authtoken]
auth_uri = http://keystone:5000/v2.0
identity_uri = http://keystone:35357
admin_tenant_name = service
admin_user = neutron
admin_password = secret
[database]
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default



[root@compute1 neutron]# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = public
verbose=True
agent_mode = dvr


[root@net1 neutron]# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#
[ml2]
type_drivers = vxlan,vlan,flat
tenant_network_types = vxlan
mechanism_drivers = openvswitch,l2population
[ml2_type_flat]
flat_networks = public
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges = 10000:100000
[securitygroup]
enable_security_group = True
enable_ipset = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[agent]
l2_population=True
polling_interval=2
arp_responder=True
tunnel_types=vxlan
enable_distributed_routing = True
[ovs]
enable_tunneling=True
integration_bridge=br-int
local_ip=10.60.0.3
tunnel_bridge=br-tun
bridge_mappings=public:public