Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig br-ex Link encap:Ethernet HWaddr 00:22:64:9b:38:46 inet addr:10.10.12.7 Bcast:10.10.12.255 Mask:255.255.255.0 inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1 RX packets:7047 errors:0 dropped:3 overruns:0 frame:0 TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:510845 (510.8 KB) TX bytes:896241 (896.2 KB)

br-int Link encap:Ethernet HWaddr 52:d5:65:a9:ef:40 inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link UP BROADCAST RUNNING MTU:1500 Metric:1 RX packets:79 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:8257 (8.2 KB) TX bytes:648 (648.0 B)

br-tun Link encap:Ethernet HWaddr fe:da:ad:8e:fc:43 inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link UP BROADCAST RUNNING MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:648 (648.0 B)

eth0 Link encap:Ethernet HWaddr 00:22:64:9b:38:46 inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:7128 errors:0 dropped:41 overruns:0 frame:0 TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:656883 (656.8 KB) TX bytes:936551 (936.5 KB)

eth1 Link encap:Ethernet HWaddr 00:22:64:9b:58:8a inet addr:10.10.13.231 Bcast:10.10.13.255 Mask:255.255.255.0 UP BROADCAST PROMISC MULTICAST MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)

lo Link encap:Local Loopback inet addr:127.0.0.1 Mask:255.0.0.0 inet6 addr: ::1/128 Scope:Host UP LOOPBACK RUNNING MTU:65536 Metric:1 RX packets:65593 errors:0 dropped:0 overruns:0 frame:0 TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:18543627 (18.5 MB) TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet HWaddr 7e:98:a3:42:eb:4f inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:47 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:5143 (5.1 KB) TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet HWaddr 7e:98:a3:42:eb:4f inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1 RX packets:53 errors:0 dropped:0 overruns:0 frame:0 TX packets:71 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:4358 (4.3 KB) TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet HWaddr 56:eb:18:2a:30:88 inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1 RX packets:71 errors:0 dropped:0 overruns:0 frame:0 TX packets:53 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:8083 (8.0 KB) TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet HWaddr fe:16:3e:6c:b6:ab inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:56 errors:0 dropped:0 overruns:0 frame:0 TX packets:59 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:500 RX bytes:6877 (6.8 KB) TX bytes:4826 (4.8 KB)

virbr0 Link encap:Ethernet HWaddr 3e:85:c4:b6:37:62 inet addr:192.168.122.1 Bcast:192.168.122.255 Mask:255.255.255.0 UP BROADCAST MULTICAST MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)

root@qa-openstack01:~# neutron agent-list +--------------------------------------+--------------------+----------------+-------+----------------+ | id | agent_type | host | alive | admin_state_up | +--------------------------------------+--------------------+----------------+-------+----------------+ | 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-) | True | | 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent | qa-openstack01 | :-) | True | | 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent | qa-openstack01 | :-) | True | | e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent | qa-openstack01 | :-) | True | +--------------------------------------+--------------------+----------------+-------+----------------+

root@qa-openstack01:~# ovs-vsctl show 72361d20-f343-469f-842c-8f09c2cf1058 Bridge br-ex Port br-ex Interface br-ex type: internal Port "eth0" Interface "eth0" Port "qg-6119ec76-62" Interface "qg-6119ec76-62" type: internal Bridge br-tun Port patch-int Interface patch-int type: patch options: {peer=patch-tun} Port br-tun Interface br-tun type: internal Bridge br-int fail_mode: secure Port "qr-20562557-85" tag: 1 Interface "qr-20562557-85" type: internal Port br-int Interface br-int type: internal Port "qvoa51d00a3-ca" tag: 1 Interface "qvoa51d00a3-ca" Port patch-tun Interface patch-tun type: patch options: {peer=patch-int} Port "tap10894d65-75" tag: 1 Interface "tap10894d65-75" type: internal ovs_version: "2.0.2"

root@qa-openstack01:~# brctl show bridge name bridge id STP enabled interfaces br-ex 8000.0022649b3846 no eth0 qbra51d00a3-ca 8000.7e98a342eb4f no qvba51d00a3-ca tapa51d00a3-ca virbr0 8000.000000000000 yes

root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig lo Link encap:Local Loopback inet addr:127.0.0.1 Mask:255.0.0.0 inet6 addr: ::1/128 Scope:Host UP LOOPBACK RUNNING MTU:65536 Metric:1 RX packets:25 errors:0 dropped:0 overruns:0 frame:0 TX packets:25 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:2467 (2.4 KB) TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet HWaddr fa:16:3e:7d:ae:bd inet addr:10.10.12.231 Bcast:10.10.12.255 Mask:255.255.255.0 inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link UP BROADCAST RUNNING MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:39 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet HWaddr fa:16:3e:10:04:03 inet addr:172.16.100.1 Bcast:172.16.100.255 Mask:255.255.255.0 inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link UP BROADCAST RUNNING MTU:1500 Metric:1 RX packets:60 errors:0 dropped:0 overruns:0 frame:0 TX packets:46 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:7211 (7.2 KB) TX bytes:3236 (3.2 KB)

root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn Kernel IP routing table Destination Gateway Genmask Flags MSS Window irtt Iface 0.0.0.0 10.10.12.1 0.0.0.0 UG 0 0 0 qg-6119ec76-62 10.10.12.0 0.0.0.0 255.255.255.0 U 0 0 0 qg-6119ec76-62 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 qr-20562557-85

root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231 PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data. 64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms ^C --- 10.10.12.231 ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms

root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3 PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data. 64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms ^C --- 172.16.100.3 ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms

root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com NOTHING...

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

KB) br-int Link encap:Ethernet HWaddr 52:d5:65:a9:ef:40 inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link UP BROADCAST RUNNING MTU:1500 Metric:1 RX packets:79 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:8257 (8.2 KB) TX bytes:648 (648.0 B)

B) br-tun Link encap:Ethernet HWaddr fe:da:ad:8e:fc:43 inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link UP BROADCAST RUNNING MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:648 (648.0 B)

B) eth0 Link encap:Ethernet HWaddr 00:22:64:9b:38:46 inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:7128 errors:0 dropped:41 overruns:0 frame:0 TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:656883 (656.8 KB) TX bytes:936551 (936.5 KB)

KB) eth1 Link encap:Ethernet HWaddr 00:22:64:9b:58:8a inet addr:10.10.13.231 Bcast:10.10.13.255 Mask:255.255.255.0 UP BROADCAST PROMISC MULTICAST MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)

B) lo Link encap:Local Loopback inet addr:127.0.0.1 Mask:255.0.0.0 inet6 addr: ::1/128 Scope:Host UP LOOPBACK RUNNING MTU:65536 Metric:1 RX packets:65593 errors:0 dropped:0 overruns:0 frame:0 TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:18543627 (18.5 MB) TX bytes:18543627 (18.5 MB)

MB) qbra51d00a3-ca Link encap:Ethernet HWaddr 7e:98:a3:42:eb:4f inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:47 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:5143 (5.1 KB) TX bytes:648 (648.0 B)

B) qvba51d00a3-ca Link encap:Ethernet HWaddr 7e:98:a3:42:eb:4f inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1 RX packets:53 errors:0 dropped:0 overruns:0 frame:0 TX packets:71 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:4358 (4.3 KB) TX bytes:8083 (8.0 KB)

KB) qvoa51d00a3-ca Link encap:Ethernet HWaddr 56:eb:18:2a:30:88 inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1 RX packets:71 errors:0 dropped:0 overruns:0 frame:0 TX packets:53 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:8083 (8.0 KB) TX bytes:4358 (4.3 KB)

KB) tapa51d00a3-ca Link encap:Ethernet HWaddr fe:16:3e:6c:b6:ab inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:56 errors:0 dropped:0 overruns:0 frame:0 TX packets:59 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:500 RX bytes:6877 (6.8 KB) TX bytes:4826 (4.8 KB)

KB) virbr0 Link encap:Ethernet HWaddr 3e:85:c4:b6:37:62 inet addr:192.168.122.1 Bcast:192.168.122.255 Mask:255.255.255.0 UP BROADCAST MULTICAST MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)

B) root@qa-openstack01:~# neutron agent-list +--------------------------------------+--------------------+----------------+-------+----------------+ | id | agent_type | host | alive | admin_state_up | +--------------------------------------+--------------------+----------------+-------+----------------+ | 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-) | True | | 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent | qa-openstack01 | :-) | True | | 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent | qa-openstack01 | :-) | True | | e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent | qa-openstack01 | :-) | True | +--------------------------------------+--------------------+----------------+-------+----------------+

+--------------------------------------+--------------------+----------------+-------+----------------+ root@qa-openstack01:~# ovs-vsctl show 72361d20-f343-469f-842c-8f09c2cf1058 Bridge br-ex Port br-ex Interface br-ex type: internal Port "eth0" Interface "eth0" Port "qg-6119ec76-62" Interface "qg-6119ec76-62" type: internal Bridge br-tun Port patch-int Interface patch-int type: patch options: {peer=patch-tun} Port br-tun Interface br-tun type: internal Bridge br-int fail_mode: secure Port "qr-20562557-85" tag: 1 Interface "qr-20562557-85" type: internal Port br-int Interface br-int type: internal Port "qvoa51d00a3-ca" tag: 1 Interface "qvoa51d00a3-ca" Port patch-tun Interface patch-tun type: patch options: {peer=patch-int} Port "tap10894d65-75" tag: 1 Interface "tap10894d65-75" type: internal ovs_version: "2.0.2"

"2.0.2" root@qa-openstack01:~# brctl show bridge name bridge id STP enabled interfaces br-ex 8000.0022649b3846 no eth0 qbra51d00a3-ca 8000.7e98a342eb4f no qvba51d00a3-ca tapa51d00a3-ca virbr0 8000.000000000000 yes

yes root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig lo Link encap:Local Loopback inet addr:127.0.0.1 Mask:255.0.0.0 inet6 addr: ::1/128 Scope:Host UP LOOPBACK RUNNING MTU:65536 Metric:1 RX packets:25 errors:0 dropped:0 overruns:0 frame:0 TX packets:25 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:2467 (2.4 KB) TX bytes:2467 (2.4 KB)

KB) qg-6119ec76-62 Link encap:Ethernet HWaddr fa:16:3e:7d:ae:bd inet addr:10.10.12.231 Bcast:10.10.12.255 Mask:255.255.255.0 inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link UP BROADCAST RUNNING MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:39 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:1998 (1.9 KB)

KB) qr-20562557-85 Link encap:Ethernet HWaddr fa:16:3e:10:04:03 inet addr:172.16.100.1 Bcast:172.16.100.255 Mask:255.255.255.0 inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link UP BROADCAST RUNNING MTU:1500 Metric:1 RX packets:60 errors:0 dropped:0 overruns:0 frame:0 TX packets:46 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:7211 (7.2 KB) TX bytes:3236 (3.2 KB)

KB) root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn Kernel IP routing table Destination Gateway Genmask Flags MSS Window irtt Iface 0.0.0.0 10.10.12.1 0.0.0.0 UG 0 0 0 qg-6119ec76-62 10.10.12.0 0.0.0.0 255.255.255.0 U 0 0 0 qg-6119ec76-62 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 qr-20562557-85

qr-20562557-85 root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231 PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data. 64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms ^C --- 10.10.12.231 ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms

ms root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3 PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data. 64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms ^C --- 172.16.100.3 ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms

ms root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com NOTHING...

NOTHING...

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below: root@qa-openstack01:~# nova secgroup-list-rules default +-------------+-----------+---------+----------+--------------+ | IP Protocol | From Port | To Port | IP Range | Source Group | +-------------+-----------+---------+----------+--------------+ | | | | | default | | | | | | default | +-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default +-------------+-----------+---------+-----------+--------------+ | IP Protocol | From Port | To Port | IP Range | Source Group | +-------------+-----------+---------+-----------+--------------+ | | | | | default | | tcp | 1 | 65535 | 0.0.0.0/0 | | | udp | 1 | 65535 | 0.0.0.0/0 | | | icmp | -1 | -1 | 0.0.0.0/0 | | | | | | | default | +-------------+-----------+---------+-----------+--------------+

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below: root@qa-openstack01:~# nova secgroup-list-rules default +-------------+-----------+---------+----------+--------------+ | IP Protocol | From Port | To Port | IP Range | Source Group | +-------------+-----------+---------+----------+--------------+ | | | | | default | | | | | | default | +-------------+-----------+---------+----------+--------------+

+-------------+-----------+---------+----------+--------------+ Adding further rules doesn't make any difference.

difference. root@qa-openstack01:~# nova secgroup-list-rules default +-------------+-----------+---------+-----------+--------------+ | IP Protocol | From Port | To Port | IP Range | Source Group | +-------------+-----------+---------+-----------+--------------+ | | | | | default | | tcp | 1 | 65535 | 0.0.0.0/0 | | | udp | 1 | 65535 | 0.0.0.0/0 | | | icmp | -1 | -1 | 0.0.0.0/0 | | | | | | | default | +-------------+-----------+---------+-----------+--------------+

+-------------+-----------+---------+-----------+--------------+

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table

Destination Gateway Genmask Flags MSS Window irtt Iface 0.0.0.0 10.10.12.1 0.0.0.0 UG 0 0 0 qg-6119ec76-62 10.10.12.0 0.0.0.0 255.255.255.0 U 0 0 0 qg-6119ec76-62 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 qr-20562557-85 qr-20562557-85

root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...



Responding to request below:
root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+



Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table

Destination Gateway Genmask Flags MSS Window irtt Iface 0.0.0.0 10.10.12.1 0.0.0.0 UG 0 0 0 qg-6119ec76-62 10.10.12.0 0.0.0.0 255.255.255.0 U 0 0 0 qg-6119ec76-62 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 qr-20562557-85

root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...



Responding to request below:
root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+
 

Response to question below: below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table

Destination Gateway Genmask Flags MSS Window irtt Iface 0.0.0.0 10.10.12.1 0.0.0.0 UG 0 0 0 qg-6119ec76-62 10.10.12.0 0.0.0.0 255.255.255.0 U 0 0 0 qg-6119ec76-62 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 qr-20562557-85

root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...
 

Responding to request below: root@qa-openstack01:~# nova secgroup-list-rules default +-------------+-----------+---------+----------+--------------+ | IP Protocol | From Port | To Port | IP Range | Source Group | +-------------+-----------+---------+----------+--------------+ | | | | | default | | | | | | default | +-------------+-----------+---------+----------+--------------+ +-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table

Destination Gateway Genmask Flags MSS Window irtt Iface 0.0.0.0 10.10.12.1 0.0.0.0 UG 0 0 0 qg-6119ec76-62 10.10.12.0 0.0.0.0 255.255.255.0 U 0 0 0 qg-6119ec76-62 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 qr-20562557-85

root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below: below:

root@qa-openstack01:~# nova secgroup-list-rules default
 +-------------+-----------+---------+----------+--------------+
 | IP Protocol | From Port | To Port | IP Range | Source Group |
 +-------------+-----------+---------+----------+--------------+
 |             |           |         |          | default      |
 |             |           |         |          | default      |
    +-------------+-----------+---------+----------+--------------+

+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
 Kernel IP routing table

Destination Gateway Genmask Flags MSS Window irtt Iface 0.0.0.0 10.10.12.1 0.0.0.0 UG 0 0 0 qg-6119ec76-62 10.10.12.0 0.0.0.0 255.255.255.0 U 0 0 0 qg-6119ec76-62 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 qr-20562557-85

qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
 Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing response below. root@qa-openstack01:~# iptables-save # Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014 *mangle :PREROUTING ACCEPT [168487:43297540] :INPUT ACCEPT [164770:43133512] :FORWARD ACCEPT [128:24606] :OUTPUT ACCEPT [165322:43377297] :POSTROUTING ACCEPT [165446:43401743] :nova-api-POSTROUTING - [0:0] :nova-network-POSTROUTING - [0:0] -A POSTROUTING -j nova-network-POSTROUTING -A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill -A POSTROUTING -j nova-api-POSTROUTING COMMIT # Completed on Fri Sep 19 18:16:05 2014 # Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014 *nat :PREROUTING ACCEPT [3690:151190] :INPUT ACCEPT [63:3164] :OUTPUT ACCEPT [570:34879] :POSTROUTING ACCEPT [586:37215] :neutron-openvswi-OUTPUT - [0:0] :neutron-openvswi-POSTROUTING - [0:0] :neutron-openvswi-PREROUTING - [0:0] :neutron-openvswi-float-snat - [0:0] :neutron-openvswi-snat - [0:0] :neutron-postrouting-bottom - [0:0] :nova-api-OUTPUT - [0:0] :nova-api-POSTROUTING - [0:0] :nova-api-PREROUTING - [0:0] :nova-api-float-snat - [0:0] :nova-api-snat - [0:0] :nova-network-OUTPUT - [0:0] :nova-network-POSTROUTING - [0:0] :nova-network-PREROUTING - [0:0] :nova-network-float-snat - [0:0] :nova-network-snat - [0:0] :nova-postrouting-bottom - [0:0] -A PREROUTING -j neutron-openvswi-PREROUTING -A PREROUTING -j nova-network-PREROUTING -A PREROUTING -j nova-api-PREROUTING -A OUTPUT -j neutron-openvswi-OUTPUT -A OUTPUT -j nova-network-OUTPUT -A OUTPUT -j nova-api-OUTPUT -A POSTROUTING -j neutron-openvswi-POSTROUTING -A POSTROUTING -j neutron-postrouting-bottom -A POSTROUTING -j nova-network-POSTROUTING -A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN -A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN -A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535 -A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535 -A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE -A POSTROUTING -j nova-api-POSTROUTING -A POSTROUTING -j nova-postrouting-bottom -A neutron-openvswi-snat -j neutron-openvswi-float-snat -A neutron-postrouting-bottom -j neutron-openvswi-snat -A nova-api-snat -j nova-api-float-snat -A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775 -A nova-network-snat -j nova-network-float-snat -A nova-postrouting-bottom -j nova-network-snat -A nova-postrouting-bottom -j nova-api-snat COMMIT # Completed on Fri Sep 19 18:16:05 2014 # Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014 *filter :INPUT ACCEPT [164766:43133258] :FORWARD ACCEPT [10:3380] :OUTPUT ACCEPT [165318:43377043] :neutron-filter-top - [0:0] :neutron-openvswi-FORWARD - [0:0] :neutron-openvswi-INPUT - [0:0] :neutron-openvswi-OUTPUT - [0:0] :neutron-openvswi-i3338a6c4-5 - [0:0] :neutron-openvswi-ia51d00a3-c - [0:0] :neutron-openvswi-local - [0:0] :neutron-openvswi-o3338a6c4-5 - [0:0] :neutron-openvswi-oa51d00a3-c - [0:0] :neutron-openvswi-s3338a6c4-5 - [0:0] :neutron-openvswi-sa51d00a3-c - [0:0] :neutron-openvswi-sg-chain - [0:0] :neutron-openvswi-sg-fallback - [0:0] :nova-api-FORWARD - [0:0] :nova-api-INPUT - [0:0] :nova-api-OUTPUT - [0:0] :nova-api-local - [0:0] :nova-filter-top - [0:0] :nova-network-FORWARD - [0:0] :nova-network-INPUT - [0:0] :nova-network-OUTPUT - [0:0] :nova-network-local - [0:0] -A INPUT -j neutron-openvswi-INPUT -A INPUT -j nova-network-INPUT -A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT -A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT -A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT -A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT -A INPUT -j nova-api-INPUT -A FORWARD -j neutron-filter-top -A FORWARD -j neutron-openvswi-FORWARD -A FORWARD -j nova-filter-top -A FORWARD -j nova-network-FORWARD -A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT -A FORWARD -i virbr0 -o virbr0 -j ACCEPT -A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable -A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable -A FORWARD -j nova-api-FORWARD -A OUTPUT -j neutron-filter-top -A OUTPUT -j neutron-openvswi-OUTPUT -A OUTPUT -j nova-filter-top -A OUTPUT -j nova-network-OUTPUT -A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT -A OUTPUT -j nova-api-OUTPUT -A neutron-filter-top -j neutron-openvswi-local -A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain -A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain -A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain -A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain -A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5 -A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c -A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP -A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN -A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN -A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN -A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback -A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP -A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN -A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN -A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN -A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback -A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN -A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5 -A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP -A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP -A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN -A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN -A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback -A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN -A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c -A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP -A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP -A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN -A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN -A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN -A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback -A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN -A neutron-openvswi-s3338a6c4-5 -j DROP -A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN -A neutron-openvswi-sa51d00a3-c -j DROP -A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5 -A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5 -A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c -A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c -A neutron-openvswi-sg-chain -j ACCEPT -A neutron-openvswi-sg-fallback -j DROP -A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT -A nova-filter-top -j nova-network-local -A nova-filter-top -j nova-api-local COMMIT

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing response below. below.

root@qa-openstack01:~# iptables-save
 # Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
 *mangle
 :PREROUTING ACCEPT [168487:43297540]
 :INPUT ACCEPT [164770:43133512]
 :FORWARD ACCEPT [128:24606]
 :OUTPUT ACCEPT [165322:43377297]
 :POSTROUTING ACCEPT [165446:43401743]
 :nova-api-POSTROUTING - [0:0]
 :nova-network-POSTROUTING - [0:0]
 -A POSTROUTING -j nova-network-POSTROUTING
 -A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
 -A POSTROUTING -j nova-api-POSTROUTING
 COMMIT
 # Completed on Fri Sep 19 18:16:05 2014
 # Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
 *nat
 :PREROUTING ACCEPT [3690:151190]
 :INPUT ACCEPT [63:3164]
 :OUTPUT ACCEPT [570:34879]
 :POSTROUTING ACCEPT [586:37215]
 :neutron-openvswi-OUTPUT - [0:0]
 :neutron-openvswi-POSTROUTING - [0:0]
 :neutron-openvswi-PREROUTING - [0:0]
 :neutron-openvswi-float-snat - [0:0]
 :neutron-openvswi-snat - [0:0]
 :neutron-postrouting-bottom - [0:0]
 :nova-api-OUTPUT - [0:0]
 :nova-api-POSTROUTING - [0:0]
 :nova-api-PREROUTING - [0:0]
 :nova-api-float-snat - [0:0]
 :nova-api-snat - [0:0]
 :nova-network-OUTPUT - [0:0]
 :nova-network-POSTROUTING - [0:0]
 :nova-network-PREROUTING - [0:0]
 :nova-network-float-snat - [0:0]
 :nova-network-snat - [0:0]
 :nova-postrouting-bottom - [0:0]
 -A PREROUTING -j neutron-openvswi-PREROUTING
 -A PREROUTING -j nova-network-PREROUTING
 -A PREROUTING -j nova-api-PREROUTING
 -A OUTPUT -j neutron-openvswi-OUTPUT
 -A OUTPUT -j nova-network-OUTPUT
 -A OUTPUT -j nova-api-OUTPUT
 -A POSTROUTING -j neutron-openvswi-POSTROUTING
 -A POSTROUTING -j neutron-postrouting-bottom
 -A POSTROUTING -j nova-network-POSTROUTING
 -A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
 -A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
 -A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
 -A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
 -A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
 -A POSTROUTING -j nova-api-POSTROUTING
 -A POSTROUTING -j nova-postrouting-bottom
 -A neutron-openvswi-snat -j neutron-openvswi-float-snat
 -A neutron-postrouting-bottom -j neutron-openvswi-snat
 -A nova-api-snat -j nova-api-float-snat
 -A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
 -A nova-network-snat -j nova-network-float-snat
 -A nova-postrouting-bottom -j nova-network-snat
 -A nova-postrouting-bottom -j nova-api-snat
 COMMIT
 # Completed on Fri Sep 19 18:16:05 2014
 # Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
 *filter
 :INPUT ACCEPT [164766:43133258]
 :FORWARD ACCEPT [10:3380]
 :OUTPUT ACCEPT [165318:43377043]
 :neutron-filter-top - [0:0]
 :neutron-openvswi-FORWARD - [0:0]
 :neutron-openvswi-INPUT - [0:0]
 :neutron-openvswi-OUTPUT - [0:0]
 :neutron-openvswi-i3338a6c4-5 - [0:0]
 :neutron-openvswi-ia51d00a3-c - [0:0]
 :neutron-openvswi-local - [0:0]
 :neutron-openvswi-o3338a6c4-5 - [0:0]
 :neutron-openvswi-oa51d00a3-c - [0:0]
 :neutron-openvswi-s3338a6c4-5 - [0:0]
 :neutron-openvswi-sa51d00a3-c - [0:0]
 :neutron-openvswi-sg-chain - [0:0]
 :neutron-openvswi-sg-fallback - [0:0]
 :nova-api-FORWARD - [0:0]
 :nova-api-INPUT - [0:0]
 :nova-api-OUTPUT - [0:0]
 :nova-api-local - [0:0]
 :nova-filter-top - [0:0]
 :nova-network-FORWARD - [0:0]
 :nova-network-INPUT - [0:0]
 :nova-network-OUTPUT - [0:0]
 :nova-network-local - [0:0]
 -A INPUT -j neutron-openvswi-INPUT
 -A INPUT -j nova-network-INPUT
 -A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
 -A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
 -A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
 -A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
 -A INPUT -j nova-api-INPUT
 -A FORWARD -j neutron-filter-top
 -A FORWARD -j neutron-openvswi-FORWARD
 -A FORWARD -j nova-filter-top
 -A FORWARD -j nova-network-FORWARD
 -A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
 -A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
 -A FORWARD -i virbr0 -o virbr0 -j ACCEPT
 -A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
 -A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
 -A FORWARD -j nova-api-FORWARD
 -A OUTPUT -j neutron-filter-top
 -A OUTPUT -j neutron-openvswi-OUTPUT
 -A OUTPUT -j nova-filter-top
 -A OUTPUT -j nova-network-OUTPUT
 -A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
 -A OUTPUT -j nova-api-OUTPUT
 -A neutron-filter-top -j neutron-openvswi-local
 -A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
 -A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
 -A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
 -A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
 -A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
 -A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
 -A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
 -A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
 -A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
 -A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
 -A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
 -A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
 -A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
 -A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
 -A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
 -A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
 -A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
 -A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
 -A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
 -A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
 -A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
 -A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
 -A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
 -A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
 -A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
 -A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
 -A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
 -A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
 -A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
 -A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
 -A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
 -A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
 -A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
 -A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
 -A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
 -A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
 -A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
 -A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
 -A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
 -A neutron-openvswi-s3338a6c4-5 -j DROP
 -A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
 -A neutron-openvswi-sa51d00a3-c -j DROP
 -A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
 -A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
 -A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
 -A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
 -A neutron-openvswi-sg-chain -j ACCEPT
 -A neutron-openvswi-sg-fallback -j DROP
 -A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
 -A nova-filter-top -j nova-network-local
 -A nova-filter-top -j nova-api-local
    COMMIT

COMMIT

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing response iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Why can my instances not ping host & vise versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination

Why can my instances not ping host & vise vice versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination

Why can my instances not ping host & vice versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination






root@qa-openstack01:~# iptables-save | grep 8775
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
<- this is not shown in your setup but is in mine.
-A INPUT -p tcp -m multiport --dports 8775 -m comment --comment "001 nova_metadata incoming" -j ACCEPT
<- this was not shown i mine so I added to match yours. Seems to make no difference.
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT


root@qa-openstack01:~# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      2127/python


root@qa-openstack01:~# ps -ef |grep 2127
nova      2127     1  0 12:32 ?        00:00:02 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2450  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2451  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2452  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2454  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2657  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2658  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2662  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2664  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2746  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2751  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2758  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2765  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
root      4335  4036  0 12:36 pts/0    00:00:00 grep --color=auto 2127


root@qa-openstack01:~# ip netns
qrouter-5111d40f-3afc-4e2f-ab74-3186f8584971
qdhcp-7e2165c0-b354-42b1-aa85-b4733fe1d1d2


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3693/python


root@qa-openstack01:~# ps -ef| grep 3693
root      3693     1  0 12:33 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var    /lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron  /metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log dir=/var/log/neutron
root      4511  4036  0 12:38 pts/0    00:00:00 grep --color=auto 3693


I noticed that some neutron processes are are not run by neutron.  Is this relevant?

root@qa-openstack01:~# ps -ef |grep neutron
neutron   2022     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-metadata-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/metadata_agent.ini --log-file=/var/log/neutron/metadata-agent.log
neutron   2024     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-openvswitch-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/plugins/ml2/ml2_conf.ini --log-file=/var/log/neutron/openvswitch-agent.log
neutron   2031     1  0 13:36 ?        00:00:01 /usr/bin/python /usr/bin/neutron-server --config-file /etc/neutron/neutron.conf --log-file /var/log/neutron/server.log --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
neutron   2208     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-dhcp-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/dhcp_agent.ini --log-file=/var/log/neutron/dhcp-agent.log
neutron   2214     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-l3-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/l3_agent.ini --config-file=/etc/neutron/fwaas_driver.ini --log-file=/var/log/neutron/l3-agent.log
root      3048  2024  0 13:36 ?        00:00:00 sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
root      3050  3048  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
nobody    3529     1  0 13:37 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --bind-interfaces --interface=tap123b69fd-3c --except-interface=lo --pid-file=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/host --addn-hosts=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/addn_hosts --dhcp-optsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/opts --leasefile-ro --dhcp-range=set:tag0,172.16.100.0,static,86400s --dhcp-lease-max=256 --conf-file=/etc/neutron/dnsmasq-neutron.conf --domain=openstacklocal
root      3587     1  0 13:37 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log-dir=/var/log/neutron
root      4150  3239  0 13:39 pts/0    00:00:00 grep --color=auto neutron
root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.

Why can my instances not ping host & vice versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination






root@qa-openstack01:~# iptables-save | grep 8775
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
<- this is not shown in your setup but is in mine.
-A INPUT -p tcp -m multiport --dports 8775 -m comment --comment "001 nova_metadata incoming" -j ACCEPT
<- this was not shown i mine so I added to match yours. Seems to make no difference.
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT


root@qa-openstack01:~# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      2127/python


root@qa-openstack01:~# ps -ef |grep 2127
nova      2127     1  0 12:32 ?        00:00:02 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2450  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2451  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2452  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2454  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2657  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2658  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2662  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2664  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2746  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2751  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2758  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2765  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
root      4335  4036  0 12:36 pts/0    00:00:00 grep --color=auto 2127


root@qa-openstack01:~# ip netns
qrouter-5111d40f-3afc-4e2f-ab74-3186f8584971
qdhcp-7e2165c0-b354-42b1-aa85-b4733fe1d1d2


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3693/python


root@qa-openstack01:~# ps -ef| grep 3693
root      3693     1  0 12:33 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var    /lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron  /metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log dir=/var/log/neutron
root      4511  4036  0 12:38 pts/0    00:00:00 grep --color=auto 3693


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.


I noticed that some neutron processes are are not run by neutron.  Is this relevant?

root@qa-openstack01:~# ps -ef |grep neutron
neutron   2022     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-metadata-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/metadata_agent.ini --log-file=/var/log/neutron/metadata-agent.log
neutron   2024     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-openvswitch-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/plugins/ml2/ml2_conf.ini --log-file=/var/log/neutron/openvswitch-agent.log
neutron   2031     1  0 13:36 ?        00:00:01 /usr/bin/python /usr/bin/neutron-server --config-file /etc/neutron/neutron.conf --log-file /var/log/neutron/server.log --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
neutron   2208     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-dhcp-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/dhcp_agent.ini --log-file=/var/log/neutron/dhcp-agent.log
neutron   2214     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-l3-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/l3_agent.ini --config-file=/etc/neutron/fwaas_driver.ini --log-file=/var/log/neutron/l3-agent.log
root      3048  2024  0 13:36 ?        00:00:00 sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
root      3050  3048  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
nobody    3529     1  0 13:37 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --bind-interfaces --interface=tap123b69fd-3c --except-interface=lo --pid-file=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/host --addn-hosts=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/addn_hosts --dhcp-optsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/opts --leasefile-ro --dhcp-range=set:tag0,172.16.100.0,static,86400s --dhcp-lease-max=256 --conf-file=/etc/neutron/dnsmasq-neutron.conf --domain=openstacklocal
root      3587     1  0 13:37 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log-dir=/var/log/neutron
root      4150  3239  0 13:39 pts/0    00:00:00 grep --color=auto neutron
root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.

Why can my instances not ping host & vice versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination






root@qa-openstack01:~# iptables-save | grep 8775
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
<- this is not shown in your setup but is in mine.
-A INPUT -p tcp -m multiport --dports 8775 -m comment --comment "001 nova_metadata incoming" -j ACCEPT
<- this was not shown i mine so I added to match yours. Seems to make no difference.
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT


root@qa-openstack01:~# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      2127/python


root@qa-openstack01:~# ps -ef |grep 2127
nova      2127     1  0 12:32 ?        00:00:02 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2450  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2451  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2452  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2454  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2657  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2658  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2662  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2664  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2746  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2751  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2758  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2765  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
root      4335  4036  0 12:36 pts/0    00:00:00 grep --color=auto 2127


root@qa-openstack01:~# ip netns
qrouter-5111d40f-3afc-4e2f-ab74-3186f8584971
qdhcp-7e2165c0-b354-42b1-aa85-b4733fe1d1d2


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3693/python


root@qa-openstack01:~# ps -ef| grep 3693
root      3693     1  0 12:33 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var    /lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron  /metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log dir=/var/log/neutron
root      4511  4036  0 12:38 pts/0    00:00:00 grep --color=auto 3693


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.


I noticed that some neutron processes are are not run by neutron.  Is this relevant?

root@qa-openstack01:~# ps -ef |grep neutron
neutron   2022     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-metadata-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/metadata_agent.ini --log-file=/var/log/neutron/metadata-agent.log
neutron   2024     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-openvswitch-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/plugins/ml2/ml2_conf.ini --log-file=/var/log/neutron/openvswitch-agent.log
neutron   2031     1  0 13:36 ?        00:00:01 /usr/bin/python /usr/bin/neutron-server --config-file /etc/neutron/neutron.conf --log-file /var/log/neutron/server.log --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
neutron   2208     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-dhcp-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/dhcp_agent.ini --log-file=/var/log/neutron/dhcp-agent.log
neutron   2214     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-l3-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/l3_agent.ini --config-file=/etc/neutron/fwaas_driver.ini --log-file=/var/log/neutron/l3-agent.log
root      3048  2024  0 13:36 ?        00:00:00 sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
root      3050  3048  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
nobody    3529     1  0 13:37 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --bind-interfaces --interface=tap123b69fd-3c --except-interface=lo --pid-file=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/host --addn-hosts=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/addn_hosts --dhcp-optsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/opts --leasefile-ro --dhcp-range=set:tag0,172.16.100.0,static,86400s --dhcp-lease-max=256 --conf-file=/etc/neutron/dnsmasq-neutron.conf --domain=openstacklocal
root      3587     1  0 13:37 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log-dir=/var/log/neutron
root      4150  3239  0 13:39 pts/0    00:00:00 grep --color=auto neutron





I configured the setting as suggested.

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex
metadata_port = 9697
enable_metadata_proxy = True
router_delete_namespaces = False
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5


root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#|grep metadata
enabled_apis=ec2,osapi_compute,metadata
metadata_workers=2
metadata_listen = 0.0.0.0
metadata_listen_port = 8775
metadata_host = 10.10.12.7
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret

Alas, still not joy.

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 curl http://169.254.169.254
curl: (7) Failed to connect to 169.254.169.254 port 80: No route to host

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.063 ms

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.1
PING 10.10.12.1 (10.10.12.1) 56(84) bytes of data.
From 10.10.12.231 icmp_seq=1 Destination Host Unreachable

Why can my instances not ping host & vice versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination






root@qa-openstack01:~# iptables-save | grep 8775
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
<- this is not shown in your setup but is in mine.
-A INPUT -p tcp -m multiport --dports 8775 -m comment --comment "001 nova_metadata incoming" -j ACCEPT
<- this was not shown i mine so I added to match yours. Seems to make no difference.
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT


root@qa-openstack01:~# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      2127/python


root@qa-openstack01:~# ps -ef |grep 2127
nova      2127     1  0 12:32 ?        00:00:02 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2450  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2451  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2452  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2454  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2657  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2658  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2662  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2664  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2746  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2751  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2758  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2765  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
root      4335  4036  0 12:36 pts/0    00:00:00 grep --color=auto 2127


root@qa-openstack01:~# ip netns
qrouter-5111d40f-3afc-4e2f-ab74-3186f8584971
qdhcp-7e2165c0-b354-42b1-aa85-b4733fe1d1d2


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3693/python


root@qa-openstack01:~# ps -ef| grep 3693
root      3693     1  0 12:33 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var    /lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron  /metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log dir=/var/log/neutron
root      4511  4036  0 12:38 pts/0    00:00:00 grep --color=auto 3693


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.


I noticed that some neutron processes are are not run by neutron.  Is this relevant?

root@qa-openstack01:~# ps -ef |grep neutron
neutron   2022     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-metadata-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/metadata_agent.ini --log-file=/var/log/neutron/metadata-agent.log
neutron   2024     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-openvswitch-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/plugins/ml2/ml2_conf.ini --log-file=/var/log/neutron/openvswitch-agent.log
neutron   2031     1  0 13:36 ?        00:00:01 /usr/bin/python /usr/bin/neutron-server --config-file /etc/neutron/neutron.conf --log-file /var/log/neutron/server.log --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
neutron   2208     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-dhcp-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/dhcp_agent.ini --log-file=/var/log/neutron/dhcp-agent.log
neutron   2214     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-l3-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/l3_agent.ini --config-file=/etc/neutron/fwaas_driver.ini --log-file=/var/log/neutron/l3-agent.log
root      3048  2024  0 13:36 ?        00:00:00 sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
root      3050  3048  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
nobody    3529     1  0 13:37 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --bind-interfaces --interface=tap123b69fd-3c --except-interface=lo --pid-file=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/host --addn-hosts=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/addn_hosts --dhcp-optsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/opts --leasefile-ro --dhcp-range=set:tag0,172.16.100.0,static,86400s --dhcp-lease-max=256 --conf-file=/etc/neutron/dnsmasq-neutron.conf --domain=openstacklocal
root      3587     1  0 13:37 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log-dir=/var/log/neutron
root      4150  3239  0 13:39 pts/0    00:00:00 grep --color=auto neutron





I configured the setting as suggested.

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex
metadata_port = 9697
enable_metadata_proxy = True
router_delete_namespaces = False
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5


root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#|grep metadata
enabled_apis=ec2,osapi_compute,metadata
metadata_workers=2
metadata_listen = 0.0.0.0
metadata_listen_port = 8775
metadata_host = 10.10.12.7
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret

Alas, still not joy.

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 curl http://169.254.169.254
curl: (7) Failed to connect to 169.254.169.254 port 80: No route to host

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.063 ms

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.1
PING 10.10.12.1 (10.10.12.1) 56(84) bytes of data.
From 10.10.12.231 icmp_seq=1 Destination Host Unreachable

I see the following in openvswitch-agent.log:

2014-09-29 10:49:59.621 20872 ERROR neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Error while processing VIF ports
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Traceback (most recent call last):
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1322, in rpc_loop
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     port_info = self.scan_ports(reg_ports, updated_ports_copy)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 873, in scan_ports
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     cur_ports = self.int_br.get_vif_port_set()
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 330, in get_vif_port_set
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     port_names = self.get_port_name_list()
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 286, in get_port_name_list
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     res = self.run_vsctl(["list-ports", self.br_name], check_error=True)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 75, in run_vsctl
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     ctxt.reraise = False
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/openstack/common/excutils.py", line 82, in __exit__
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     six.reraise(self.type_, self.value, self.tb)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 68, in run_vsctl
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     return utils.execute(full_args, root_helper=self.root_helper)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/utils.py", line 76, in execute
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     raise RuntimeError(m)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent RuntimeError:
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Command: ['sudo', '/usr/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ovs-vsctl', '--timeout=10', 'list-ports', 'br-int']
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Exit code: 1
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Stdout: ''
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Stderr: '2014-09-29T09:49:59Z|00001|reconnect|WARN|unix:/var/run/openvswitch/db.sock: connection attempt failed (No such file or directory)\novs-vsctl: unix:/var/run/openvswitch/db.sock: database connection failed (No such file or directory)\n'
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent
2014-09-29 10:49:59.702 20872 ERROR neutron.agent.linux.ovs_lib [-] Unable to execute ['ovs-ofctl', 'dump-flows', 'br-int', 'table=22']. Exception:
Command: ['sudo', '/usr/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ovs-ofctl', 'dump-flows', 'br-int', 'table=22']
Exit code: 1
Stdout: ''
Stderr: 'ovs-ofctl: br-int is not a bridge or a socket\n'
2014-09-29 10:50:00.128 20872 ERROR neutron.agent.linux.ovs_lib [-] Unable to execute ['ovs-ofctl', 'del-flows',    'br-int']. Exception:
Command: ['sudo', '/usr/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ovs-ofctl', 'del-flows', 'br-int']
Exit code: 143
Stdout: ''
Stderr: ''


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 15c32013-90b7-4df5-b289-54688dabdf2b | DHCP agent         | qa-openstack01 | :-)   | True           |
| 3c29c923-a2cd-4258-bcc5-4166b02295f8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 3efb355b-e066-4b02-91bd-0544607e663a | L3 agent           | qa-openstack02 | :-)   | True           |
| 5ebd995e-896e-4b68-9bc8-22cdb2453496 | L3 agent           | qa-openstack01 | :-)   | True           |
| 66a85ad0-45c4-4573-a26f-8ba40cf08b63 | Open vSwitch agent | qa-openstack02 | :-)   | True           |
| b84181ea-fa04-4459-8068-bd4c887d1ecc | Metadata agent     | qa-openstack01 | :-)   | True           |
| de151f92-887d-4b1c-9ff1-29899184e4cd | Metadata agent     | qa-openstack02 | :-)   | True           |
| f14ec87e-472e-4d2d-b0ba-50217d92f82f | DHCP agent         | qa-openstack02 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+

Why can my instances not ping host & vice versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination






root@qa-openstack01:~# iptables-save | grep 8775
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
<- this is not shown in your setup but is in mine.
-A INPUT -p tcp -m multiport --dports 8775 -m comment --comment "001 nova_metadata incoming" -j ACCEPT
<- this was not shown i mine so I added to match yours. Seems to make no difference.
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT


root@qa-openstack01:~# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      2127/python


root@qa-openstack01:~# ps -ef |grep 2127
nova      2127     1  0 12:32 ?        00:00:02 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2450  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2451  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2452  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2454  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2657  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2658  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2662  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2664  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2746  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2751  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2758  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2765  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
root      4335  4036  0 12:36 pts/0    00:00:00 grep --color=auto 2127


root@qa-openstack01:~# ip netns
qrouter-5111d40f-3afc-4e2f-ab74-3186f8584971
qdhcp-7e2165c0-b354-42b1-aa85-b4733fe1d1d2


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3693/python


root@qa-openstack01:~# ps -ef| grep 3693
root      3693     1  0 12:33 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var    /lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron  /metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log dir=/var/log/neutron
root      4511  4036  0 12:38 pts/0    00:00:00 grep --color=auto 3693


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.


I noticed that some neutron processes are are not run by neutron.  Is this relevant?

root@qa-openstack01:~# ps -ef |grep neutron
neutron   2022     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-metadata-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/metadata_agent.ini --log-file=/var/log/neutron/metadata-agent.log
neutron   2024     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-openvswitch-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/plugins/ml2/ml2_conf.ini --log-file=/var/log/neutron/openvswitch-agent.log
neutron   2031     1  0 13:36 ?        00:00:01 /usr/bin/python /usr/bin/neutron-server --config-file /etc/neutron/neutron.conf --log-file /var/log/neutron/server.log --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
neutron   2208     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-dhcp-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/dhcp_agent.ini --log-file=/var/log/neutron/dhcp-agent.log
neutron   2214     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-l3-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/l3_agent.ini --config-file=/etc/neutron/fwaas_driver.ini --log-file=/var/log/neutron/l3-agent.log
root      3048  2024  0 13:36 ?        00:00:00 sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
root      3050  3048  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
nobody    3529     1  0 13:37 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --bind-interfaces --interface=tap123b69fd-3c --except-interface=lo --pid-file=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/host --addn-hosts=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/addn_hosts --dhcp-optsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/opts --leasefile-ro --dhcp-range=set:tag0,172.16.100.0,static,86400s --dhcp-lease-max=256 --conf-file=/etc/neutron/dnsmasq-neutron.conf --domain=openstacklocal
root      3587     1  0 13:37 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log-dir=/var/log/neutron
root      4150  3239  0 13:39 pts/0    00:00:00 grep --color=auto neutron





I configured the setting as suggested.

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex
metadata_port = 9697
enable_metadata_proxy = True
router_delete_namespaces = False
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5


root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#|grep metadata
enabled_apis=ec2,osapi_compute,metadata
metadata_workers=2
metadata_listen = 0.0.0.0
metadata_listen_port = 8775
metadata_host = 10.10.12.7
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret

Alas, still not joy.

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 curl http://169.254.169.254
curl: (7) Failed to connect to 169.254.169.254 port 80: No route to host

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.063 ms

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.1
PING 10.10.12.1 (10.10.12.1) 56(84) bytes of data.
From 10.10.12.231 icmp_seq=1 Destination Host Unreachable

I see the following in openvswitch-agent.log:

2014-09-29 10:49:59.621 20872 ERROR neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Error while processing VIF ports
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Traceback (most recent call last):
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1322, in rpc_loop
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     port_info = self.scan_ports(reg_ports, updated_ports_copy)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 873, in scan_ports
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     cur_ports = self.int_br.get_vif_port_set()
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 330, in get_vif_port_set
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     port_names = self.get_port_name_list()
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 286, in get_port_name_list
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     res = self.run_vsctl(["list-ports", self.br_name], check_error=True)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 75, in run_vsctl
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     ctxt.reraise = False
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/openstack/common/excutils.py", line 82, in __exit__
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     six.reraise(self.type_, self.value, self.tb)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 68, in run_vsctl
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     return utils.execute(full_args, root_helper=self.root_helper)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/utils.py", line 76, in execute
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     raise RuntimeError(m)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent RuntimeError:
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Command: ['sudo', '/usr/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ovs-vsctl', '--timeout=10', 'list-ports', 'br-int']
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Exit code: 1
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Stdout: ''
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Stderr: '2014-09-29T09:49:59Z|00001|reconnect|WARN|unix:/var/run/openvswitch/db.sock: connection attempt failed (No such file or directory)\novs-vsctl: unix:/var/run/openvswitch/db.sock: database connection failed (No such file or directory)\n'
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent
2014-09-29 10:49:59.702 20872 ERROR neutron.agent.linux.ovs_lib [-] Unable to execute ['ovs-ofctl', 'dump-flows', 'br-int', 'table=22']. Exception:
Command: ['sudo', '/usr/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ovs-ofctl', 'dump-flows', 'br-int', 'table=22']
Exit code: 1
Stdout: ''
Stderr: 'ovs-ofctl: br-int is not a bridge or a socket\n'
2014-09-29 10:50:00.128 20872 ERROR neutron.agent.linux.ovs_lib [-] Unable to execute ['ovs-ofctl', 'del-flows',    'br-int']. Exception:
Command: ['sudo', '/usr/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ovs-ofctl', 'del-flows', 'br-int']
Exit code: 143
Stdout: ''
Stderr: ''


I can run these commands manually, e.g.:

root@qa-openstack01:~# /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovs-ofctl dump-flows br-int table=22
NXST_FLOW reply (xid=0x4):
cookie=0x0, duration=595.775s, table=22, n_packets=0, n_bytes=0, idle_age=595, priority=0 actions=drop

Is there a permission thing going on here?

root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 15c32013-90b7-4df5-b289-54688dabdf2b | DHCP agent         | qa-openstack01 | :-)   | True           |
| 3c29c923-a2cd-4258-bcc5-4166b02295f8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 3efb355b-e066-4b02-91bd-0544607e663a | L3 agent           | qa-openstack02 | :-)   | True           |
| 5ebd995e-896e-4b68-9bc8-22cdb2453496 | L3 agent           | qa-openstack01 | :-)   | True           |
| 66a85ad0-45c4-4573-a26f-8ba40cf08b63 | Open vSwitch agent | qa-openstack02 | :-)   | True           |
| b84181ea-fa04-4459-8068-bd4c887d1ecc | Metadata agent     | qa-openstack01 | :-)   | True           |
| de151f92-887d-4b1c-9ff1-29899184e4cd | Metadata agent     | qa-openstack02 | :-)   | True           |
| f14ec87e-472e-4d2d-b0ba-50217d92f82f | DHCP agent         | qa-openstack02 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+

Why can my instances not ping host & vice versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination






root@qa-openstack01:~# iptables-save | grep 8775
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
<- this is not shown in your setup but is in mine.
-A INPUT -p tcp -m multiport --dports 8775 -m comment --comment "001 nova_metadata incoming" -j ACCEPT
<- this was not shown i mine so I added to match yours. Seems to make no difference.
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT


root@qa-openstack01:~# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      2127/python


root@qa-openstack01:~# ps -ef |grep 2127
nova      2127     1  0 12:32 ?        00:00:02 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2450  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2451  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2452  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2454  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2657  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2658  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2662  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2664  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2746  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2751  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2758  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2765  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
root      4335  4036  0 12:36 pts/0    00:00:00 grep --color=auto 2127


root@qa-openstack01:~# ip netns
qrouter-5111d40f-3afc-4e2f-ab74-3186f8584971
qdhcp-7e2165c0-b354-42b1-aa85-b4733fe1d1d2


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3693/python


root@qa-openstack01:~# ps -ef| grep 3693
root      3693     1  0 12:33 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var    /lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron  /metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log dir=/var/log/neutron
root      4511  4036  0 12:38 pts/0    00:00:00 grep --color=auto 3693


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.


I noticed that some neutron processes are are not run by neutron.  Is this relevant?

root@qa-openstack01:~# ps -ef |grep neutron
neutron   2022     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-metadata-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/metadata_agent.ini --log-file=/var/log/neutron/metadata-agent.log
neutron   2024     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-openvswitch-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/plugins/ml2/ml2_conf.ini --log-file=/var/log/neutron/openvswitch-agent.log
neutron   2031     1  0 13:36 ?        00:00:01 /usr/bin/python /usr/bin/neutron-server --config-file /etc/neutron/neutron.conf --log-file /var/log/neutron/server.log --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
neutron   2208     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-dhcp-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/dhcp_agent.ini --log-file=/var/log/neutron/dhcp-agent.log
neutron   2214     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-l3-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/l3_agent.ini --config-file=/etc/neutron/fwaas_driver.ini --log-file=/var/log/neutron/l3-agent.log
root      3048  2024  0 13:36 ?        00:00:00 sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
root      3050  3048  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
nobody    3529     1  0 13:37 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --bind-interfaces --interface=tap123b69fd-3c --except-interface=lo --pid-file=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/host --addn-hosts=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/addn_hosts --dhcp-optsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/opts --leasefile-ro --dhcp-range=set:tag0,172.16.100.0,static,86400s --dhcp-lease-max=256 --conf-file=/etc/neutron/dnsmasq-neutron.conf --domain=openstacklocal
root      3587     1  0 13:37 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log-dir=/var/log/neutron
root      4150  3239  0 13:39 pts/0    00:00:00 grep --color=auto neutron





I configured the setting as suggested.

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex
metadata_port = 9697
enable_metadata_proxy = True
router_delete_namespaces = False
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5


root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#|grep metadata
enabled_apis=ec2,osapi_compute,metadata
metadata_workers=2
metadata_listen = 0.0.0.0
metadata_listen_port = 8775
metadata_host = 10.10.12.7
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret

Alas, still not joy.

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 curl http://169.254.169.254
curl: (7) Failed to connect to 169.254.169.254 port 80: No route to host

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.063 ms

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.1
PING 10.10.12.1 (10.10.12.1) 56(84) bytes of data.
From 10.10.12.231 icmp_seq=1 Destination Host Unreachable

I see the following in openvswitch-agent.log:

2014-09-29 10:49:59.621 20872 ERROR neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Error while processing VIF ports
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Traceback (most recent call last):
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1322, in rpc_loop
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     port_info = self.scan_ports(reg_ports, updated_ports_copy)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 873, in scan_ports
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     cur_ports = self.int_br.get_vif_port_set()
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 330, in get_vif_port_set
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     port_names = self.get_port_name_list()
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 286, in get_port_name_list
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     res = self.run_vsctl(["list-ports", self.br_name], check_error=True)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 75, in run_vsctl
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     ctxt.reraise = False
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/openstack/common/excutils.py", line 82, in __exit__
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     six.reraise(self.type_, self.value, self.tb)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovs_lib.py", line 68, in run_vsctl
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     return utils.execute(full_args, root_helper=self.root_helper)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/utils.py", line 76, in execute
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     raise RuntimeError(m)
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent RuntimeError:
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Command: ['sudo', '/usr/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ovs-vsctl', '--timeout=10', 'list-ports', 'br-int']
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Exit code: 1
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Stdout: ''
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Stderr: '2014-09-29T09:49:59Z|00001|reconnect|WARN|unix:/var/run/openvswitch/db.sock: connection attempt failed (No such file or directory)\novs-vsctl: unix:/var/run/openvswitch/db.sock: database connection failed (No such file or directory)\n'
2014-09-29 10:49:59.621 20872 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent
2014-09-29 10:49:59.702 20872 ERROR neutron.agent.linux.ovs_lib [-] Unable to execute ['ovs-ofctl', 'dump-flows', 'br-int', 'table=22']. Exception:
Command: ['sudo', '/usr/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ovs-ofctl', 'dump-flows', 'br-int', 'table=22']
Exit code: 1
Stdout: ''
Stderr: 'ovs-ofctl: br-int is not a bridge or a socket\n'
2014-09-29 10:50:00.128 20872 ERROR neutron.agent.linux.ovs_lib [-] Unable to execute ['ovs-ofctl', 'del-flows',    'br-int']. Exception:
Command: ['sudo', '/usr/bin/neutron-rootwrap', '/etc/neutron/rootwrap.conf', 'ovs-ofctl', 'del-flows', 'br-int']
Exit code: 143
Stdout: ''
Stderr: ''


I can run these commands manually, e.g.:

root@qa-openstack01:~# /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovs-ofctl dump-flows br-int table=22
NXST_FLOW reply (xid=0x4):
cookie=0x0, duration=595.775s, table=22, n_packets=0, n_bytes=0, idle_age=595, priority=0 actions=drop

Is there a permission thing going on here?

root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 15c32013-90b7-4df5-b289-54688dabdf2b | DHCP agent         | qa-openstack01 | :-)   | True           |
| 3c29c923-a2cd-4258-bcc5-4166b02295f8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 3efb355b-e066-4b02-91bd-0544607e663a | L3 agent           | qa-openstack02 | :-)   | True           |
| 5ebd995e-896e-4b68-9bc8-22cdb2453496 | L3 agent           | qa-openstack01 | :-)   | True           |
| 66a85ad0-45c4-4573-a26f-8ba40cf08b63 | Open vSwitch agent | qa-openstack02 | :-)   | True           |
| b84181ea-fa04-4459-8068-bd4c887d1ecc | Metadata agent     | qa-openstack01 | :-)   | True           |
| de151f92-887d-4b1c-9ff1-29899184e4cd | Metadata agent     | qa-openstack02 | :-)   | True           |
| f14ec87e-472e-4d2d-b0ba-50217d92f82f | DHCP agent         | qa-openstack02 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+

Why can my instances not ping host & vice versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination






root@qa-openstack01:~# iptables-save | grep 8775
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
<- this is not shown in your setup but is in mine.
-A INPUT -p tcp -m multiport --dports 8775 -m comment --comment "001 nova_metadata incoming" -j ACCEPT
<- this was not shown i mine so I added to match yours. Seems to make no difference.
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT


root@qa-openstack01:~# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      2127/python


root@qa-openstack01:~# ps -ef |grep 2127
nova      2127     1  0 12:32 ?        00:00:02 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2450  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2451  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2452  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2454  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2657  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2658  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2662  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2664  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2746  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2751  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2758  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2765  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
root      4335  4036  0 12:36 pts/0    00:00:00 grep --color=auto 2127


root@qa-openstack01:~# ip netns
qrouter-5111d40f-3afc-4e2f-ab74-3186f8584971
qdhcp-7e2165c0-b354-42b1-aa85-b4733fe1d1d2


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3693/python


root@qa-openstack01:~# ps -ef| grep 3693
root      3693     1  0 12:33 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var    /lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron  /metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log dir=/var/log/neutron
root      4511  4036  0 12:38 pts/0    00:00:00 grep --color=auto 3693


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.


I noticed that some neutron processes are are not run by neutron.  Is this relevant?

root@qa-openstack01:~# ps -ef |grep neutron
neutron   2022     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-metadata-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/metadata_agent.ini --log-file=/var/log/neutron/metadata-agent.log
neutron   2024     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-openvswitch-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/plugins/ml2/ml2_conf.ini --log-file=/var/log/neutron/openvswitch-agent.log
neutron   2031     1  0 13:36 ?        00:00:01 /usr/bin/python /usr/bin/neutron-server --config-file /etc/neutron/neutron.conf --log-file /var/log/neutron/server.log --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
neutron   2208     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-dhcp-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/dhcp_agent.ini --log-file=/var/log/neutron/dhcp-agent.log
neutron   2214     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-l3-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/l3_agent.ini --config-file=/etc/neutron/fwaas_driver.ini --log-file=/var/log/neutron/l3-agent.log
root      3048  2024  0 13:36 ?        00:00:00 sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
root      3050  3048  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
nobody    3529     1  0 13:37 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --bind-interfaces --interface=tap123b69fd-3c --except-interface=lo --pid-file=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/host --addn-hosts=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/addn_hosts --dhcp-optsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/opts --leasefile-ro --dhcp-range=set:tag0,172.16.100.0,static,86400s --dhcp-lease-max=256 --conf-file=/etc/neutron/dnsmasq-neutron.conf --domain=openstacklocal
root      3587     1  0 13:37 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log-dir=/var/log/neutron
root      4150  3239  0 13:39 pts/0    00:00:00 grep --color=auto neutron





I configured the setting as suggested.

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex
metadata_port = 9697
enable_metadata_proxy = True
router_delete_namespaces = False
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5


root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#|grep metadata
enabled_apis=ec2,osapi_compute,metadata
metadata_workers=2
metadata_listen = 0.0.0.0
metadata_listen_port = 8775
metadata_host = 10.10.12.7
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret

Alas, still not joy.

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 curl http://169.254.169.254
curl: (7) Failed to connect to 169.254.169.254 port 80: No route to host

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.063 ms

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.1
PING 10.10.12.1 (10.10.12.1) 56(84) bytes of data.
From 10.10.12.231 icmp_seq=1 Destination Host Unreachable

I'm still periodically investigating the issue. I summary I cannot access beyond external interface of router (including host IP). Should the following that show DOWN be showing UP?

root@qa-openstack01:~# ovs-ofctl show br-ex
OFPT_FEATURES_REPLY (xid=0x2): dpid:00001e0dcb6e184a
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
 1(qg-416e130c-f8): addr:00:00:00:00:00:00
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

root@qa-openstack01:~# ovs-ofctl show br-int
OFPT_FEATURES_REPLY (xid=0x2): dpid:0000ba900a62a94c
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
 1(patch-tun): addr:3e:4a:1c:49:73:a8
 config:     0
 state:      0
 speed: 0 Mbps now, 0 Mbps max
 2(tap7ed05795-4c): addr:00:00:00:00:00:00
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
 3(qr-768d4f2c-eb): addr:00:00:00:00:00:00
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
 LOCAL(br-int): addr:06:fa:62:7e:84:d8
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

root@qa-openstack01:~# ovs-ofctl show br-tun
OFPT_FEATURES_REPLY (xid=0x2): dpid:0000ee2bec51d74e
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE

1(patch-int): addr:92:cf:64:e5:d8:aa config: 0 state: 0 speed: 0 Mbps now, 0 Mbps max LOCAL(br-tun): addr:ee:2b:ec:51:d7:4e config: 0 state: 0 speed: 0 Mbps now, 0 Mbps max OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

ip neighbor
10.10.12.1 dev br-ex lladdr d0:67:e5:af:ab:6c REACHABLE

Why can my instances not ping host & vice versa?

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination






root@qa-openstack01:~# iptables-save | grep 8775
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
<- this is not shown in your setup but is in mine.
-A INPUT -p tcp -m multiport --dports 8775 -m comment --comment "001 nova_metadata incoming" -j ACCEPT
<- this was not shown i mine so I added to match yours. Seems to make no difference.
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT


root@qa-openstack01:~# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      2127/python


root@qa-openstack01:~# ps -ef |grep 2127
nova      2127     1  0 12:32 ?        00:00:02 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2450  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2451  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2452  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2454  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2657  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2658  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2662  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2664  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2746  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2751  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2758  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2765  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
root      4335  4036  0 12:36 pts/0    00:00:00 grep --color=auto 2127


root@qa-openstack01:~# ip netns
qrouter-5111d40f-3afc-4e2f-ab74-3186f8584971
qdhcp-7e2165c0-b354-42b1-aa85-b4733fe1d1d2


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3693/python


root@qa-openstack01:~# ps -ef| grep 3693
root      3693     1  0 12:33 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var    /lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron  /metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log dir=/var/log/neutron
root      4511  4036  0 12:38 pts/0    00:00:00 grep --color=auto 3693


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.


I noticed that some neutron processes are are not run by neutron.  Is this relevant?

root@qa-openstack01:~# ps -ef |grep neutron
neutron   2022     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-metadata-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/metadata_agent.ini --log-file=/var/log/neutron/metadata-agent.log
neutron   2024     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-openvswitch-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/plugins/ml2/ml2_conf.ini --log-file=/var/log/neutron/openvswitch-agent.log
neutron   2031     1  0 13:36 ?        00:00:01 /usr/bin/python /usr/bin/neutron-server --config-file /etc/neutron/neutron.conf --log-file /var/log/neutron/server.log --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
neutron   2208     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-dhcp-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/dhcp_agent.ini --log-file=/var/log/neutron/dhcp-agent.log
neutron   2214     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-l3-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/l3_agent.ini --config-file=/etc/neutron/fwaas_driver.ini --log-file=/var/log/neutron/l3-agent.log
root      3048  2024  0 13:36 ?        00:00:00 sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
root      3050  3048  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
nobody    3529     1  0 13:37 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --bind-interfaces --interface=tap123b69fd-3c --except-interface=lo --pid-file=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/host --addn-hosts=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/addn_hosts --dhcp-optsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/opts --leasefile-ro --dhcp-range=set:tag0,172.16.100.0,static,86400s --dhcp-lease-max=256 --conf-file=/etc/neutron/dnsmasq-neutron.conf --domain=openstacklocal
root      3587     1  0 13:37 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log-dir=/var/log/neutron
root      4150  3239  0 13:39 pts/0    00:00:00 grep --color=auto neutron





I configured the setting as suggested.

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex
metadata_port = 9697
enable_metadata_proxy = True
router_delete_namespaces = False
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5


root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#|grep metadata
enabled_apis=ec2,osapi_compute,metadata
metadata_workers=2
metadata_listen = 0.0.0.0
metadata_listen_port = 8775
metadata_host = 10.10.12.7
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret

Alas, still not joy.

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 curl http://169.254.169.254
curl: (7) Failed to connect to 169.254.169.254 port 80: No route to host

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.063 ms

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.1
PING 10.10.12.1 (10.10.12.1) 56(84) bytes of data.
From 10.10.12.231 icmp_seq=1 Destination Host Unreachable

I'm still periodically investigating the issue. I In summary I cannot access beyond external interface of router (including host IP). Should the following that show DOWN be showing UP?

root@qa-openstack01:~# ovs-ofctl show br-ex
OFPT_FEATURES_REPLY (xid=0x2): dpid:00001e0dcb6e184a
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
 1(qg-416e130c-f8): addr:00:00:00:00:00:00
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

root@qa-openstack01:~# ovs-ofctl show br-int
OFPT_FEATURES_REPLY (xid=0x2): dpid:0000ba900a62a94c
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
 1(patch-tun): addr:3e:4a:1c:49:73:a8
 config:     0
 state:      0
 speed: 0 Mbps now, 0 Mbps max
 2(tap7ed05795-4c): addr:00:00:00:00:00:00
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
 3(qr-768d4f2c-eb): addr:00:00:00:00:00:00
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
 LOCAL(br-int): addr:06:fa:62:7e:84:d8
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

root@qa-openstack01:~# ovs-ofctl show br-tun
OFPT_FEATURES_REPLY (xid=0x2): dpid:0000ee2bec51d74e
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE

1(patch-int): addr:92:cf:64:e5:d8:aa config: 0 state: 0 speed: 0 Mbps now, 0 Mbps max LOCAL(br-tun): addr:ee:2b:ec:51:d7:4e config: 0 state: 0 speed: 0 Mbps now, 0 Mbps max OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

miss_send_len=0


ip neighbor
10.10.12.1 dev br-ex lladdr d0:67:e5:af:ab:6c REACHABLE

Why can my instances not ping host & vice versa?

I've added this to the top so that it may save you reading through all that follows. I have as yet not achieved a fully working system comprising of icehouse with neutron on ubuntu after some weeks of effort. I have however achieved it on CentOS after two days. I'd advise those who are not tied to ubuntu as a host OS to use RHEL/CentOS and use packstack. :-) I will continue investigation on ubuntu when time allows and follow below.

I have installed OpenStack Icehouse on Ubuntu 14.04 LTS and configured neutron. I have configured external & internal networks, subnets and router etc. I can create instances and they can ping each other and external interface of qrouter etc.

I cannot ping beyond qrouter, e.g. host IP and beyond and I cannot ping instances from the host. I suspect that eth0 on host is still being used, where br-ex should be?

Anyone have some tips?

root@qa-openstack01:~# ifconfig
br-ex     Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet addr:10.10.12.7  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:7047 errors:0 dropped:3 overruns:0 frame:0
          TX packets:3980 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:510845 (510.8 KB)  TX bytes:896241 (896.2 KB)

br-int    Link encap:Ethernet  HWaddr 52:d5:65:a9:ef:40
          inet6 addr: fe80::50e6:6bff:fecd:112a/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:79 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:8257 (8.2 KB)  TX bytes:648 (648.0 B)

br-tun    Link encap:Ethernet  HWaddr fe:da:ad:8e:fc:43
          inet6 addr: fe80::707a:55ff:fefb:49d6/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)

eth0      Link encap:Ethernet  HWaddr 00:22:64:9b:38:46
          inet6 addr: fe80::222:64ff:fe9b:3846/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7128 errors:0 dropped:41 overruns:0 frame:0
          TX packets:4257 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:656883 (656.8 KB)  TX bytes:936551 (936.5 KB)

eth1      Link encap:Ethernet  HWaddr 00:22:64:9b:58:8a
          inet addr:10.10.13.231  Bcast:10.10.13.255  Mask:255.255.255.0
          UP BROADCAST PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:65593 errors:0 dropped:0 overruns:0 frame:0
          TX packets:65593 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:18543627 (18.5 MB)  TX bytes:18543627 (18.5 MB)

qbra51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::50a4:ddff:fe7a:6d7e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:47 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:5143 (5.1 KB)  TX bytes:648 (648.0 B)

qvba51d00a3-ca Link encap:Ethernet  HWaddr 7e:98:a3:42:eb:4f
          inet6 addr: fe80::7c98:a3ff:fe42:eb4f/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:53 errors:0 dropped:0 overruns:0 frame:0
          TX packets:71 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4358 (4.3 KB)  TX bytes:8083 (8.0 KB)

qvoa51d00a3-ca Link encap:Ethernet  HWaddr 56:eb:18:2a:30:88
          inet6 addr: fe80::54eb:18ff:fe2a:3088/64 Scope:Link
          UP BROADCAST RUNNING PROMISC MULTICAST  MTU:1500  Metric:1
          RX packets:71 errors:0 dropped:0 overruns:0 frame:0
          TX packets:53 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:8083 (8.0 KB)  TX bytes:4358 (4.3 KB)

tapa51d00a3-ca Link encap:Ethernet  HWaddr fe:16:3e:6c:b6:ab
          inet6 addr: fe80::fc16:3eff:fe6c:b6ab/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:56 errors:0 dropped:0 overruns:0 frame:0
          TX packets:59 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:500
          RX bytes:6877 (6.8 KB)  TX bytes:4826 (4.8 KB)

virbr0    Link encap:Ethernet  HWaddr 3e:85:c4:b6:37:62
          inet addr:192.168.122.1  Bcast:192.168.122.255  Mask:255.255.255.0
          UP BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 5d5fef6b-cf6b-473d-9766-e9e3f2f3b6a8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| 655052f6-6baf-4efe-a4ad-0df5388f7bd6 | Metadata agent     | qa-openstack01 | :-)   | True           |
| 9948e608-ce07-4576-b259-8d7f117ab44e | DHCP agent         | qa-openstack01 | :-)   | True           |
| e3d3eb97-62e6-4f7b-a157-4834150e1675 | L3 agent           | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# ovs-vsctl show
72361d20-f343-469f-842c-8f09c2cf1058
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-6119ec76-62"
            Interface "qg-6119ec76-62"
                type: internal
    Bridge br-tun
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-20562557-85"
            tag: 1
            Interface "qr-20562557-85"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoa51d00a3-ca"
            tag: 1
            Interface "qvoa51d00a3-ca"
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "tap10894d65-75"
            tag: 1
            Interface "tap10894d65-75"
                type: internal
    ovs_version: "2.0.2"


root@qa-openstack01:~# brctl show
bridge name     bridge id               STP enabled     interfaces
br-ex           8000.0022649b3846       no              eth0
qbra51d00a3-ca          8000.7e98a342eb4f       no              qvba51d00a3-ca
                                                        tapa51d00a3-ca
virbr0          8000.000000000000       yes


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:25 errors:0 dropped:0 overruns:0 frame:0
          TX packets:25 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2467 (2.4 KB)  TX bytes:2467 (2.4 KB)

qg-6119ec76-62 Link encap:Ethernet  HWaddr fa:16:3e:7d:ae:bd
          inet addr:10.10.12.231  Bcast:10.10.12.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe7d:aebd/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:1998 (1.9 KB)

qr-20562557-85 Link encap:Ethernet  HWaddr fa:16:3e:10:04:03
          inet addr:172.16.100.1  Bcast:172.16.100.255  Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fe10:403/64 Scope:Link
          UP BROADCAST RUNNING  MTU:1500  Metric:1
          RX packets:60 errors:0 dropped:0 overruns:0 frame:0
          TX packets:46 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:7211 (7.2 KB)  TX bytes:3236 (3.2 KB)


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG        0 0          0 qg-6119ec76-62
10.10.12.0      0.0.0.0         255.255.255.0   U         0 0          0 qg-6119ec76-62
172.16.100.0    0.0.0.0         255.255.255.0   U         0 0          0 qr-20562557-85


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.055 ms
^C
--- 10.10.12.231 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.055/0.055/0.055/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping 172.16.100.3
PING 172.16.100.3 (172.16.100.3) 56(84) bytes of data.
64 bytes from 172.16.100.3: icmp_seq=1 ttl=64 time=0.638 ms
^C
--- 172.16.100.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.638/0.638/0.638/0.000 ms


root@qa-openstack01:~# ip netns exec qrouter-243bab22-444d-4a62-bed7-e6b675c61df4 ping www.cisco.com
NOTHING...

Responding to request below:

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+----------+--------------+
| IP Protocol | From Port | To Port | IP Range | Source Group |
+-------------+-----------+---------+----------+--------------+
|             |           |         |          | default      |
|             |           |         |          | default      |
+-------------+-----------+---------+----------+--------------+

Adding further rules doesn't make any difference.

root@qa-openstack01:~# nova secgroup-list-rules default
+-------------+-----------+---------+-----------+--------------+
| IP Protocol | From Port | To Port | IP Range  | Source Group |
+-------------+-----------+---------+-----------+--------------+
|             |           |         |           | default      |
| tcp         | 1         | 65535   | 0.0.0.0/0 |              |
| udp         | 1         | 65535   | 0.0.0.0/0 |              |
| icmp        | -1        | -1      | 0.0.0.0/0 |              |
|             |           |         |           | default      |
+-------------+-----------+---------+-----------+--------------+

Response to question below:

root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini
[ml2]
# (ListOpt) List of network type driver entrypoints to be loaded from
# the neutron.ml2.type_drivers namespace.
#
type_drivers = gre
# Example: type_drivers = flat,vlan,gre,vxlan

# (ListOpt) Ordered list of network_types to allocate as tenant
# networks. The default value 'local' is useful for single-box testing
# but provides no connectivity between hosts.
#
tenant_network_types = gre
# Example: tenant_network_types = vlan,gre,vxlan

# (ListOpt) Ordered list of networking mechanism driver entrypoints
# to be loaded from the neutron.ml2.mechanism_drivers namespace.
mechanism_drivers = openvswitch
# Example: mechanism_drivers = openvswitch,mlnx
# Example: mechanism_drivers = arista
# Example: mechanism_drivers = cisco,logger
# Example: mechanism_drivers = openvswitch,brocade
# Example: mechanism_drivers = linuxbridge,brocade

[ml2_type_flat]
# (ListOpt) List of physical_network names with which flat networks
# can be created. Use * to allow flat networks with arbitrary
# physical_network names.
#
# flat_networks = physnet1
# Example:flat_networks = physnet1,physnet2
# Example:flat_networks = *

[ml2_type_vlan]
# (ListOpt) List of <physical_network>[:<vlan_min>:<vlan_max>] tuples
# specifying physical_network names usable for VLAN provider and
# tenant networks, as well as ranges of VLAN tags on each
# physical_network available for allocation as tenant networks.
#
# network_vlan_ranges = physnet1:1000:2999
# Example: network_vlan_ranges = physnet1:1000:2999,physnet2

[ml2_type_gre]
# (ListOpt) Comma-separated list of <tun_min>:<tun_max> tuples enumerating ranges of GRE tunnel IDs that are available for tenant network allocation
tunnel_id_ranges = 1:1000

[ml2_type_vxlan]
# (ListOpt) Comma-separated list of <vni_min>:<vni_max> tuples enumerating
# ranges of VXLAN VNI IDs that are available for tenant network allocation.
#
# vni_ranges =

# (StrOpt) Multicast group for the VXLAN interface. When configured, will
# enable sending all broadcast traffic to this multicast group. When left
# unconfigured, will disable multicast VXLAN mode.
#
# vxlan_group =
# Example: vxlan_group = 239.1.1.1

[securitygroup]
# Controls if neutron security group is enabled or not.
# It should be false when you use nova security group.
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True

Referencing iptables double check requested below.

root@qa-openstack01:~# iptables-save
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*mangle
:PREROUTING ACCEPT [168487:43297540]
:INPUT ACCEPT [164770:43133512]
:FORWARD ACCEPT [128:24606]
:OUTPUT ACCEPT [165322:43377297]
:POSTROUTING ACCEPT [165446:43401743]
:nova-api-POSTROUTING - [0:0]
:nova-network-POSTROUTING - [0:0]
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
-A POSTROUTING -j nova-api-POSTROUTING
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*nat
:PREROUTING ACCEPT [3690:151190]
:INPUT ACCEPT [63:3164]
:OUTPUT ACCEPT [570:34879]
:POSTROUTING ACCEPT [586:37215]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-POSTROUTING - [0:0]
:nova-network-PREROUTING - [0:0]
:nova-network-float-snat - [0:0]
:nova-network-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-network-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-network-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
-A nova-network-snat -j nova-network-float-snat
-A nova-postrouting-bottom -j nova-network-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Fri Sep 19 18:16:05 2014
# Generated by iptables-save v1.4.21 on Fri Sep 19 18:16:05 2014
*filter
:INPUT ACCEPT [164766:43133258]
:FORWARD ACCEPT [10:3380]
:OUTPUT ACCEPT [165318:43377043]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i3338a6c4-5 - [0:0]
:neutron-openvswi-ia51d00a3-c - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o3338a6c4-5 - [0:0]
:neutron-openvswi-oa51d00a3-c - [0:0]
:neutron-openvswi-s3338a6c4-5 - [0:0]
:neutron-openvswi-sa51d00a3-c - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
:nova-network-FORWARD - [0:0]
:nova-network-INPUT - [0:0]
:nova-network-OUTPUT - [0:0]
:nova-network-local - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-network-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A INPUT -j nova-api-INPUT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-network-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -j nova-api-FORWARD
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-network-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A OUTPUT -j nova-api-OUTPUT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-INPUT -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-oa51d00a3-c
-A neutron-openvswi-i3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-i3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-i3338a6c4-5 -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-ia51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-ia51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-ia51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-ia51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-ia51d00a3-c -s 172.16.100.2/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-ia51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-s3338a6c4-5
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state INVALID -j DROP
-A neutron-openvswi-o3338a6c4-5 -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p icmp -j RETURN
-A neutron-openvswi-o3338a6c4-5 -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o3338a6c4-5 -j neutron-openvswi-sg-fallback
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sa51d00a3-c
-A neutron-openvswi-oa51d00a3-c -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state INVALID -j DROP
-A neutron-openvswi-oa51d00a3-c -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-oa51d00a3-c -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -p icmp -j RETURN
-A neutron-openvswi-oa51d00a3-c -p udp -m udp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-oa51d00a3-c -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s3338a6c4-5 -s 172.16.100.5/32 -m mac --mac-source FA:16:3E:47:92:0E -j RETURN
-A neutron-openvswi-s3338a6c4-5 -j DROP
-A neutron-openvswi-sa51d00a3-c -s 172.16.100.3/32 -m mac --mac-source FA:16:3E:6C:B6:AB -j RETURN
-A neutron-openvswi-sa51d00a3-c -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-i3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap3338a6c4-52 --physdev-is-bridged -j neutron-openvswi-o3338a6c4-5
-A neutron-openvswi-sg-chain -m physdev --physdev-out tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-ia51d00a3-c
-A neutron-openvswi-sg-chain -m physdev --physdev-in tapa51d00a3-ca --physdev-is-bridged -j neutron-openvswi-wioa51d00a3-c
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-network-local
-A nova-filter-top -j nova-api-local
COMMIT

Adding all conf files and latest iptables as not making much progress. :-(

conf files:

root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#
[DEFAULT]
verbose=True
debug=False
logdir=/var/log/nova
auth_strategy=keystone
state_path=/var/lib/nova
lock_path=/run/lock/nova
rootwrap_config=/etc/nova/rootwrap.conf
api_paste_config=/etc/nova/api-paste.ini
rabbit_host=10.10.12.7
rabbit_port=5672
rpc_backend = nova.openstack.common.rpc.impl_kombu
rabbit_userid=guest
rabbit_password=guest
compute_scheduler_driver=nova.scheduler.filter_scheduler.FilterScheduler
network_manager=nova.network.manager.FlatDHCPManager
force_dhcp_release=True
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
firewall_driver = nova.virt.firewall.NoopFirewallDriver
my_ip=10.10.12.7
public_interface=br-ex
vlan_interface=br-ex
flat_network_bridge=br-ex
flat_interface=br-ex
dnsmasq_config_file=/etc/nova/dnsmasq-nova.conf
fixed_range=''
enable_ipv6=False
image_service=nova.image.glance.GlanceImageService
glance_api_servers=10.10.12.7:9292
glance_host=10.10.12.7
volume_api_class=nova.volume.cinder.API
osapi_volume_listen_port=5900
snapshot_image_format=qcow2
iscsi_helper=tgtadm
network_api_class = nova.network.neutronv2.api.API
security_group_api = neutron
compute_manager=nova.compute.manager.ComputeManager
connection_type=libvirt
compute_driver=libvirt.LibvirtDriver
libvirt_type=kvm
libvirt_inject_key=false
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
remove_unused_base_images=true
remove_unused_resized_minimum_age_seconds=3600
remove_unused_original_minimum_age_seconds=3600
checksum_base_images=false
start_guests_on_host_boot=true
resume_guests_state_on_host_boot=true
volumes_path=/var/lib/nova/volumes
quota_security_groups=50
quota_fixed_ips=40
quota_instances=20
force_config_drive=false
cpu_allocation_ratio=16.0
ram_allocation_ratio=1.5
keystone_ec2_url=http://10.10.12.7:5000/v2.0/ec2tokens
my_ip=10.10.12.7
novnc_enabled=true
novncproxy_base_url=http://10.10.12.7:6080/vnc_auto.html
xvpvncproxy_base_url=http://10.10.12.7:6081/console
novncproxy_host=10.10.12.7
novncproxy_port=6080
vncserver_listen=10.10.12.7
vncserver_proxyclient_address=10.10.12.7
osapi_max_limit=1000
enabled_apis=ec2,osapi_compute,metadata
osapi_compute_extension = nova.api.openstack.compute.contrib.standard_extensions
ec2_workers=4
osapi_compute_workers=4
metadata_workers=4
osapi_volume_workers=4
osapi_compute_listen=10.10.12.7
osapi_compute_listen_port=8774
ec2_listen=10.10.12.7
ec2_listen_port=8773
ec2_host=10.10.12.7
ec2_private_dns_show_ip=True

service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = Passw0rd
neutron_admin_auth_url = http://controller:35357/v2.0

allow_resize_to_same_host=True
[database]
connection = mysql://nova:Passw0rd@10.10.12.7/nova
[keystone_authtoken]
auth_uri = http://10.10.12.7:5000
auth_host = 10.10.12.7
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = Passw0rd


root@qa-openstack01:~# cat /etc/neutron/neutron.conf | grep -v ^$ | grep -v ^#

[DEFAULT]
state_path = /var/lib/neutron
lock_path = $state_path/lock
core_plugin = ml2
service_plugins = router
auth_strategy = keystone
allow_overlapping_ips = True
rpc_backend = neutron.openstack.common.rpc.impl_kombu
rabbit_host = controller
rabbit_password = guest
notification_driver = neutron.openstack.common.notifier.rpc_notifier
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://controller:8774/v2
nova_admin_username = nova
nova_admin_tenant_id = 0372ee0381fe4415a862b798c7024e37
nova_admin_password = Passw0rd
nova_admin_auth_url = http://controller:35357/v2.0
[quotas]
[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
[keystone_authtoken]
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
signing_dir = $state_path/keystone-signing
auth_uri = http://controller:5000
[database]
connection = mysql://neutron:Passw0rd@controller/neutron
[service_providers]
service_provider=LOADBALANCER:Haproxy:neutron.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
service_provider=VPN:openswan:neutron.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default


root@qa-openstack01:~# cat /etc/neutron/plugins/ml2/ml2_conf.ini | grep -v ^$ | grep -v ^#

[ml2]
type_drivers = gre
tenant_network_types = gre
mechanism_drivers = openvswitch
[ml2_type_flat]
[ml2_type_vlan]
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
[securitygroup]
enable_security_group = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.10.12.7
tunnel_type = gre
enable_tunneling = True


root@qa-openstack01:~# cat /etc/neutron/dhcp_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
dnsmasq_config_file = /etc/neutron/dnsmasq-neutron.conf
external_network_bridge = br-ex

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex


root@qa-openstack01:~# cat /etc/neutron/dnsmasq-neutron.conf

dhcp-option-force=26,1454


root@qa-openstack01:~# cat /etc/neutron/metadata_agent.ini | grep -v ^$ | grep -v ^#

[DEFAULT]
auth_url = http://controller:5000/v2.0
auth_region = AMS
admin_tenant_name = service
admin_user = neutron
admin_password = Passw0rd
nova_metadata_ip = controller
metadata_proxy_shared_secret = secret

iptables:

root@qa-openstack01:~# iptables -L

Chain INPUT (policy ACCEPT)
target     prot opt source               destination
neutron-openvswi-INPUT  all  --  anywhere             anywhere
nova-network-INPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:domain
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:domain
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootps
ACCEPT     tcp  --  anywhere             anywhere             tcp dpt:bootps
nova-api-INPUT  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
ACCEPT     icmp --  anywhere             anywhere
ACCEPT     all  --  anywhere             anywhere
ACCEPT     tcp  --  ip-192-169-142-97.ip.secureserver.net  anywhere             multiport dports 5671,amqp /* 001 amqp incoming amqp_192.169.142.97 */
ACCEPT     tcp  --  anywhere             anyhere             multiport dports iscsi-target /* 001 cinder incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports http /* 001 horizon incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9292 /* 001 glance incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5000,35357 /* 001 keystone incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports mysql /* 001 mariadb incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 6080 /* 001 novncproxy incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8770:8780 /* 001 novaapi incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9696 /* 001 neutron incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9697 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports amqp /* 001 qpid incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8700 /* 001 metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8775 /* 001 nova_metadata incoming */
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpt:ssh
ACCEPT     tcp  --  anywhere             anywhere             state NEW tcp dpts:5900:5999
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited
ACCEPT     gre  --  anywhere             anywhere
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 5900 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8774 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 8773 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 35357 /* 001  incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 registry incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports 9191 /* 001 incoming */
ACCEPT     tcp  --  anywhere             anywhere             multiport dports afs3-fileserver /* 001 incoming */

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-FORWARD  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-FORWARD  all  --  anywhere             anywhere
ACCEPT     all  --  anywhere             192.168.122.0/24     ctstate RELATED,ESTABLISHED
ACCEPT     all  --  192.168.122.0/24     anywhere
ACCEPT     all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
REJECT     all  --  anywhere             anywhere             reject-with icmp-port-unreachable
nova-api-FORWARD  all  --  anywhere             anywhere
REJECT     all  --  anywhere             anywhere             reject-with icmp-host-prohibited

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
neutron-filter-top  all  --  anywhere             anywhere
neutron-openvswi-OUTPUT  all  --  anywhere             anywhere
nova-filter-top  all  --  anywhere             anywhere
nova-network-OUTPUT  all  --  anywhere             anywhere
ACCEPT     udp  --  anywhere             anywhere             udp dpt:bootpc
nova-api-OUTPUT  all  --  anywhere             anywhere
ACCEPT     gre  --  anywhere             anywhere

Chain neutron-filter-top (2 references)
target     prot opt source               destination
neutron-openvswi-local  all  --  anywhere             anywhere

Chain neutron-openvswi-FORWARD (1 references)
target     prot opt source               destination
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-sg-chain  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-INPUT (1 references)
target     prot opt source               destination
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged 
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged

Chain neutron-openvswi-OUTPUT (1 references)
target     prot opt source               destination

Chain neutron-openvswi-i3338a6c4-5 (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-ia51d00a3-c (1 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  172.16.100.2         anywhere             udp spt:bootps dpt:bootpc

neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-local (1 references)

target     prot opt source               destination

Chain neutron-openvswi-o3338a6c4-5 (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-s3338a6c4-5  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-oa51d00a3-c (2 references)
target     prot opt source               destination
RETURN     udp  --  anywhere             anywhere             udp spt:bootpc dpt:bootps
neutron-openvswi-sa51d00a3-c  all  --  anywhere             anywhere
DROP       udp  --  anywhere             anywhere             udp spt:bootps dpt:bootpc
DROP       all  --  anywhere             anywhere             state INVALID
RETURN     all  --  anywhere             anywhere             state RELATED,ESTABLISHED
RETURN     tcp  --  anywhere             anywhere             tcp multiport dports tcpmux:65535
RETURN     icmp --  anywhere             anywhere
RETURN     udp  --  anywhere             anywhere             udp multiport dports 1:65535
neutron-openvswi-sg-fallback  all  --  anywhere             anywhere

Chain neutron-openvswi-s3338a6c4-5 (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.5         anywhere             MAC FA:16:3E:47:92:0E
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sa51d00a3-c (1 references)
target     prot opt source               destination
RETURN     all  --  172.16.100.3         anywhere             MAC FA:16:3E:6C:B6:AB
DROP       all  --  anywhere             anywhere

Chain neutron-openvswi-sg-chain (4 references)
target     prot opt source               destination
neutron-openvswi-i3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-o3338a6c4-5  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tap3338a6c4-52 --physdev-is-bridged
neutron-openvswi-ia51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-out tapa51d00a3-ca --physdev-is-bridged
neutron-openvswi-oa51d00a3-c  all  --  anywhere             anywhere             PHYSDEV match --physdev-in tapa51d00a3-ca --physdev-is-bridged
ACCEPT     all  --  anywhere             anywhere

Chain neutron-openvswi-sg-fallback (4 references)
target     prot opt source               destination
DROP       all  --  anywhere             anywhere

Chain nova-api-FORWARD (1 references)
target     prot opt source               destination

Chain nova-api-INPUT (1 references)
target     prot opt source               destination
ACCEPT     tcp  --  anywhere             qa-openstack01       tcp dpt:8775

Chain nova-api-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-api-local (1 references)
target     prot opt source               destination

Chain nova-filter-top (2 references)
target     prot opt source               destination
nova-network-local  all  --  anywhere             anywhere
nova-api-local  all  --  anywhere             anywhere

Chain nova-network-FORWARD (1 references)
target     prot opt source               destination

Chain nova-network-INPUT (1 references)
target     prot opt source               destination

Chain nova-network-OUTPUT (1 references)
target     prot opt source               destination

Chain nova-network-local (1 references)
target     prot opt source               destination






root@qa-openstack01:~# iptables-save | grep 8775
-A nova-network-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.10.12.7:8775
<- this is not shown in your setup but is in mine.
-A INPUT -p tcp -m multiport --dports 8775 -m comment --comment "001 nova_metadata incoming" -j ACCEPT
<- this was not shown i mine so I added to match yours. Seems to make no difference.
-A nova-api-INPUT -d 10.10.12.7/32 -p tcp -m tcp --dport 8775 -j ACCEPT


root@qa-openstack01:~# netstat -antp | grep 8775
tcp        0      0 0.0.0.0:8775            0.0.0.0:*               LISTEN      2127/python


root@qa-openstack01:~# ps -ef |grep 2127
nova      2127     1  0 12:32 ?        00:00:02 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2450  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2451  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2452  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2454  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2657  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2658  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2662  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2664  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2746  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2751  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2758  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
nova      2765  2127  0 12:32 ?        00:00:00 /usr/bin/python /usr/bin/nova-api --config-file=/etc/nova/nova.conf
root      4335  4036  0 12:36 pts/0    00:00:00 grep --color=auto 2127


root@qa-openstack01:~# ip netns
qrouter-5111d40f-3afc-4e2f-ab74-3186f8584971
qdhcp-7e2165c0-b354-42b1-aa85-b4733fe1d1d2


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca iptables -S -t nat | grep 169.254
-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697


root@qa-openstack01:~# ip netns exec qrouter-b01cb847-decc-413a-a5fd-d664a22c70ca netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      3693/python


root@qa-openstack01:~# ps -ef| grep 3693
root      3693     1  0 12:33 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var    /lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron  /metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log dir=/var/log/neutron
root      4511  4036  0 12:38 pts/0    00:00:00 grep --color=auto 3693


root@qa-openstack01:~# neutron agent-list
+--------------------------------------+--------------------+----------------+-------+----------------+
| id                                   | agent_type         | host           | alive | admin_state_up |
+--------------------------------------+--------------------+----------------+-------+----------------+
| 1aff756d-2c9f-4233-a9cf-e32e77dcdf0a | DHCP agent         | qa-openstack01 | :-)   | True           |
| 287f0a0d-b63a-45b6-b63a-b5fe8b0039de | L3 agent           | qa-openstack01 | :-)   | True           |
| 3650216f-6852-42e7-b266-f06fc53ad1b8 | Open vSwitch agent | qa-openstack01 | :-)   | True           |
| c37362de-d599-48da-b998-b75e4458f288 | Metadata agent     | qa-openstack01 | :-)   | True           |
+--------------------------------------+--------------------+----------------+-------+----------------+


root@qa-openstack01:~# nova-manage service list
Binary           Host                                 Zone             Status     State Updated_At
nova-conductor   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-consoleauth qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-network     qa-openstack01                       internal         enabled    :-)   2014-09-22 11:39:59
nova-cert        qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-scheduler   qa-openstack01                       internal         enabled    :-)   2014-09-22 11:40:04
nova-compute     qa-openstack01                       nova             enabled    :-)   2014-09-22 11:39:57


root@qa-openstack01:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.12.1      0.0.0.0         UG    0      0        0 br-ex
10.10.12.0      0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

There is no route specified to 169.254.0.0 as with yours.  I added one but it made no difference.


I noticed that some neutron processes are are not run by neutron.  Is this relevant?

root@qa-openstack01:~# ps -ef |grep neutron
neutron   2022     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-metadata-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/metadata_agent.ini --log-file=/var/log/neutron/metadata-agent.log
neutron   2024     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-openvswitch-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/plugins/ml2/ml2_conf.ini --log-file=/var/log/neutron/openvswitch-agent.log
neutron   2031     1  0 13:36 ?        00:00:01 /usr/bin/python /usr/bin/neutron-server --config-file /etc/neutron/neutron.conf --log-file /var/log/neutron/server.log --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
neutron   2208     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-dhcp-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/dhcp_agent.ini --log-file=/var/log/neutron/dhcp-agent.log
neutron   2214     1  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-l3-agent --config-file=/etc/neutron/neutron.conf --config-file=/etc/neutron/l3_agent.ini --config-file=/etc/neutron/fwaas_driver.ini --log-file=/var/log/neutron/l3-agent.log
root      3048  2024  0 13:36 ?        00:00:00 sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
root      3050  3048  0 13:36 ?        00:00:00 /usr/bin/python /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf ovsdb-client monitor Interface name,ofport --format=json
nobody    3529     1  0 13:37 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --bind-interfaces --interface=tap123b69fd-3c --except-interface=lo --pid-file=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/host --addn-hosts=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/addn_hosts --dhcp-optsfile=/var/lib/neutron/dhcp/7e2165c0-b354-42b1-aa85-b4733fe1d1d2/opts --leasefile-ro --dhcp-range=set:tag0,172.16.100.0,static,86400s --dhcp-lease-max=256 --conf-file=/etc/neutron/dnsmasq-neutron.conf --domain=openstacklocal
root      3587     1  0 13:37 ?        00:00:00 /usr/bin/python /usr/bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/5111d40f-3afc-4e2f-ab74-3186f8584971.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=5111d40f-3afc-4e2f-ab74-3186f8584971 --state_path=/var/lib/neutron --metadata_port=9697 --log-file=neutron-ns-metadata-proxy-5111d40f-3afc-4e2f-ab74-3186f8584971.log --log-dir=/var/log/neutron
root      4150  3239  0 13:39 pts/0    00:00:00 grep --color=auto neutron





I configured the setting as suggested.

root@qa-openstack01:~# cat /etc/neutron/l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge = br-ex
metadata_port = 9697
enable_metadata_proxy = True
router_delete_namespaces = False
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5


root@qa-openstack01:~# cat /etc/nova/nova.conf | grep -v ^$ | grep -v ^#|grep metadata
enabled_apis=ec2,osapi_compute,metadata
metadata_workers=2
metadata_listen = 0.0.0.0
metadata_listen_port = 8775
metadata_host = 10.10.12.7
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = secret

Alas, still not joy.

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 curl http://169.254.169.254
curl: (7) Failed to connect to 169.254.169.254 port 80: No route to host

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.231
PING 10.10.12.231 (10.10.12.231) 56(84) bytes of data.
64 bytes from 10.10.12.231: icmp_seq=1 ttl=64 time=0.063 ms

root@qa-openstack01:~# ip netns exec qrouter-fd75821c-0348-4f52-8129-553c8e82a6c3 ping 10.10.12.1
PING 10.10.12.1 (10.10.12.1) 56(84) bytes of data.
From 10.10.12.231 icmp_seq=1 Destination Host Unreachable

I'm still periodically investigating the issue. In summary I cannot access beyond external interface of router (including host IP). Should the following that show DOWN be showing UP?

root@qa-openstack01:~# ovs-ofctl show br-ex
OFPT_FEATURES_REPLY (xid=0x2): dpid:00001e0dcb6e184a
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
 1(qg-416e130c-f8): addr:00:00:00:00:00:00
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

root@qa-openstack01:~# ovs-ofctl show br-int
OFPT_FEATURES_REPLY (xid=0x2): dpid:0000ba900a62a94c
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
 1(patch-tun): addr:3e:4a:1c:49:73:a8
 config:     0
 state:      0
 speed: 0 Mbps now, 0 Mbps max
 2(tap7ed05795-4c): addr:00:00:00:00:00:00
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
 3(qr-768d4f2c-eb): addr:00:00:00:00:00:00
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
 LOCAL(br-int): addr:06:fa:62:7e:84:d8
 config:     PORT_DOWN
 state:      LINK_DOWN
 speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0

root@qa-openstack01:~# ovs-ofctl show br-tun
OFPT_FEATURES_REPLY (xid=0x2): dpid:0000ee2bec51d74e
n_tables:254, n_buffers:256
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: OUTPUT SET_VLAN_VID SET_VLAN_PCP STRIP_VLAN SET_DL_SRC SET_DL_DST SET_NW_SRC SET_NW_DST SET_NW_TOS SET_TP_SRC SET_TP_DST ENQUEUE
1(patch-int): addr:92:cf:64:e5:d8:aa
 config:     0
 state:      0
 speed: 0 Mbps now, 0 Mbps max
LOCAL(br-tun): addr:ee:2b:ec:51:d7:4e
 config:     0
 state:      0
 speed: 0 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0


ip neighbor
10.10.12.1 dev br-ex lladdr d0:67:e5:af:ab:6c REACHABLE