Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed : Network is unreachable

I have 2 servers both uses Openstack Icehouse installation on Ubuntu 14.04.1 System:

  • Server1 - Single node installation: Having controller, compute and network node in one system

  • Server2 - Compute node: Only compute service is running

I'm able to launch Ubuntu 14.04 cloud image on Server1 successfully. But when i launch Ubuntu VM on Server2 it gives:

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed [46/120s]: request error [HTTPConnectionPool(host='169.254.169.254', port=80): Max retries exceeded with url: /2009-04-04/meta-data/instance-id (Caused by <class 'socket.error'="">: [Errno 101] Network is unreachable)]

nova.conf on Server1:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = controller
vncserver_listen = controller
vncserver_proxyclient_address = controller
novncproxy_base_url = http://10.1.12.2:6080/vnc_auto.html

auth_strategy = keystone

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host=10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

nova.conf on Server2:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

auth_strategy=keystone

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = 10.1.12.17
vnc_enabled = True
vncserver_listen = 10.1.12.17
vncserver_proxyclient_address = 10.1.12.17
novncproxy_base_url = http://controller:6080/vnc_auto.html

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host = 10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed : Network is unreachable

I have 2 servers both uses Openstack Icehouse installation on Ubuntu 14.04.1 System:

  • Server1 - Single node installation: Having controller, compute and network node in one system

  • Server2 - Compute node: Only compute service is running

I'm able to launch Ubuntu 14.04 cloud image on Server1 successfully. But when i launch Ubuntu VM on Server2 it gives:

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed [46/120s]: request error [HTTPConnectionPool(host='169.254.169.254', port=80): Max retries exceeded with url: /2009-04-04/meta-data/instance-id (Caused by <class 'socket.error'="">: [Errno 101] Network is unreachable)]

nova.conf on Server1:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = controller
vncserver_listen = controller
vncserver_proxyclient_address = controller
novncproxy_base_url = http://10.1.12.2:6080/vnc_auto.html

auth_strategy = keystone

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host=10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

nova.conf on Server2:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

auth_strategy=keystone

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = 10.1.12.17
vnc_enabled = True
vncserver_listen = 10.1.12.17
vncserver_proxyclient_address = 10.1.12.17
novncproxy_base_url = http://controller:6080/vnc_auto.html

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host = 10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

ovs-vsctl show on compute node:

Bridge br-int
        fail_mode: secure
        Port "int-br-eth0"
            Interface "int-br-eth0"
        Port "qvo454b3c53-1f"
            tag: 3
            Interface "qvo454b3c53-1f"
        Port br-int
            Interface br-int
                type: internal
    Bridge "br-eth0"
        Port "eth0"
            Interface "eth0"
        Port "br-eth0"
            Interface "br-eth0"
                type: internal
        Port "phy-br-eth0"
            Interface "phy-br-eth0"
    ovs_version: "2.0.2"

route -n on controller:

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.1.12.1       0.0.0.0         UG    0      0        0 br-ex
10.1.12.0       0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

ip netns on controller:

qdhcp-bf111ff7-99a4-4840-88fa-bfc334dad3d6
qrouter-cd88cf9f-753e-4941-8234-9edcd9b2363a

neutron-openvswitch-agent is installed on Compute Node: nova service-list (iseserver is Server1 and MCAServer is Server2) :

+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+
| Id | Binary           | Host      | Zone     | Status  | State | Updated_at                 | Disabled Reason |
+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+
| 1  | nova-cert        | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 2  | nova-consoleauth | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 3  | nova-scheduler   | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 4  | nova-conductor   | iseserver | internal | enabled | up    | 2014-11-08T18:32:11.000000 | -               |
| 6  | nova-compute     | iseserver | nova     | enabled | up    | 2014-11-08T18:32:08.000000 | -               |
| 7  | nova-compute     | MCAServer | nova     | enabled | up    | 2014-11-08T18:32:11.000000 | -               |
+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+

neutron agent-list (iseserver is Server1 and MCAServer is Server2):

+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+
| id                                   | agent_type         | host      | alive | admin_state_up | binary                    |
+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+
| 06e2025f-bb99-41c2-b3ca-369325a59ee9 | Metadata agent     | iseserver | :-)   | True           | neutron-metadata-agent    |
| 2f8db4c6-e328-4d6e-9d35-4cc34f7232b1 | Open vSwitch agent | MCAServer | :-)   | True           | neutron-openvswitch-agent |
| 87267e85-9f8f-4cf8-b5c7-1f16c4137c97 | Open vSwitch agent | iseserver | :-)   | True           | neutron-openvswitch-agent |
| 99fb263a-d3a0-4dcb-8a48-76c83b5dabcf | DHCP agent         | iseserver | :-)   | True           | neutron-dhcp-agent        |
| ad57613c-fb04-4c2a-b207-64ab8eab5a18 | L3 agent           | iseserver | :-)   | True           | neutron-l3-agent          |
+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed : Network is unreachable

I have 2 servers both uses Openstack Icehouse installation on Ubuntu 14.04.1 System:

  • Server1 - Single node installation: Having controller, compute and network node in one system

  • Server2 - Compute node: Only compute service is running

I'm able to launch Ubuntu 14.04 cloud image on Server1 successfully. But when i launch Ubuntu VM on Server2 it gives:

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed [46/120s]: request error [HTTPConnectionPool(host='169.254.169.254', port=80): Max retries exceeded with url: /2009-04-04/meta-data/instance-id (Caused by <class 'socket.error'="">: [Errno 101] Network is unreachable)]

nova.conf on Server1:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = controller
vncserver_listen = controller
vncserver_proxyclient_address = controller
novncproxy_base_url = http://10.1.12.2:6080/vnc_auto.html

auth_strategy = keystone

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host=10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

nova.conf on Server2:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

auth_strategy=keystone

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = 10.1.12.17
vnc_enabled = True
vncserver_listen = 10.1.12.17
vncserver_proxyclient_address = 10.1.12.17
novncproxy_base_url = http://controller:6080/vnc_auto.html

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host = 10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

ovs-vsctl show on compute node:

Bridge br-int
        fail_mode: secure
        Port "int-br-eth0"
            Interface "int-br-eth0"
        Port "qvo454b3c53-1f"
            tag: 3
            Interface "qvo454b3c53-1f"
        Port br-int
            Interface br-int
                type: internal
    Bridge "br-eth0"
        Port "eth0"
            Interface "eth0"
        Port "br-eth0"
            Interface "br-eth0"
                type: internal
        Port "phy-br-eth0"
            Interface "phy-br-eth0"
    ovs_version: "2.0.2"

route -n on controller:

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.1.12.1       0.0.0.0         UG    0      0        0 br-ex
10.1.12.0       0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

ip netns on controller:

qdhcp-bf111ff7-99a4-4840-88fa-bfc334dad3d6
qrouter-cd88cf9f-753e-4941-8234-9edcd9b2363a

neutron-openvswitch-agent is installed on Compute Node: nova service-list (iseserver is Server1 and MCAServer is Server2) :

+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+
| Id | Binary           | Host      | Zone     | Status  | State | Updated_at                 | Disabled Reason |
+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+
| 1  | nova-cert        | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 2  | nova-consoleauth | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 3  | nova-scheduler   | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 4  | nova-conductor   | iseserver | internal | enabled | up    | 2014-11-08T18:32:11.000000 | -               |
| 6  | nova-compute     | iseserver | nova     | enabled | up    | 2014-11-08T18:32:08.000000 | -               |
| 7  | nova-compute     | MCAServer | nova     | enabled | up    | 2014-11-08T18:32:11.000000 | -               |
+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+

neutron agent-list (iseserver is Server1 and MCAServer is Server2):

+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+
| id                                   | agent_type         | host      | alive | admin_state_up | binary                    |
+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+
| 06e2025f-bb99-41c2-b3ca-369325a59ee9 | Metadata agent     | iseserver | :-)   | True           | neutron-metadata-agent    |
| 2f8db4c6-e328-4d6e-9d35-4cc34f7232b1 | Open vSwitch agent | MCAServer | :-)   | True           | neutron-openvswitch-agent |
| 87267e85-9f8f-4cf8-b5c7-1f16c4137c97 | Open vSwitch agent | iseserver | :-)   | True           | neutron-openvswitch-agent |
| 99fb263a-d3a0-4dcb-8a48-76c83b5dabcf | DHCP agent         | iseserver | :-)   | True           | neutron-dhcp-agent        |
| ad57613c-fb04-4c2a-b207-64ab8eab5a18 | L3 agent           | iseserver | :-)   | True           | neutron-l3-agent          |
+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+

Configured for VLAN aswell error still persists! On checking log files only problem i find is with openvswitch-agent.log on both Server1 and Server2. openvswitch-agent.log on Server1:

2014-11-09 18:03:23.443 2555 ERROR neutron.agent.linux.ovsdb_monitor [-] Error received from ovsdb monitor: 2014-11-09T12:33:23Z|00001|fatal_signal|WARN|terminating with signal 15 (Terminated)
2014-11-09 18:03:23.477 2555 CRITICAL neutron [-] Trying to re-send() an already-triggered event.

openvswitch-agent.log on Server2:

2014-11-09 17:32:05.912 16331 ERROR neutron.agent.linux.ovsdb_monitor [-] Error received from ovsdb monitor: 2014-11-09T12:02:05Z|00001|fatal_signal|WARN|terminating with signal 15 (Terminated)
2014-11-09 17:32:06.988 16331 CRITICAL neutron [-] Trying to re-send() an already-triggered event.
2014-11-09 17:32:06.988 16331 TRACE neutron Traceback (most recent call last):
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/bin/neutron-openvswitch-agent", line 10, in <module>
2014-11-09 17:32:06.988 16331 TRACE neutron     sys.exit(main())
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1476, in main
2014-11-09 17:32:06.988 16331 TRACE neutron     agent.daemon_loop()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1404, in daemon_loop
2014-11-09 17:32:06.988 16331 TRACE neutron     self.rpc_loop(polling_manager=pm)
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/contextlib.py", line 24, in __exit__
2014-11-09 17:32:06.988 16331 TRACE neutron     self.gen.next()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/polling.py", line 41, in get_polling_manager
2014-11-09 17:32:06.988 16331 TRACE neutron     pm.stop()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/polling.py", line 108, in stop
2014-11-09 17:32:06.988 16331 TRACE neutron     self._monitor.stop()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/async_process.py", line 91, in stop
2014-11-09 17:32:06.988 16331 TRACE neutron     self._kill()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovsdb_monitor.py", line 108, in _kill
2014-11-09 17:32:06.988 16331 TRACE neutron     super(SimpleInterfaceMonitor, self)._kill(*args, **kwargs)
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/async_process.py", line 118, in _kill
2014-11-09 17:32:06.988 16331 TRACE neutron     self._kill_event.send()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/local/lib/python2.7/dist-packages/eventlet/event.py", line 155, in send
2014-11-09 17:32:06.988 16331 TRACE neutron     assert self._result is NOT_USED, 'Trying to re-send() an already-triggered event.'
2014-11-09 17:32:06.988 16331 TRACE neutron AssertionError: Trying to re-send() an already-triggered event.
2014-11-09 17:32:06.988 16331 TRACE neutron
2014-11-09 17:33:39.101 2196 INFO neutron.common.config [-] Logging enabled!
2014-11-09 17:33:40.118 2196 INFO neutron.openstack.common.rpc.common [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Connected to AMQP server on controller:5672
2014-11-09 17:33:40.139 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Mapping physical network physnet1 to bridge br-eth0
2014-11-09 17:33:40.158 2196 INFO neutron.openstack.common.rpc.common [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Connected to AMQP server on controller:5672
2014-11-09 17:33:41.353 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Agent initialized successfully, now running...
2014-11-09 17:33:41.356 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Agent out of sync with plugin!
2014-11-09 17:34:44.309 2196 INFO neutron.agent.securitygroups_rpc [req-ae754bf6-c1c0-46aa-82c7-2f610a7cd2e5 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:34:47.833 2196 INFO neutron.agent.securitygroups_rpc [-] Preparing filters for devices set([u'cf51921e-261e-4b0d-ac7b-9e90fd4277cd'])
2014-11-09 17:34:47.847 2196 INFO neutron.openstack.common.rpc.common [-] Connected to AMQP server on controller:5672
2014-11-09 17:34:48.563 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Port cf51921e-261e-4b0d-ac7b-9e90fd4277cd updated. Details: {u'admin_state_up': True, u'network_id': u'bd0d9a3e-4361-4d09-ab9c-eacafe18ba01', u'segmentation_id': None, u'physical_network': None, u'device': u'cf51921e-261e-4b0d-ac7b-9e90fd4277cd', u'port_id': u'cf51921e-261e-4b0d-ac7b-9e90fd4277cd', u'network_type': u'local'}
2014-11-09 17:34:48.563 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Assigning 1 as local vlan for net-id=bd0d9a3e-4361-4d09-ab9c-eacafe18ba01
2014-11-09 17:34:48.817 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Configuration for device cf51921e-261e-4b0d-ac7b-9e90fd4277cd completed.
2014-11-09 17:40:28.741 2196 INFO neutron.agent.securitygroups_rpc [req-000ad03e-11e1-428d-ac8a-97a1aa7011cb None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:40:30.149 2196 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 17:42:10.734 2196 INFO neutron.agent.securitygroups_rpc [req-7e34244c-2ca3-4221-9be2-3a835bb2a2d4 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:42:12.197 2196 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 17:42:12.567 2196 INFO neutron.agent.securitygroups_rpc [-] Remove device filter for set([u'cf51921e-261e-4b0d-ac7b-9e90fd4277cd'])
2014-11-09 17:42:12.892 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Attachment cf51921e-261e-4b0d-ac7b-9e90fd4277cd removed
2014-11-09 17:42:12.909 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Reclaiming vlan = 1 from net-id = bd0d9a3e-4361-4d09-ab9c-eacafe18ba01
2014-11-09 17:42:23.650 2196 INFO neutron.agent.securitygroups_rpc [req-9762fbb5-eaa8-4149-a000-822aa60ab69a None] Provider rule updated
2014-11-09 17:42:24.199 2196 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 17:42:24.199 2196 INFO neutron.agent.securitygroups_rpc [-] No ports here to refresh firewall
2014-11-09 17:45:08.604 2196 INFO neutron.agent.securitygroups_rpc [req-e2f37d62-4e7a-40c6-925e-96dc24933523 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:45:08.891 2196 INFO neutron.agent.securitygroups_rpc [req-b933e10e-1864-46f2-a897-3569fb17f172 None] Provider rule updated
2014-11-09 17:45:10.303 2196 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 17:45:10.303 2196 INFO neutron.agent.securitygroups_rpc [-] No ports here to refresh firewall
2014-11-09 17:47:15.171 2196 INFO neutron.agent.securitygroups_rpc [req-b33a55e1-0cbf-41eb-b186-81026b0949cf None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:47:18.412 2196 INFO neutron.agent.securitygroups_rpc [-] Preparing filters for devices set([u'1dc18405-6062-4a03-a2d1-3bf2636691bb'])
2014-11-09 17:47:19.053 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Port 1dc18405-6062-4a03-a2d1-3bf2636691bb updated. Details: {u'admin_state_up': True, u'network_id': u'1f09f7ce-4367-4fbf-a631-c22e928a4d56', u'segmentation_id': None, u'physical_network': None, u'device': u'1dc18405-6062-4a03-a2d1-3bf2636691bb', u'port_id': u'1dc18405-6062-4a03-a2d1-3bf2636691bb', u'network_type': u'local'}
2014-11-09 17:47:19.053 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Assigning 2 as local vlan for net-id=1f09f7ce-4367-4fbf-a631-c22e928a4d56
2014-11-09 17:47:19.354 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Configuration for device 1dc18405-6062-4a03-a2d1-3bf2636691bb completed.

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed : Network is unreachable

I have 2 servers both uses Openstack Icehouse installation on Ubuntu 14.04.1 System:

  • Server1 - Single node installation: Having controller, compute and network node in one system

  • Server2 - Compute node: Only compute service is running

I'm able to launch Ubuntu 14.04 cloud image on Server1 successfully. But when i launch Ubuntu VM on Server2 it gives:

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed [46/120s]: request error [HTTPConnectionPool(host='169.254.169.254', port=80): Max retries exceeded with url: /2009-04-04/meta-data/instance-id (Caused by <class 'socket.error'="">: [Errno 101] Network is unreachable)]

nova.conf on Server1:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = controller
vncserver_listen = controller
vncserver_proxyclient_address = controller
novncproxy_base_url = http://10.1.12.2:6080/vnc_auto.html

auth_strategy = keystone

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host=10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

nova.conf on Server2:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

auth_strategy=keystone

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = 10.1.12.17
vnc_enabled = True
vncserver_listen = 10.1.12.17
vncserver_proxyclient_address = 10.1.12.17
novncproxy_base_url = http://controller:6080/vnc_auto.html

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host = 10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

ovs-vsctl show on compute node:

Bridge br-int
        fail_mode: secure
        Port "int-br-eth0"
            Interface "int-br-eth0"
        Port "qvo454b3c53-1f"
            tag: 3
            Interface "qvo454b3c53-1f"
        Port br-int
            Interface br-int
                type: internal
    Bridge "br-eth0"
        Port "eth0"
            Interface "eth0"
        Port "br-eth0"
            Interface "br-eth0"
                type: internal
        Port "phy-br-eth0"
            Interface "phy-br-eth0"
    ovs_version: "2.0.2"

route -n on controller:

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.1.12.1       0.0.0.0         UG    0      0        0 br-ex
10.1.12.0       0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

ip netns on controller:

qdhcp-bf111ff7-99a4-4840-88fa-bfc334dad3d6
qrouter-cd88cf9f-753e-4941-8234-9edcd9b2363a

neutron-openvswitch-agent is installed on Compute Node: nova service-list (iseserver is Server1 and MCAServer is Server2) :

+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+
| Id | Binary           | Host      | Zone     | Status  | State | Updated_at                 | Disabled Reason |
+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+
| 1  | nova-cert        | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 2  | nova-consoleauth | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 3  | nova-scheduler   | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 4  | nova-conductor   | iseserver | internal | enabled | up    | 2014-11-08T18:32:11.000000 | -               |
| 6  | nova-compute     | iseserver | nova     | enabled | up    | 2014-11-08T18:32:08.000000 | -               |
| 7  | nova-compute     | MCAServer | nova     | enabled | up    | 2014-11-08T18:32:11.000000 | -               |
+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+

neutron agent-list (iseserver is Server1 and MCAServer is Server2):

+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+
| id                                   | agent_type         | host      | alive | admin_state_up | binary                    |
+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+
| 06e2025f-bb99-41c2-b3ca-369325a59ee9 | Metadata agent     | iseserver | :-)   | True           | neutron-metadata-agent    |
| 2f8db4c6-e328-4d6e-9d35-4cc34f7232b1 | Open vSwitch agent | MCAServer | :-)   | True           | neutron-openvswitch-agent |
| 87267e85-9f8f-4cf8-b5c7-1f16c4137c97 | Open vSwitch agent | iseserver | :-)   | True           | neutron-openvswitch-agent |
| 99fb263a-d3a0-4dcb-8a48-76c83b5dabcf | DHCP agent         | iseserver | :-)   | True           | neutron-dhcp-agent        |
| ad57613c-fb04-4c2a-b207-64ab8eab5a18 | L3 agent           | iseserver | :-)   | True           | neutron-l3-agent          |
+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+

Configured for VLAN aswell error still persists! On checking log files only problem i find is with openvswitch-agent.log on both Server1 and Server2. openvswitch-agent.log on Server1:

2014-11-09 18:03:23.443 2555 20:32:29.827 3069 ERROR neutron.agent.linux.ovsdb_monitor [-] Error received from ovsdb monitor: 2014-11-09T12:33:23Z|00001|fatal_signal|WARN|terminating 2014-11-09T15:02:29Z|00001|fatal_signal|WARN|terminating with signal 15 (Terminated)
2014-11-09 18:03:23.477 2555 20:32:30.732 3069 CRITICAL neutron [-] Trying to re-send() an already-triggered event.
2014-11-09 20:32:32.088 16747 WARNING neutron.agent.securitygroups_rpc [req-43a5a383-c5d1-4f04-b4fd-422ac5e00d41 None] Security group agent binding currently not set. This should be set by the end of the init process.

openvswitch-agent.log on Server2:

2014-11-09 17:32:05.912 16331 ERROR neutron.agent.linux.ovsdb_monitor [-] Error received from ovsdb monitor: 2014-11-09T12:02:05Z|00001|fatal_signal|WARN|terminating with signal 15 (Terminated)
2014-11-09 17:32:06.988 16331 CRITICAL neutron [-] Trying to re-send() an already-triggered event.
2014-11-09 17:32:06.988 16331 TRACE neutron Traceback (most recent call last):
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/bin/neutron-openvswitch-agent", line 10, in <module>
2014-11-09 17:32:06.988 16331 TRACE neutron     sys.exit(main())
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1476, in main
2014-11-09 17:32:06.988 16331 TRACE neutron     agent.daemon_loop()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1404, in daemon_loop
2014-11-09 17:32:06.988 16331 TRACE neutron     self.rpc_loop(polling_manager=pm)
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/contextlib.py", line 24, in __exit__
2014-11-09 17:32:06.988 16331 TRACE neutron     self.gen.next()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/polling.py", line 41, in get_polling_manager
2014-11-09 17:32:06.988 16331 TRACE neutron     pm.stop()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/polling.py", line 108, in stop
2014-11-09 17:32:06.988 16331 TRACE neutron     self._monitor.stop()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/async_process.py", line 91, in stop
2014-11-09 17:32:06.988 16331 TRACE neutron     self._kill()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/ovsdb_monitor.py", line 108, in _kill
2014-11-09 17:32:06.988 16331 TRACE neutron     super(SimpleInterfaceMonitor, self)._kill(*args, **kwargs)
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/lib/python2.7/dist-packages/neutron/agent/linux/async_process.py", line 118, in _kill
2014-11-09 17:32:06.988 16331 TRACE neutron     self._kill_event.send()
2014-11-09 17:32:06.988 16331 TRACE neutron   File "/usr/local/lib/python2.7/dist-packages/eventlet/event.py", line 155, in send
2014-11-09 17:32:06.988 16331 TRACE neutron     assert self._result is NOT_USED, 'Trying to re-send() an already-triggered event.'
2014-11-09 17:32:06.988 16331 TRACE neutron AssertionError: Trying to re-send() an already-triggered event.
2014-11-09 17:32:06.988 16331 TRACE neutron
2014-11-09 17:33:39.101 2196 INFO neutron.common.config [-] Logging enabled!
2014-11-09 17:33:40.118 2196 INFO neutron.openstack.common.rpc.common [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Connected to AMQP server on controller:5672
2014-11-09 17:33:40.139 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Mapping physical network physnet1 to bridge br-eth0
2014-11-09 17:33:40.158 2196 INFO neutron.openstack.common.rpc.common [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Connected to AMQP server on controller:5672
2014-11-09 17:33:41.353 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Agent initialized successfully, now running...
2014-11-09 17:33:41.356 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [req-f66d9766-4c8b-4500-b578-014114ac5563 None] Agent out of sync with plugin!
2014-11-09 17:34:44.309 2196 20:29:59.269 2037 INFO neutron.agent.securitygroups_rpc [req-ae754bf6-c1c0-46aa-82c7-2f610a7cd2e5 [req-9a5b141c-2984-449f-9510-181b21aa17ce None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:34:47.833 2196 20:32:32.090 2037 INFO neutron.agent.securitygroups_rpc [-] Preparing filters for devices set([u'cf51921e-261e-4b0d-ac7b-9e90fd4277cd'])
2014-11-09 17:34:47.847 2196 INFO neutron.openstack.common.rpc.common [-] Connected to AMQP server on controller:5672
2014-11-09 17:34:48.563 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Port cf51921e-261e-4b0d-ac7b-9e90fd4277cd updated. Details: {u'admin_state_up': True, u'network_id': u'bd0d9a3e-4361-4d09-ab9c-eacafe18ba01', u'segmentation_id': None, u'physical_network': None, u'device': u'cf51921e-261e-4b0d-ac7b-9e90fd4277cd', u'port_id': u'cf51921e-261e-4b0d-ac7b-9e90fd4277cd', u'network_type': u'local'}
2014-11-09 17:34:48.563 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Assigning 1 as local vlan for net-id=bd0d9a3e-4361-4d09-ab9c-eacafe18ba01
2014-11-09 17:34:48.817 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Configuration for device cf51921e-261e-4b0d-ac7b-9e90fd4277cd completed.
2014-11-09 17:40:28.741 2196 INFO neutron.agent.securitygroups_rpc [req-000ad03e-11e1-428d-ac8a-97a1aa7011cb [req-43a5a383-c5d1-4f04-b4fd-422ac5e00d41 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:40:30.149 2196 Provider rule updated
2014-11-09 20:32:34.121 2037 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 17:42:10.734 2196 INFO neutron.agent.securitygroups_rpc [req-7e34244c-2ca3-4221-9be2-3a835bb2a2d4 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:42:12.197 2196 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 17:42:12.567 2196 INFO neutron.agent.securitygroups_rpc [-] Remove device filter for set([u'cf51921e-261e-4b0d-ac7b-9e90fd4277cd'])
2014-11-09 17:42:12.892 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Attachment cf51921e-261e-4b0d-ac7b-9e90fd4277cd removed
2014-11-09 17:42:12.909 2196 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Reclaiming vlan = 1 from net-id = bd0d9a3e-4361-4d09-ab9c-eacafe18ba01
2014-11-09 17:42:23.650 2196 INFO neutron.agent.securitygroups_rpc [req-9762fbb5-eaa8-4149-a000-822aa60ab69a None] Provider rule updated
2014-11-09 17:42:24.199 2196 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 17:42:24.199 2196 20:32:34.121 2037 INFO neutron.agent.securitygroups_rpc [-] No ports here to refresh firewall
2014-11-09 17:45:08.604 2196 20:34:23.231 2037 INFO neutron.agent.securitygroups_rpc [req-e2f37d62-4e7a-40c6-925e-96dc24933523 [req-5a4a566a-337c-4b97-926c-68baada44432 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:45:08.891 2196 20:34:36.029 2037 INFO neutron.agent.securitygroups_rpc [req-b933e10e-1864-46f2-a897-3569fb17f172 [req-589efb3e-e114-4199-a701-c1309b916e89 None] Provider rule updated
2014-11-09 17:45:10.303 2196 20:34:38.227 2037 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 17:45:10.303 2196 20:34:38.227 2037 INFO neutron.agent.securitygroups_rpc [-] No ports here to refresh firewall
2014-11-09 17:47:15.171 2196 20:34:57.227 2037 INFO neutron.agent.securitygroups_rpc [req-b33a55e1-0cbf-41eb-b186-81026b0949cf [req-a9e3950f-89dc-4cac-b759-edb811e3d111 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 17:47:18.412 2196 20:34:57.544 2037 INFO neutron.agent.securitygroups_rpc [req-43a5a383-c5d1-4f04-b4fd-422ac5e00d41 None] Provider rule updated
2014-11-09 20:34:58.235 2037 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 20:34:58.235 2037 INFO neutron.agent.securitygroups_rpc [-] No ports here to refresh firewall
2014-11-09 20:37:41.222 2037 INFO neutron.agent.securitygroups_rpc [req-a32d566d-46ac-44e4-ad82-5bb33d2c5948 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 20:37:44.358 2037 INFO neutron.agent.securitygroups_rpc [-] Preparing filters for devices set([u'1dc18405-6062-4a03-a2d1-3bf2636691bb'])
2014-11-09 17:47:19.053 2196 set([u'b4811b1d-e59e-4667-8d64-0d8cab74d615'])
2014-11-09 20:37:45.036 2037 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Port 1dc18405-6062-4a03-a2d1-3bf2636691bb b4811b1d-e59e-4667-8d64-0d8cab74d615 updated. Details: {u'admin_state_up': True, u'network_id': u'1f09f7ce-4367-4fbf-a631-c22e928a4d56', u'178a4eee-f73d-4a58-b611-fb05c1fa32a4', u'segmentation_id': None, u'physical_network': None, u'device': u'1dc18405-6062-4a03-a2d1-3bf2636691bb', u'b4811b1d-e59e-4667-8d64-0d8cab74d615', u'port_id': u'1dc18405-6062-4a03-a2d1-3bf2636691bb', u'b4811b1d-e59e-4667-8d64-0d8cab74d615', u'network_type': u'local'}
2014-11-09 17:47:19.053 2196 20:37:45.037 2037 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Assigning 2 as local vlan for net-id=1f09f7ce-4367-4fbf-a631-c22e928a4d56
2014-11-09 17:47:19.354 2196 net-id=178a4eee-f73d-4a58-b611-fb05c1fa32a4
2014-11-09 20:37:45.356 2037 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Configuration for device 1dc18405-6062-4a03-a2d1-3bf2636691bb b4811b1d-e59e-4667-8d64-0d8cab74d615 completed.

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed : Network is unreachable

I have 2 servers both uses Openstack Icehouse installation on Ubuntu 14.04.1 System:

  • Server1 - Single node installation: Having controller, compute and network node in one system

  • Server2 - Compute node: Only compute service is running

I'm able to launch Ubuntu 14.04 cloud image on Server1 successfully. But when i launch Ubuntu VM on Server2 it gives:

Calling 'http://169.254.169.254/2009-04-04/meta-data/instance-id' failed [46/120s]: request error [HTTPConnectionPool(host='169.254.169.254', port=80): Max retries exceeded with url: /2009-04-04/meta-data/instance-id (Caused by <class 'socket.error'="">: [Errno 101] Network is unreachable)]

nova.conf on Server1:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = controller
vncserver_listen = controller
vncserver_proxyclient_address = controller
novncproxy_base_url = http://10.1.12.2:6080/vnc_auto.html

auth_strategy = keystone

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host=10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

nova.conf on Server2:

[DEFAULT]
dhcpbridge_flagfile=/etc/nova/nova.conf
dhcpbridge=/usr/bin/nova-dhcpbridge
logdir=/var/log/nova
state_path=/var/lib/nova
lock_path=/var/lock/nova
force_dhcp_release=True
iscsi_helper=tgtadm
libvirt_use_virtio_for_bridges=True
connection_type=libvirt
root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
verbose=True
ec2_private_dns_show_ip=True
api_paste_config=/etc/nova/api-paste.ini
volumes_path=/var/lib/nova/volumes
enabled_apis=ec2,osapi_compute,metadata

auth_strategy=keystone

rpc_backend = rabbit
rabbit_host = controller
rabbit_password = openstack

my_ip = 10.1.12.17
vnc_enabled = True
vncserver_listen = 10.1.12.17
vncserver_proxyclient_address = 10.1.12.17
novncproxy_base_url = http://controller:6080/vnc_auto.html

glance_host = controller

network_api_class = nova.network.neutronv2.api.API
neutron_url = http://controller:9696
neutron_auth_strategy = keystone
neutron_admin_tenant_name = service
neutron_admin_username = neutron
neutron_admin_password = openstack
neutron_admin_auth_url = http://controller:35357/v2.0
linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
firewall_driver = nova.virt.firewall.NoopFirewallDriver
security_group_api = neutron

metadata_host = 10.1.12.2
service_neutron_metadata_proxy = true
neutron_metadata_proxy_shared_secret = openstack

[database]
connection = mysql://nova:openstack@controller/nova

[keystone_authtoken]
auth_uri = http://controller:5000
auth_host = controller
auth_port = 35357
auth_protocol = http
admin_tenant_name = service
admin_user = nova
admin_password = openstack

ovs-vsctl show on compute node:

Bridge br-int
        fail_mode: secure
        Port "int-br-eth0"
            Interface "int-br-eth0"
        Port "qvo454b3c53-1f"
            tag: 3
            Interface "qvo454b3c53-1f"
        Port br-int
            Interface br-int
                type: internal
    Bridge "br-eth0"
        Port "eth0"
            Interface "eth0"
        Port "br-eth0"
            Interface "br-eth0"
                type: internal
        Port "phy-br-eth0"
            Interface "phy-br-eth0"
    ovs_version: "2.0.2"

route -n on controller:

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.1.12.1       0.0.0.0         UG    0      0        0 br-ex
10.1.12.0       0.0.0.0         255.255.255.0   U     0      0        0 br-ex
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0

ip netns on controller:

qdhcp-bf111ff7-99a4-4840-88fa-bfc334dad3d6
qrouter-cd88cf9f-753e-4941-8234-9edcd9b2363a

neutron-openvswitch-agent is installed on Compute Node: nova service-list (iseserver is Server1 and MCAServer is Server2) :

+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+
| Id | Binary           | Host      | Zone     | Status  | State | Updated_at                 | Disabled Reason |
+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+
| 1  | nova-cert        | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 2  | nova-consoleauth | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 3  | nova-scheduler   | iseserver | internal | enabled | up    | 2014-11-08T18:32:14.000000 | -               |
| 4  | nova-conductor   | iseserver | internal | enabled | up    | 2014-11-08T18:32:11.000000 | -               |
| 6  | nova-compute     | iseserver | nova     | enabled | up    | 2014-11-08T18:32:08.000000 | -               |
| 7  | nova-compute     | MCAServer | nova     | enabled | up    | 2014-11-08T18:32:11.000000 | -               |
+----+------------------+-----------+----------+---------+-------+----------------------------+-----------------+

neutron agent-list (iseserver is Server1 and MCAServer is Server2):

+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+
| id                                   | agent_type         | host      | alive | admin_state_up | binary                    |
+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+
| 06e2025f-bb99-41c2-b3ca-369325a59ee9 | Metadata agent     | iseserver | :-)   | True           | neutron-metadata-agent    |
| 2f8db4c6-e328-4d6e-9d35-4cc34f7232b1 | Open vSwitch agent | MCAServer | :-)   | True           | neutron-openvswitch-agent |
| 87267e85-9f8f-4cf8-b5c7-1f16c4137c97 | Open vSwitch agent | iseserver | :-)   | True           | neutron-openvswitch-agent |
| 99fb263a-d3a0-4dcb-8a48-76c83b5dabcf | DHCP agent         | iseserver | :-)   | True           | neutron-dhcp-agent        |
| ad57613c-fb04-4c2a-b207-64ab8eab5a18 | L3 agent           | iseserver | :-)   | True           | neutron-l3-agent          |
+--------------------------------------+--------------------+-----------+-------+----------------+---------------------------+

Configured for VLAN aswell error still persists! On checking log files only problem i find is with openvswitch-agent.log on both Server1 and Server2. openvswitch-agent.log on Server1:

2014-11-09 20:32:29.827 3069 ERROR neutron.agent.linux.ovsdb_monitor [-] Error received from ovsdb monitor: 2014-11-09T15:02:29Z|00001|fatal_signal|WARN|terminating with signal 15 (Terminated)
2014-11-09 20:32:30.732 3069 CRITICAL neutron [-] Trying to re-send() an already-triggered event.
2014-11-09 20:32:32.088 16747 WARNING neutron.agent.securitygroups_rpc [req-43a5a383-c5d1-4f04-b4fd-422ac5e00d41 None] Security group agent binding currently not set. This should be set by the end of the init process.

openvswitch-agent.log on Server2:

2014-11-09 20:29:59.269 2037 INFO neutron.agent.securitygroups_rpc [req-9a5b141c-2984-449f-9510-181b21aa17ce None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 20:32:32.090 2037 INFO neutron.agent.securitygroups_rpc [req-43a5a383-c5d1-4f04-b4fd-422ac5e00d41 None] Provider rule updated
2014-11-09 20:32:34.121 2037 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 20:32:34.121 2037 INFO neutron.agent.securitygroups_rpc [-] No ports here to refresh firewall
2014-11-09 20:34:23.231 2037 INFO neutron.agent.securitygroups_rpc [req-5a4a566a-337c-4b97-926c-68baada44432 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 20:34:36.029 2037 INFO neutron.agent.securitygroups_rpc [req-589efb3e-e114-4199-a701-c1309b916e89 None] Provider rule updated
2014-11-09 20:34:38.227 2037 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 20:34:38.227 2037 INFO neutron.agent.securitygroups_rpc [-] No ports here to refresh firewall
2014-11-09 20:34:57.227 2037 INFO neutron.agent.securitygroups_rpc [req-a9e3950f-89dc-4cac-b759-edb811e3d111 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 20:34:57.544 2037 INFO neutron.agent.securitygroups_rpc [req-43a5a383-c5d1-4f04-b4fd-422ac5e00d41 None] Provider rule updated
2014-11-09 20:34:58.235 2037 INFO neutron.agent.securitygroups_rpc [-] Refresh firewall rules
2014-11-09 20:34:58.235 2037 INFO neutron.agent.securitygroups_rpc [-] No ports here to refresh firewall
2014-11-09 20:37:41.222 2037 INFO neutron.agent.securitygroups_rpc [req-a32d566d-46ac-44e4-ad82-5bb33d2c5948 None] Security group member updated [u'1765233e-093d-420d-85db-c2d4b80f5803']
2014-11-09 20:37:44.358 2037 INFO neutron.agent.securitygroups_rpc [-] Preparing filters for devices set([u'b4811b1d-e59e-4667-8d64-0d8cab74d615'])
2014-11-09 20:37:45.036 2037 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Port b4811b1d-e59e-4667-8d64-0d8cab74d615 updated. Details: {u'admin_state_up': True, u'network_id': u'178a4eee-f73d-4a58-b611-fb05c1fa32a4', u'segmentation_id': None, u'physical_network': None, u'device': u'b4811b1d-e59e-4667-8d64-0d8cab74d615', u'port_id': u'b4811b1d-e59e-4667-8d64-0d8cab74d615', u'network_type': u'local'}
2014-11-09 20:37:45.037 2037 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Assigning 2 as local vlan for net-id=178a4eee-f73d-4a58-b611-fb05c1fa32a4
2014-11-09 20:37:45.356 2037 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent [-] Configuration for device b4811b1d-e59e-4667-8d64-0d8cab74d615 completed.

sudo iptables-save (Server1):

# Generated by iptables-save v1.4.21 on Sun Nov  9 21:11:55 2014
*mangle
:PREROUTING ACCEPT [462549:144137277]
:INPUT ACCEPT [461827:144041633]
:FORWARD ACCEPT [362:36690]
:OUTPUT ACCEPT [451561:144981426]
:POSTROUTING ACCEPT [451923:145018116]
:nova-api-POSTROUTING - [0:0]
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
COMMIT
# Completed on Sun Nov  9 21:11:55 2014
# Generated by iptables-save v1.4.21 on Sun Nov  9 21:11:55 2014
*nat
:PREROUTING ACCEPT [895:104131]
:INPUT ACCEPT [466:32977]
:OUTPUT ACCEPT [1946:118386]
:POSTROUTING ACCEPT [2000:125903]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-POSTROUTING - [0:0]
:neutron-openvswi-PREROUTING - [0:0]
:neutron-openvswi-float-snat - [0:0]
:neutron-openvswi-snat - [0:0]
:neutron-postrouting-bottom - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-POSTROUTING - [0:0]
:nova-api-PREROUTING - [0:0]
:nova-api-float-snat - [0:0]
:nova-api-snat - [0:0]
:nova-postrouting-bottom - [0:0]
-A PREROUTING -j neutron-openvswi-PREROUTING
-A PREROUTING -j nova-api-PREROUTING
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-api-OUTPUT
-A POSTROUTING -j neutron-openvswi-POSTROUTING
-A POSTROUTING -j neutron-postrouting-bottom
-A POSTROUTING -j nova-api-POSTROUTING
-A POSTROUTING -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE
-A POSTROUTING -j nova-postrouting-bottom
-A neutron-openvswi-snat -j neutron-openvswi-float-snat
-A neutron-postrouting-bottom -j neutron-openvswi-snat
-A nova-api-snat -j nova-api-float-snat
-A nova-postrouting-bottom -j nova-api-snat
COMMIT
# Completed on Sun Nov  9 21:11:55 2014
# Generated by iptables-save v1.4.21 on Sun Nov  9 21:11:55 2014
*filter
:INPUT ACCEPT [461543:143987681]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [451477:144952097]
:neutron-filter-top - [0:0]
:neutron-openvswi-FORWARD - [0:0]
:neutron-openvswi-INPUT - [0:0]
:neutron-openvswi-OUTPUT - [0:0]
:neutron-openvswi-i72b4f682-b - [0:0]
:neutron-openvswi-local - [0:0]
:neutron-openvswi-o72b4f682-b - [0:0]
:neutron-openvswi-s72b4f682-b - [0:0]
:neutron-openvswi-sg-chain - [0:0]
:neutron-openvswi-sg-fallback - [0:0]
:nova-api-FORWARD - [0:0]
:nova-api-INPUT - [0:0]
:nova-api-OUTPUT - [0:0]
:nova-api-local - [0:0]
:nova-filter-top - [0:0]
-A INPUT -j neutron-openvswi-INPUT
-A INPUT -j nova-api-INPUT
-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT
-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT
-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT
-A FORWARD -j neutron-filter-top
-A FORWARD -j neutron-openvswi-FORWARD
-A FORWARD -j nova-filter-top
-A FORWARD -j nova-api-FORWARD
-A FORWARD -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT
-A FORWARD -i virbr0 -o virbr0 -j ACCEPT
-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
-A OUTPUT -j neutron-filter-top
-A OUTPUT -j neutron-openvswi-OUTPUT
-A OUTPUT -j nova-filter-top
-A OUTPUT -j nova-api-OUTPUT
-A OUTPUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT
-A neutron-filter-top -j neutron-openvswi-local
-A neutron-openvswi-FORWARD -m physdev --physdev-out tap72b4f682-bc --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-FORWARD -m physdev --physdev-in tap72b4f682-bc --physdev-is-bridged -j neutron-openvswi-sg-chain
-A neutron-openvswi-INPUT -m physdev --physdev-in tap72b4f682-bc --physdev-is-bridged -j neutron-openvswi-o72b4f682-b
-A neutron-openvswi-i72b4f682-b -m state --state INVALID -j DROP
-A neutron-openvswi-i72b4f682-b -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-i72b4f682-b -s 192.168.199.4/32 -j RETURN
-A neutron-openvswi-i72b4f682-b -p icmp -m icmp --icmp-type 22/22 -j RETURN
-A neutron-openvswi-i72b4f682-b -p icmp -j RETURN
-A neutron-openvswi-i72b4f682-b -p tcp -m tcp --dport 22 -j RETURN
-A neutron-openvswi-i72b4f682-b -s 192.168.199.3/32 -p udp -m udp --sport 67 --dport 68 -j RETURN
-A neutron-openvswi-i72b4f682-b -j neutron-openvswi-sg-fallback
-A neutron-openvswi-o72b4f682-b -p udp -m udp --sport 68 --dport 67 -j RETURN
-A neutron-openvswi-o72b4f682-b -j neutron-openvswi-s72b4f682-b
-A neutron-openvswi-o72b4f682-b -p udp -m udp --sport 67 --dport 68 -j DROP
-A neutron-openvswi-o72b4f682-b -m state --state INVALID -j DROP
-A neutron-openvswi-o72b4f682-b -m state --state RELATED,ESTABLISHED -j RETURN
-A neutron-openvswi-o72b4f682-b -p tcp -m tcp -m multiport --dports 1:65535 -j RETURN
-A neutron-openvswi-o72b4f682-b -j RETURN
-A neutron-openvswi-o72b4f682-b -p icmp -j RETURN
-A neutron-openvswi-o72b4f682-b -j neutron-openvswi-sg-fallback
-A neutron-openvswi-s72b4f682-b -s 192.168.199.2/32 -m mac --mac-source FA:16:3E:72:72:21 -j RETURN
-A neutron-openvswi-s72b4f682-b -j DROP
-A neutron-openvswi-sg-chain -m physdev --physdev-out tap72b4f682-bc --physdev-is-bridged -j neutron-openvswi-i72b4f682-b
-A neutron-openvswi-sg-chain -m physdev --physdev-in tap72b4f682-bc --physdev-is-bridged -j neutron-openvswi-o72b4f682-b
-A neutron-openvswi-sg-chain -j ACCEPT
-A neutron-openvswi-sg-fallback -j DROP
-A nova-api-INPUT -d 10.1.12.2/32 -p tcp -m tcp --dport 8775 -j ACCEPT
-A nova-filter-top -j nova-api-local
COMMIT
# Completed on Sun Nov  9 21:11:55 2014