Revision history [back]

click to hide/show revision 1
initial version

Authorization Failed: [Errno 111] Connection refused

Hello,

I am using ubuntu 13.04 with openstack grizzly version. my openstack was working fine and i was able to commnicate to my vm instances from external network and vice versa, but suddenly due to some issue i started facing problem. I am getting "[Errno 111] Connection refused" error, I have faced this same issue many times in past and i use to install from scratch, but after some again i will end up in same problem. This happens not only for me all my peers also facing the same issue on their lab setup. This behavior make me think like after some time the database is getting corrupted or something similar to this . i would really appreciate if someone can help me to narrow down the root cause ?

Thanks in advance.

root@ubuntu150:/var# keystone --debug user-list

REQ: curl -i http://10.19.11.150:5000/v2.0/tokens -X POST -H "Content-Type: application/json" -H "User-Agent: python-keystoneclient" REQ BODY: {"auth": {"tenantName": "admin", "passwordCredentials": {"username": "admin", "password": "admin_pass"}}}

Authorization Failed: [Errno 111] Connection refused

root@ubuntu150:/var#

root@ubuntu150:~# quantum net-list [Errno 111] Connection refused root@ubuntu150:~# keystone user-list Authorization Failed: [Errno 111] Connection refused root@ubuntu150:~# quantum subnet-list [Errno 111] Connection refused

root@ubuntu150:/var# tail -n 30 /var/log/keystone/keystone.log

2013-05-28 23:40:48 WARNING [keystone.common.wsgi] You are not authorized to perform the requested action: admin_required 2013-05-29 02:08:20 WARNING [keystone.common.wsgi] You are not authorized to perform the requested action: admin_required

root@ubuntu150:/var# cat /var/log/nova/nova-* | grep -i error

2013-05-28 18:29:37.251 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.254 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.259 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.263 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.267 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.273 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:54:54.073 ERROR nova.openstack.common.rpc.common [req-f9508f02-6770-458b-a3ba-29aa6577efff None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55.075 ERROR nova.openstack.common.rpc.common [req-f9508f02-6770-458b-a3ba-29aa6577efff None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:25.688 ERROR nova.openstack.common.rpc.common [req-c65c9eb2-6e59-4a7c-9954-75d959db5901 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:26.690 ERROR nova.openstack.common.rpc.common [req-c65c9eb2-6e59-4a7c-9954-75d959db5901 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:30:19.131 49818 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.131 49818 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 18:30:19.131 49818 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.137 49818 ERROR nova.virt.libvirt.driver [-] Nova requires libvirt version 0.9.6 or greater. 2013-05-28 18:30:19.181 49818 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.181 49818 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 18:30:19.181 49818 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.184 49818 TRACE nova AttributeError: 'NoneType' object has no attribute 'numOfDomains' 2013-05-28 20:14:57.730 32112 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.730 32112 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 20:14:57.730 32112 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.734 32112 ERROR nova.virt.libvirt.driver [-] Nova requires libvirt version 0.9.6 or greater. 2013-05-28 20:14:57.774 32112 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.774 32112 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 20:14:57.774 32112 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.778 32112 TRACE nova AttributeError: 'NoneType' object has no attribute 'numOfDomains' 2013-05-28 20:54:52.612 ERROR nova.openstack.common.rpc.common [req-890cb581-daa6-47fe-ac61-3c06e18beda5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:53.614 ERROR nova.openstack.common.rpc.common [req-890cb581-daa6-47fe-ac61-3c06e18beda5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:25.539 ERROR nova.openstack.common.rpc.common [req-36fce978-cb96-4c48-b906-2846b3787a39 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:26.541 ERROR nova.openstack.common.rpc.common [req-36fce978-cb96-4c48-b906-2846b3787a39 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-29 00:52:16.821 ERROR nova.openstack.common.rpc.common [req-0fbb9609-e762-4654-a8e8-bdb974d6647c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-29 00:52:17.823 ERROR nova.openstack.common.rpc.common [req-0fbb9609-e762-4654-a8e8-bdb974d6647c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:37.246 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.250 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.254 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.259 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.263 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.269 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:54:54.130 ERROR nova.openstack.common.rpc.common [req-45e50907-5585-44a5-8b71-b031c7cf1a0c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55.132 ERROR nova.openstack.common.rpc.common [req-45e50907-5585-44a5-8b71-b031c7cf1a0c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:26.338 ERROR nova.openstack.common.rpc.common [req-d640e7fd-f84b-4970-8ab6-9f121df8c4b5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:37.289 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.292 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.297 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.301 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.306 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.312 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:54:54.128 ERROR nova.openstack.common.rpc.common [req-df46dfc1-76fa-47f4-aab6-ca2d2065cf78 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55.130 ERROR nova.openstack.common.rpc.common [req-df46dfc1-76fa-47f4-aab6-ca2d2065cf78 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:25.802 ERROR nova.openstack.common.rpc.common [req-d90f68f7-3ba2-4f60-9fda-b25e74e9e251 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:26.804 ERROR nova.openstack.common.rpc.common [req-d90f68f7-3ba2-4f60-9fda-b25e74e9e251 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:37.403 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.406 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.410 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.415 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.419 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.425 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:46:35.324 WARNING nova.scheduler.driver [req-d854503d-51d6-4e2f-82fc-1fbfd47f3927 aa1deb7e6dec40f980661cb013f8a025 7beece3fd5a1410aa0313bd6b3439179] [instance: a5415427-9dcb-4e7e-be28-1f2323518485] Setting instance to ERROR state. 2013-05-28 20:51:29.620 WARNING nova.scheduler.driver [req-3e31f901-4afb-4ca1-a726-c3e12be34b84 aa1deb7e6dec40f980661cb013f8a025 7beece3fd5a1410aa0313bd6b3439179] [instance: 3ea5803d-a2c3-407b-9bc6-e3ed31321821] Setting instance to ERROR state. 2013-05-28 20:54:51.655 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:52.657 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 20:54:55.661 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 23:56:26.338 ERROR nova.openstack.common.rpc.common [req-ff85637c-43cc-4405-9592-7f9d087ae997 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.

root@ubuntu150:/var# cat /var/log/mysql/error.log | grep -i error

ERROR: 1064 You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'ALTER TABLE user ADD column Show_view_priv enum('N','Y') CHARACTER SET utf8 NOT ' at line 1 130528 18:09:13 [ERROR] Aborting ERROR: 1050 Table 'plugin' already exists 130528 18:09:16 [ERROR] Aborting root@ubuntu150:/var#


root@ubuntu150:/var# cat /var/log/quantum/* | grep -i error

RuntimeError: 2013-05-29 04:08:06 ERROR [quantum.agent.dhcp_agent] Unable to sync network state. raise RuntimeError(m) RuntimeError: 2013-05-29 04:08:13 ERROR [quantum.agent.dhcp_agent] Unable to sync network state. raise RuntimeError(m) RuntimeError: 2013-05-28 18:31:15 ERROR [quantum.agent.l3_agent] Failed reporting state! yield self.ensure(_error_callback, _consume) error_callback(e) File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback 2013-05-28 18:31:19 ERROR [quantum.agent.l3_agent] Failed synchronizing routers yield self.ensure(_error_callback, _consume) error_callback(e) File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback 2013-05-28 18:31:19 ERROR [quantum.agent.l3_agent] The external network bridge 'br-ex' does not exist 2013-05-28 20:54:46 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:46 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:47 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 20:54:47 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 20:54:50 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 20:54:50 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 20:54:55 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 20:54:55 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 23:56:22 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:22 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:23 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:23 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:31 ERROR [quantum.agent.l3_agent] Failed synchronizing routers raise RuntimeError(m) RuntimeError: Stderr: '2013-05-28 23:56:31 ERROR [quantum.agent.linux.daemon] Pidfile /var/lib/quantum/external/pids/b0c375df-dcb3-407c-804e-7adceac255f5.pid already exist. Daemon already running?\n' 2013-05-28 23:57:26 ERROR [quantum.agent.l3_agent] Failed reporting state! yield self.ensure(_error_callback, _consume) error_callback(e) File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback 2013-05-29 00:52:13 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-29 00:52:13 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-29 00:52:14 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-29 00:52:14 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-29 00:52:17 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-29 00:52:17 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:30:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'del-flows', 'br-int']. Exception: 2013-05-28 18:30:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'add-flow', 'br-int', 'hard_timeout=0,idle_timeout=0,priority=1,actions=normal']. Exception: 2013-05-28 19:58:16 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 20:42:09 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 20:54:47 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 23:32:47 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 23:56:23 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-29 00:14:46 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-29 00:52:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-vsctl', '--timeout=2', '--', '--if-exists', 'del-port', 'br-int', 'patch-tun']. Exception: 2013-05-29 00:52:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'del-flows', 'br-int']. Exception: 2013-05-29 00:52:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'add-flow', 'br-int', 'hard_timeout=0,idle_timeout=0,priority=1,actions=normal']. Exception: 2013-05-29 00:52:16 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-vsctl', '--timeout=2', 'add-port', 'br-int', 'patch-tun']. Exception: 2013-05-29 00:52:17 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 23:56:31 ERROR [quantum.agent.linux.daemon] Pidfile /var/lib/quantum/external/pids/b0c375df-dcb3-407c-804e-7adceac255f5.pid already exist. Daemon already running? 2013-05-28 18:29:32 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:33 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:36 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:41 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:48 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:29:57 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 18:30:08 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 13 seconds. 2013-05-28 19:58:15 ERROR [quantum.openstack.common.rpc.amqp] Exception during message handling 2013-05-28 19:58:15 ERROR [quantum.openstack.common.rpc.common] Returning exception Agent with agent_type=DHCP agent and host=ubuntu150 could not be found to caller 2013-05-28 19:58:15 ERROR [quantum.openstack.common.rpc.common] ['Traceback (most recent call last):\n', ' File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/amqp.py", line 430, in _process_data\n rval = self.proxy.dispatch(ctxt, version, method, *args)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/common/rpc.py", line 43, in dispatch\n quantum_ctxt, version, method, *kwargs)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/dispatcher.py", line 133, in dispatch\n return getattr(proxyobj, method)(ctxt, **kwargs)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/db/dhcp_rpc_base.py", line 42, in get_active_networks\n context, host)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/db/agentschedulers_db.py", line 137, in list_active_networks_on_active_dhcp_agent\n context, constants.AGENT_TYPE_DHCP, host)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/db/agents_db.py", line 125, in _get_agent_by_type_and_host\n host=host)\n', 'AgentNotFoundByTypeHost: Agent with agent_type=DHCP agent and host=ubuntu150 could not be found\n'] 2013-05-28 20:35:12 ERROR [keystoneclient.common.cms] Verify error: Error opening certificate file /var/lib/quantum/keystone-signing/signing_cert.pem 140661410227904:error:02001002:system library:fopen:No such file or directory:bss_file.c:169:fopen('/var/lib/quantum/keystone-signing/signing_cert.pem','r') 140661410227904:error:2006D080:BIO routines:BIO_new_file:no such file:bss_file.c:172: 2013-05-28 20:35:12 ERROR [keystoneclient.common.cms] Verify error: Error loading file /var/lib/quantum/keystone-signing/cacert.pem 139860540176064:error:02001002:system library:fopen:No such file or directory:bss_file.c:169:fopen('/var/lib/quantum/keystone-signing/cacert.pem','r') 139860540176064:error:2006D080:BIO routines:BIO_new_file:no such file:bss_file.c:172: 139860540176064:error:0B084002:x509 certificate routines:X509_load_cert_crl_file:system lib:by_file.c:274: 2013-05-28 20:54:54 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 22:22:53 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:23:42 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:26:00 ERROR [quantum.api.v2.resource] delete failed 2013-05-28 22:30:28 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:30:45 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:30:54 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:36:59 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:37:13 ERROR [quantum.api.v2.resource] delete failed 2013-05-28 22:38:16 ERROR [quantum.api.v2.resource] show failed 2013-05-28 22:39:00 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:39:25 ERROR [quantum.api.v2.resource] delete failed 2013-05-28 22:57:21 ERROR [quantum.api.v2.resource] delete failed

root@ubuntu150:/var#

Authorization Failed: [Errno 111] Connection refused

Hello,

I am using ubuntu 13.04 with openstack grizzly version. my openstack was working fine and i was able to commnicate to my vm instances from external network and vice versa, but suddenly due to some issue i started facing problem. I am getting "[Errno 111] Connection refused" error, I have faced this same issue many times in past and i use to install reinstall from scratch, but after some sometime again i will end up in same problem. This happens every time not only for me all my peers also facing the same issue on their lab setup. This behavior make me think like after some time the database is getting corrupted or something similar to this . i would really appreciate if someone can help me to narrow down the root cause ?

Thanks in advance.

root@ubuntu150:/var# keystone --debug user-list

REQ: curl -i http://10.19.11.150:5000/v2.0/tokens -X POST -H "Content-Type: application/json" -H "User-Agent: python-keystoneclient" REQ BODY: {"auth": {"tenantName": "admin", "passwordCredentials": {"username": "admin", "password": "admin_pass"}}}

Authorization Failed: [Errno 111] Connection refused

root@ubuntu150:/var#

root@ubuntu150:~# quantum net-list [Errno 111] Connection refused root@ubuntu150:~# keystone user-list Authorization Failed: [Errno 111] Connection refused root@ubuntu150:~# quantum subnet-list [Errno 111] Connection refused

root@ubuntu150:/var# tail -n 30 /var/log/keystone/keystone.log

2013-05-28 23:40:48 WARNING [keystone.common.wsgi] You are not authorized to perform the requested action: admin_required 2013-05-29 02:08:20 WARNING [keystone.common.wsgi] You are not authorized to perform the requested action: admin_required

root@ubuntu150:/var# cat /var/log/nova/nova-* | grep -i error

2013-05-28 18:29:37.251 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.254 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.259 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.263 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.267 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.273 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:54:54.073 ERROR nova.openstack.common.rpc.common [req-f9508f02-6770-458b-a3ba-29aa6577efff None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55.075 ERROR nova.openstack.common.rpc.common [req-f9508f02-6770-458b-a3ba-29aa6577efff None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:25.688 ERROR nova.openstack.common.rpc.common [req-c65c9eb2-6e59-4a7c-9954-75d959db5901 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:26.690 ERROR nova.openstack.common.rpc.common [req-c65c9eb2-6e59-4a7c-9954-75d959db5901 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:30:19.131 49818 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.131 49818 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 18:30:19.131 49818 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.137 49818 ERROR nova.virt.libvirt.driver [-] Nova requires libvirt version 0.9.6 or greater. 2013-05-28 18:30:19.181 49818 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.181 49818 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 18:30:19.181 49818 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.184 49818 TRACE nova AttributeError: 'NoneType' object has no attribute 'numOfDomains' 2013-05-28 20:14:57.730 32112 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.730 32112 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 20:14:57.730 32112 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.734 32112 ERROR nova.virt.libvirt.driver [-] Nova requires libvirt version 0.9.6 or greater. 2013-05-28 20:14:57.774 32112 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.774 32112 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 20:14:57.774 32112 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.778 32112 TRACE nova AttributeError: 'NoneType' object has no attribute 'numOfDomains' 2013-05-28 20:54:52.612 ERROR nova.openstack.common.rpc.common [req-890cb581-daa6-47fe-ac61-3c06e18beda5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:53.614 ERROR nova.openstack.common.rpc.common [req-890cb581-daa6-47fe-ac61-3c06e18beda5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:25.539 ERROR nova.openstack.common.rpc.common [req-36fce978-cb96-4c48-b906-2846b3787a39 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:26.541 ERROR nova.openstack.common.rpc.common [req-36fce978-cb96-4c48-b906-2846b3787a39 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-29 00:52:16.821 ERROR nova.openstack.common.rpc.common [req-0fbb9609-e762-4654-a8e8-bdb974d6647c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-29 00:52:17.823 ERROR nova.openstack.common.rpc.common [req-0fbb9609-e762-4654-a8e8-bdb974d6647c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:37.246 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.250 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.254 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.259 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.263 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.269 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:54:54.130 ERROR nova.openstack.common.rpc.common [req-45e50907-5585-44a5-8b71-b031c7cf1a0c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55.132 ERROR nova.openstack.common.rpc.common [req-45e50907-5585-44a5-8b71-b031c7cf1a0c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:26.338 ERROR nova.openstack.common.rpc.common [req-d640e7fd-f84b-4970-8ab6-9f121df8c4b5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:37.289 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.292 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.297 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.301 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.306 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.312 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:54:54.128 ERROR nova.openstack.common.rpc.common [req-df46dfc1-76fa-47f4-aab6-ca2d2065cf78 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55.130 ERROR nova.openstack.common.rpc.common [req-df46dfc1-76fa-47f4-aab6-ca2d2065cf78 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:25.802 ERROR nova.openstack.common.rpc.common [req-d90f68f7-3ba2-4f60-9fda-b25e74e9e251 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:26.804 ERROR nova.openstack.common.rpc.common [req-d90f68f7-3ba2-4f60-9fda-b25e74e9e251 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:37.403 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.406 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.410 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.415 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.419 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.425 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:46:35.324 WARNING nova.scheduler.driver [req-d854503d-51d6-4e2f-82fc-1fbfd47f3927 aa1deb7e6dec40f980661cb013f8a025 7beece3fd5a1410aa0313bd6b3439179] [instance: a5415427-9dcb-4e7e-be28-1f2323518485] Setting instance to ERROR state. 2013-05-28 20:51:29.620 WARNING nova.scheduler.driver [req-3e31f901-4afb-4ca1-a726-c3e12be34b84 aa1deb7e6dec40f980661cb013f8a025 7beece3fd5a1410aa0313bd6b3439179] [instance: 3ea5803d-a2c3-407b-9bc6-e3ed31321821] Setting instance to ERROR state. 2013-05-28 20:54:51.655 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:52.657 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 20:54:55.661 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 23:56:26.338 ERROR nova.openstack.common.rpc.common [req-ff85637c-43cc-4405-9592-7f9d087ae997 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.

root@ubuntu150:/var# cat /var/log/mysql/error.log | grep -i error

ERROR: 1064 You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'ALTER TABLE user ADD column Show_view_priv enum('N','Y') CHARACTER SET utf8 NOT ' at line 1 130528 18:09:13 [ERROR] Aborting ERROR: 1050 Table 'plugin' already exists 130528 18:09:16 [ERROR] Aborting root@ubuntu150:/var#


root@ubuntu150:/var# cat /var/log/quantum/* | grep -i error

RuntimeError: 2013-05-29 04:08:06 ERROR [quantum.agent.dhcp_agent] Unable to sync network state. raise RuntimeError(m) RuntimeError: 2013-05-29 04:08:13 ERROR [quantum.agent.dhcp_agent] Unable to sync network state. raise RuntimeError(m) RuntimeError: 2013-05-28 18:31:15 ERROR [quantum.agent.l3_agent] Failed reporting state! yield self.ensure(_error_callback, _consume) error_callback(e) File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback 2013-05-28 18:31:19 ERROR [quantum.agent.l3_agent] Failed synchronizing routers yield self.ensure(_error_callback, _consume) error_callback(e) File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback 2013-05-28 18:31:19 ERROR [quantum.agent.l3_agent] The external network bridge 'br-ex' does not exist 2013-05-28 20:54:46 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:46 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:47 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 20:54:47 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 20:54:50 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 20:54:50 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 20:54:55 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 20:54:55 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 23:56:22 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:22 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:23 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:23 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:31 ERROR [quantum.agent.l3_agent] Failed synchronizing routers raise RuntimeError(m) RuntimeError: Stderr: '2013-05-28 23:56:31 ERROR [quantum.agent.linux.daemon] Pidfile /var/lib/quantum/external/pids/b0c375df-dcb3-407c-804e-7adceac255f5.pid already exist. Daemon already running?\n' 2013-05-28 23:57:26 ERROR [quantum.agent.l3_agent] Failed reporting state! yield self.ensure(_error_callback, _consume) error_callback(e) File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback 2013-05-29 00:52:13 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-29 00:52:13 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-29 00:52:14 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-29 00:52:14 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-29 00:52:17 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-29 00:52:17 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:30:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'del-flows', 'br-int']. Exception: 2013-05-28 18:30:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'add-flow', 'br-int', 'hard_timeout=0,idle_timeout=0,priority=1,actions=normal']. Exception: 2013-05-28 19:58:16 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 20:42:09 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 20:54:47 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 23:32:47 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 23:56:23 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-29 00:14:46 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-29 00:52:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-vsctl', '--timeout=2', '--', '--if-exists', 'del-port', 'br-int', 'patch-tun']. Exception: 2013-05-29 00:52:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'del-flows', 'br-int']. Exception: 2013-05-29 00:52:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'add-flow', 'br-int', 'hard_timeout=0,idle_timeout=0,priority=1,actions=normal']. Exception: 2013-05-29 00:52:16 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-vsctl', '--timeout=2', 'add-port', 'br-int', 'patch-tun']. Exception: 2013-05-29 00:52:17 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 23:56:31 ERROR [quantum.agent.linux.daemon] Pidfile /var/lib/quantum/external/pids/b0c375df-dcb3-407c-804e-7adceac255f5.pid already exist. Daemon already running? 2013-05-28 18:29:32 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:33 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:36 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:41 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:48 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:29:57 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 18:30:08 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 13 seconds. 2013-05-28 19:58:15 ERROR [quantum.openstack.common.rpc.amqp] Exception during message handling 2013-05-28 19:58:15 ERROR [quantum.openstack.common.rpc.common] Returning exception Agent with agent_type=DHCP agent and host=ubuntu150 could not be found to caller 2013-05-28 19:58:15 ERROR [quantum.openstack.common.rpc.common] ['Traceback (most recent call last):\n', ' File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/amqp.py", line 430, in _process_data\n rval = self.proxy.dispatch(ctxt, version, method, *args)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/common/rpc.py", line 43, in dispatch\n quantum_ctxt, version, method, *kwargs)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/dispatcher.py", line 133, in dispatch\n return getattr(proxyobj, method)(ctxt, **kwargs)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/db/dhcp_rpc_base.py", line 42, in get_active_networks\n context, host)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/db/agentschedulers_db.py", line 137, in list_active_networks_on_active_dhcp_agent\n context, constants.AGENT_TYPE_DHCP, host)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/db/agents_db.py", line 125, in _get_agent_by_type_and_host\n host=host)\n', 'AgentNotFoundByTypeHost: Agent with agent_type=DHCP agent and host=ubuntu150 could not be found\n'] 2013-05-28 20:35:12 ERROR [keystoneclient.common.cms] Verify error: Error opening certificate file /var/lib/quantum/keystone-signing/signing_cert.pem 140661410227904:error:02001002:system library:fopen:No such file or directory:bss_file.c:169:fopen('/var/lib/quantum/keystone-signing/signing_cert.pem','r') 140661410227904:error:2006D080:BIO routines:BIO_new_file:no such file:bss_file.c:172: 2013-05-28 20:35:12 ERROR [keystoneclient.common.cms] Verify error: Error loading file /var/lib/quantum/keystone-signing/cacert.pem 139860540176064:error:02001002:system library:fopen:No such file or directory:bss_file.c:169:fopen('/var/lib/quantum/keystone-signing/cacert.pem','r') 139860540176064:error:2006D080:BIO routines:BIO_new_file:no such file:bss_file.c:172: 139860540176064:error:0B084002:x509 certificate routines:X509_load_cert_crl_file:system lib:by_file.c:274: 2013-05-28 20:54:54 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 22:22:53 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:23:42 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:26:00 ERROR [quantum.api.v2.resource] delete failed 2013-05-28 22:30:28 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:30:45 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:30:54 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:36:59 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:37:13 ERROR [quantum.api.v2.resource] delete failed 2013-05-28 22:38:16 ERROR [quantum.api.v2.resource] show failed 2013-05-28 22:39:00 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:39:25 ERROR [quantum.api.v2.resource] delete failed 2013-05-28 22:57:21 ERROR [quantum.api.v2.resource] delete failed

root@ubuntu150:/var#

Authorization Failed: [Errno 111] Connection refused

Hello,

I am using ubuntu 13.04 with openstack grizzly version. my openstack was working fine and i was able to commnicate to my vm instances from external network and vice versa, but suddenly due to some issue i started facing problem. I am getting "[Errno 111] Connection refused" error, I have faced this same issue many times in past and i use to reinstall from scratch, but after sometime again i will end up in same problem. This happens every time not only for me all my peers also facing the same issue on their lab setup. This behavior make me think like after some time the database is getting corrupted or something similar to this . i would really appreciate if someone can help me to narrow down the root cause ?

Thanks in advance.

root@ubuntu150:/var# keystone --debug user-list

REQ: curl -i http://10.19.11.150:5000/v2.0/tokens -X POST -H "Content-Type: application/json" -H "User-Agent: python-keystoneclient" REQ BODY: {"auth": {"tenantName": "admin", "passwordCredentials": {"username": "admin", "password": "admin_pass"}}}"admin_pass"}}} Authorization Failed: [Errno 111] Connection refused

Authorization Failed: [Errno 111] Connection refusedroot@ubuntu150:~# quantum net-list

root@ubuntu150:/var#

root@ubuntu150:~# quantum net-list [Errno 111] Connection refused root@ubuntu150:~# keystone user-list Authorization Failed: [Errno 111] Connection refused refused

root@ubuntu150:~# quantum subnet-list subnet-list

[Errno 111] Connection refused

root@ubuntu150:/var# tail -n 30 /var/log/keystone/keystone.log

2013-05-28 23:40:48 WARNING [keystone.common.wsgi] You are not authorized to perform the requested action: admin_required 2013-05-29 02:08:20 WARNING [keystone.common.wsgi] You are not authorized to perform the requested action: admin_required

root@ubuntu150:/var# cat /var/log/nova/nova-* | grep -i error

2013-05-28 18:29:37.251 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.254 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.259 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.263 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.267 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.273 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:54:54.073 ERROR nova.openstack.common.rpc.common [req-f9508f02-6770-458b-a3ba-29aa6577efff None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55.075 ERROR nova.openstack.common.rpc.common [req-f9508f02-6770-458b-a3ba-29aa6577efff None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:25.688 ERROR nova.openstack.common.rpc.common [req-c65c9eb2-6e59-4a7c-9954-75d959db5901 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:26.690 ERROR nova.openstack.common.rpc.common [req-c65c9eb2-6e59-4a7c-9954-75d959db5901 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:30:19.131 49818 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.131 49818 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 18:30:19.131 49818 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.137 49818 ERROR nova.virt.libvirt.driver [-] Nova requires libvirt version 0.9.6 or greater. 2013-05-28 18:30:19.181 49818 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.181 49818 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 18:30:19.181 49818 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 18:30:19.184 49818 TRACE nova AttributeError: 'NoneType' object has no attribute 'numOfDomains' 2013-05-28 20:14:57.730 32112 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.730 32112 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 20:14:57.730 32112 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.734 32112 ERROR nova.virt.libvirt.driver [-] Nova requires libvirt version 0.9.6 or greater. 2013-05-28 20:14:57.774 32112 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.774 32112 TRACE nova.virt.libvirt.driver if ret is None:raise libvirtError('virConnectOpenAuth() failed') 2013-05-28 20:14:57.774 32112 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory 2013-05-28 20:14:57.778 32112 TRACE nova AttributeError: 'NoneType' object has no attribute 'numOfDomains' 2013-05-28 20:54:52.612 ERROR nova.openstack.common.rpc.common [req-890cb581-daa6-47fe-ac61-3c06e18beda5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:53.614 ERROR nova.openstack.common.rpc.common [req-890cb581-daa6-47fe-ac61-3c06e18beda5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:25.539 ERROR nova.openstack.common.rpc.common [req-36fce978-cb96-4c48-b906-2846b3787a39 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:26.541 ERROR nova.openstack.common.rpc.common [req-36fce978-cb96-4c48-b906-2846b3787a39 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-29 00:52:16.821 ERROR nova.openstack.common.rpc.common [req-0fbb9609-e762-4654-a8e8-bdb974d6647c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-29 00:52:17.823 ERROR nova.openstack.common.rpc.common [req-0fbb9609-e762-4654-a8e8-bdb974d6647c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:37.246 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.250 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.254 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.259 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.263 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.269 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:54:54.130 ERROR nova.openstack.common.rpc.common [req-45e50907-5585-44a5-8b71-b031c7cf1a0c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55.132 ERROR nova.openstack.common.rpc.common [req-45e50907-5585-44a5-8b71-b031c7cf1a0c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:26.338 ERROR nova.openstack.common.rpc.common [req-d640e7fd-f84b-4970-8ab6-9f121df8c4b5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:37.289 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.292 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.297 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.301 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.306 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.312 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:54:54.128 ERROR nova.openstack.common.rpc.common [req-df46dfc1-76fa-47f4-aab6-ca2d2065cf78 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55.130 ERROR nova.openstack.common.rpc.common [req-df46dfc1-76fa-47f4-aab6-ca2d2065cf78 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:25.802 ERROR nova.openstack.common.rpc.common [req-d90f68f7-3ba2-4f60-9fda-b25e74e9e251 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:26.804 ERROR nova.openstack.common.rpc.common [req-d90f68f7-3ba2-4f60-9fda-b25e74e9e251 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:37.403 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:38.406 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:41.410 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:46.415 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:53.419 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:30:02.425 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 20:46:35.324 WARNING nova.scheduler.driver [req-d854503d-51d6-4e2f-82fc-1fbfd47f3927 aa1deb7e6dec40f980661cb013f8a025 7beece3fd5a1410aa0313bd6b3439179] [instance: a5415427-9dcb-4e7e-be28-1f2323518485] Setting instance to ERROR state. 2013-05-28 20:51:29.620 WARNING nova.scheduler.driver [req-3e31f901-4afb-4ca1-a726-c3e12be34b84 aa1deb7e6dec40f980661cb013f8a025 7beece3fd5a1410aa0313bd6b3439179] [instance: 3ea5803d-a2c3-407b-9bc6-e3ed31321821] Setting instance to ERROR state. 2013-05-28 20:54:51.655 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:52.657 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 20:54:55.661 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 23:56:26.338 ERROR nova.openstack.common.rpc.common [req-ff85637c-43cc-4405-9592-7f9d087ae997 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.

root@ubuntu150:/var# cat /var/log/mysql/error.log | grep -i error

ERROR: 1064 You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'ALTER TABLE user ADD column Show_view_priv enum('N','Y') CHARACTER SET utf8 NOT ' at line 1 130528 18:09:13 [ERROR] Aborting ERROR: 1050 Table 'plugin' already exists 130528 18:09:16 [ERROR] Aborting root@ubuntu150:/var#


root@ubuntu150:/var# cat /var/log/quantum/* | grep -i error

RuntimeError: 2013-05-29 04:08:06 ERROR [quantum.agent.dhcp_agent] Unable to sync network state. raise RuntimeError(m) RuntimeError: 2013-05-29 04:08:13 ERROR [quantum.agent.dhcp_agent] Unable to sync network state. raise RuntimeError(m) RuntimeError: 2013-05-28 18:31:15 ERROR [quantum.agent.l3_agent] Failed reporting state! yield self.ensure(_error_callback, _consume) error_callback(e) File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback 2013-05-28 18:31:19 ERROR [quantum.agent.l3_agent] Failed synchronizing routers yield self.ensure(_error_callback, _consume) error_callback(e) File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback 2013-05-28 18:31:19 ERROR [quantum.agent.l3_agent] The external network bridge 'br-ex' does not exist 2013-05-28 20:54:46 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:46 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:47 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 20:54:47 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 20:54:50 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 20:54:50 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 20:54:55 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 20:54:55 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 23:56:22 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:22 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 23:56:23 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:23 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 23:56:31 ERROR [quantum.agent.l3_agent] Failed synchronizing routers raise RuntimeError(m) RuntimeError: Stderr: '2013-05-28 23:56:31 ERROR [quantum.agent.linux.daemon] Pidfile /var/lib/quantum/external/pids/b0c375df-dcb3-407c-804e-7adceac255f5.pid already exist. Daemon already running?\n' 2013-05-28 23:57:26 ERROR [quantum.agent.l3_agent] Failed reporting state! yield self.ensure(_error_callback, _consume) error_callback(e) File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback 2013-05-29 00:52:13 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-29 00:52:13 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-29 00:52:14 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-29 00:52:14 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-29 00:52:17 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-29 00:52:17 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:30:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'del-flows', 'br-int']. Exception: 2013-05-28 18:30:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'add-flow', 'br-int', 'hard_timeout=0,idle_timeout=0,priority=1,actions=normal']. Exception: 2013-05-28 19:58:16 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 20:42:09 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 20:54:47 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 23:32:47 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 23:56:23 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-29 00:14:46 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-29 00:52:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-vsctl', '--timeout=2', '--', '--if-exists', 'del-port', 'br-int', 'patch-tun']. Exception: 2013-05-29 00:52:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'del-flows', 'br-int']. Exception: 2013-05-29 00:52:15 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'add-flow', 'br-int', 'hard_timeout=0,idle_timeout=0,priority=1,actions=normal']. Exception: 2013-05-29 00:52:16 ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-vsctl', '--timeout=2', 'add-port', 'br-int', 'patch-tun']. Exception: 2013-05-29 00:52:17 ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated! 2013-05-28 23:56:31 ERROR [quantum.agent.linux.daemon] Pidfile /var/lib/quantum/external/pids/b0c375df-dcb3-407c-804e-7adceac255f5.pid already exist. Daemon already running? 2013-05-28 18:29:32 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 18:29:33 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 18:29:36 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds. 2013-05-28 18:29:41 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds. 2013-05-28 18:29:48 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds. 2013-05-28 18:29:57 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds. 2013-05-28 18:30:08 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 13 seconds. 2013-05-28 19:58:15 ERROR [quantum.openstack.common.rpc.amqp] Exception during message handling 2013-05-28 19:58:15 ERROR [quantum.openstack.common.rpc.common] Returning exception Agent with agent_type=DHCP agent and host=ubuntu150 could not be found to caller 2013-05-28 19:58:15 ERROR [quantum.openstack.common.rpc.common] ['Traceback (most recent call last):\n', ' File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/amqp.py", line 430, in _process_data\n rval = self.proxy.dispatch(ctxt, version, method, *args)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/common/rpc.py", line 43, in dispatch\n quantum_ctxt, version, method, *kwargs)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/dispatcher.py", line 133, in dispatch\n return getattr(proxyobj, method)(ctxt, **kwargs)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/db/dhcp_rpc_base.py", line 42, in get_active_networks\n context, host)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/db/agentschedulers_db.py", line 137, in list_active_networks_on_active_dhcp_agent\n context, constants.AGENT_TYPE_DHCP, host)\n', ' File "/usr/lib/python2.7/dist-packages/quantum/db/agents_db.py", line 125, in _get_agent_by_type_and_host\n host=host)\n', 'AgentNotFoundByTypeHost: Agent with agent_type=DHCP agent and host=ubuntu150 could not be found\n'] 2013-05-28 20:35:12 ERROR [keystoneclient.common.cms] Verify error: Error opening certificate file /var/lib/quantum/keystone-signing/signing_cert.pem 140661410227904:error:02001002:system library:fopen:No such file or directory:bss_file.c:169:fopen('/var/lib/quantum/keystone-signing/signing_cert.pem','r') 140661410227904:error:2006D080:BIO routines:BIO_new_file:no such file:bss_file.c:172: 2013-05-28 20:35:12 ERROR [keystoneclient.common.cms] Verify error: Error loading file /var/lib/quantum/keystone-signing/cacert.pem 139860540176064:error:02001002:system library:fopen:No such file or directory:bss_file.c:169:fopen('/var/lib/quantum/keystone-signing/cacert.pem','r') 139860540176064:error:2006D080:BIO routines:BIO_new_file:no such file:bss_file.c:172: 139860540176064:error:0B084002:x509 certificate routines:X509_load_cert_crl_file:system lib:by_file.c:274: 2013-05-28 20:54:54 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2013-05-28 20:54:55 ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds. 2013-05-28 22:22:53 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:23:42 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:26:00 ERROR [quantum.api.v2.resource] delete failed 2013-05-28 22:30:28 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:30:45 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:30:54 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:36:59 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:37:13 ERROR [quantum.api.v2.resource] delete failed 2013-05-28 22:38:16 ERROR [quantum.api.v2.resource] show failed 2013-05-28 22:39:00 ERROR [quantum.api.v2.resource] update failed 2013-05-28 22:39:25 ERROR [quantum.api.v2.resource] delete failed 2013-05-28 22:57:21 ERROR [quantum.api.v2.resource] delete failed

root@ubuntu150:/var#

Authorization Failed: [Errno 111] Connection refused

Hello,

I am using ubuntu 13.04 with openstack grizzly version. my openstack was working fine and i was able to commnicate to my vm instances from external network and vice versa, but suddenly due to some issue i started facing problem. I am getting "[Errno 111] Connection refused" error, I have faced this same issue many times in past and i use to reinstall from scratch, but after sometime again i will end up in same problem. This happens every time not only for me all my peers also facing the same issue on their lab setup. This behavior make me think like after some time the database is getting corrupted or something similar to this . i would really appreciate if someone can help me to narrow down the root cause ?

Thanks in advance.

root@ubuntu150:/var# keystone --debug user-list

REQ: curl -i http://10.19.11.150:5000/v2.0/tokens -X POST -H "Content-Type: application/json" -H "User-Agent: python-keystoneclient"
REQ BODY: {"auth": {"tenantName": "admin", "passwordCredentials": {"username": "admin", "password": "admin_pass"}}}
Authorization Failed: [Errno 111] Connection refused

refused

root@ubuntu150:~# quantum net-list

[Errno 111] Connection refused

root@ubuntu150:~# keystone user-list user-list

Authorization Failed: [Errno 111] Connection refused

refused

root@ubuntu150:~# quantum subnet-list

[Errno 111] Connection refused

refused

root@ubuntu150:/var# tail -n 30 /var/log/keystone/keystone.log

2013-05-28 23:40:48  WARNING [keystone.common.wsgi] You are not authorized to perform the requested action: admin_required
2013-05-29 02:08:20  WARNING [keystone.common.wsgi] You are not authorized to perform the requested action: admin_required

admin_required

root@ubuntu150:/var# cat /var/log/nova/nova-* | grep -i error

2013-05-28 18:29:37.251 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 18:29:38.254 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 18:29:41.259 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-28 18:29:46.263 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds.
2013-05-28 18:29:53.267 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds.
2013-05-28 18:30:02.273 ERROR nova.openstack.common.rpc.common [req-46d05539-0ed3-4104-a72f-c77359c657c7 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds.
2013-05-28 20:54:54.073 ERROR nova.openstack.common.rpc.common [req-f9508f02-6770-458b-a3ba-29aa6577efff None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 20:54:55.075 ERROR nova.openstack.common.rpc.common [req-f9508f02-6770-458b-a3ba-29aa6577efff None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 23:56:25.688 ERROR nova.openstack.common.rpc.common [req-c65c9eb2-6e59-4a7c-9954-75d959db5901 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 23:56:26.690 ERROR nova.openstack.common.rpc.common [req-c65c9eb2-6e59-4a7c-9954-75d959db5901 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 18:30:19.131 49818 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory
2013-05-28 18:30:19.131 49818 TRACE nova.virt.libvirt.driver     if ret is None:raise libvirtError('virConnectOpenAuth() failed')
2013-05-28 18:30:19.131 49818 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory
2013-05-28 18:30:19.137 49818 ERROR nova.virt.libvirt.driver [-] Nova requires libvirt version 0.9.6 or greater.
2013-05-28 18:30:19.181 49818 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory
2013-05-28 18:30:19.181 49818 TRACE nova.virt.libvirt.driver     if ret is None:raise libvirtError('virConnectOpenAuth() failed')
2013-05-28 18:30:19.181 49818 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory
2013-05-28 18:30:19.184 49818 TRACE nova AttributeError: 'NoneType' object has no attribute 'numOfDomains'
2013-05-28 20:14:57.730 32112 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory
2013-05-28 20:14:57.730 32112 TRACE nova.virt.libvirt.driver     if ret is None:raise libvirtError('virConnectOpenAuth() failed')
2013-05-28 20:14:57.730 32112 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory
2013-05-28 20:14:57.734 32112 ERROR nova.virt.libvirt.driver [-] Nova requires libvirt version 0.9.6 or greater.
2013-05-28 20:14:57.774 32112 ERROR nova.virt.libvirt.driver [-] Connection to libvirt failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory
2013-05-28 20:14:57.774 32112 TRACE nova.virt.libvirt.driver     if ret is None:raise libvirtError('virConnectOpenAuth() failed')
2013-05-28 20:14:57.774 32112 TRACE nova.virt.libvirt.driver libvirtError: Failed to connect socket to '/var/run/libvirt/libvirt-sock': No such file or directory
2013-05-28 20:14:57.778 32112 TRACE nova AttributeError: 'NoneType' object has no attribute 'numOfDomains'
2013-05-28 20:54:52.612 ERROR nova.openstack.common.rpc.common [req-890cb581-daa6-47fe-ac61-3c06e18beda5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 20:54:53.614 ERROR nova.openstack.common.rpc.common [req-890cb581-daa6-47fe-ac61-3c06e18beda5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 23:56:25.539 ERROR nova.openstack.common.rpc.common [req-36fce978-cb96-4c48-b906-2846b3787a39 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 23:56:26.541 ERROR nova.openstack.common.rpc.common [req-36fce978-cb96-4c48-b906-2846b3787a39 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-29 00:52:16.821 ERROR nova.openstack.common.rpc.common [req-0fbb9609-e762-4654-a8e8-bdb974d6647c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-29 00:52:17.823 ERROR nova.openstack.common.rpc.common [req-0fbb9609-e762-4654-a8e8-bdb974d6647c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 18:29:37.246 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 18:29:38.250 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 18:29:41.254 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-28 18:29:46.259 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds.
2013-05-28 18:29:53.263 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds.
2013-05-28 18:30:02.269 ERROR nova.openstack.common.rpc.common [req-346ff4d2-fd19-4c0f-9007-e836bfa5b321 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds.
2013-05-28 20:54:54.130 ERROR nova.openstack.common.rpc.common [req-45e50907-5585-44a5-8b71-b031c7cf1a0c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 20:54:55.132 ERROR nova.openstack.common.rpc.common [req-45e50907-5585-44a5-8b71-b031c7cf1a0c None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 23:56:26.338 ERROR nova.openstack.common.rpc.common [req-d640e7fd-f84b-4970-8ab6-9f121df8c4b5 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 18:29:37.289 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 18:29:38.292 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 18:29:41.297 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-28 18:29:46.301 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds.
2013-05-28 18:29:53.306 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds.
2013-05-28 18:30:02.312 ERROR nova.openstack.common.rpc.common [req-d0a968d6-d97a-4d2c-9476-f0a9f41f7b90 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds.
2013-05-28 20:54:54.128 ERROR nova.openstack.common.rpc.common [req-df46dfc1-76fa-47f4-aab6-ca2d2065cf78 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 20:54:55.130 ERROR nova.openstack.common.rpc.common [req-df46dfc1-76fa-47f4-aab6-ca2d2065cf78 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 23:56:25.802 ERROR nova.openstack.common.rpc.common [req-d90f68f7-3ba2-4f60-9fda-b25e74e9e251 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 23:56:26.804 ERROR nova.openstack.common.rpc.common [req-d90f68f7-3ba2-4f60-9fda-b25e74e9e251 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 18:29:37.403 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 18:29:38.406 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 18:29:41.410 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-28 18:29:46.415 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds.
2013-05-28 18:29:53.419 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds.
2013-05-28 18:30:02.425 ERROR nova.openstack.common.rpc.common [req-f83c9d6c-b21d-4dc6-b0dc-4acaa038c6e2 None None] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds.
2013-05-28 20:46:35.324 WARNING nova.scheduler.driver [req-d854503d-51d6-4e2f-82fc-1fbfd47f3927 aa1deb7e6dec40f980661cb013f8a025 7beece3fd5a1410aa0313bd6b3439179] [instance: a5415427-9dcb-4e7e-be28-1f2323518485] Setting instance to ERROR state.
2013-05-28 20:51:29.620 WARNING nova.scheduler.driver [req-3e31f901-4afb-4ca1-a726-c3e12be34b84 aa1deb7e6dec40f980661cb013f8a025 7beece3fd5a1410aa0313bd6b3439179] [instance: 3ea5803d-a2c3-407b-9bc6-e3ed31321821] Setting instance to ERROR state.
2013-05-28 20:54:51.655 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 20:54:52.657 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 20:54:55.661 ERROR nova.openstack.common.rpc.common [req-ecd3631b-0d10-4ef8-a3c4-0d5fb550cee2 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-28 23:56:26.338 ERROR nova.openstack.common.rpc.common [req-ff85637c-43cc-4405-9592-7f9d087ae997 None None] AMQP server on 10.19.11.150:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.

seconds.

root@ubuntu150:/var# cat /var/log/mysql/error.log | grep -i error

ERROR: 1064  You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'ALTER TABLE user ADD column Show_view_priv enum('N','Y') CHARACTER SET utf8 NOT ' at line 1
130528 18:09:13 [ERROR] Aborting
ERROR: 1050  Table 'plugin' already exists
130528 18:09:16 [ERROR] Aborting

root@ubuntu150:/var#


root@ubuntu150:/var# cat /var/log/quantum/* | grep -i error

RuntimeError:
2013-05-29 04:08:06    ERROR [quantum.agent.dhcp_agent] Unable to sync network state.
    raise RuntimeError(m)
RuntimeError:
2013-05-29 04:08:13    ERROR [quantum.agent.dhcp_agent] Unable to sync network state.
    raise RuntimeError(m)
RuntimeError:
2013-05-28 18:31:15    ERROR [quantum.agent.l3_agent] Failed reporting state!
    yield self.ensure(_error_callback, _consume)
    error_callback(e)
  File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback
2013-05-28 18:31:19    ERROR [quantum.agent.l3_agent] Failed synchronizing routers
    yield self.ensure(_error_callback, _consume)
    error_callback(e)
  File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback
2013-05-28 18:31:19    ERROR [quantum.agent.l3_agent] The external network bridge 'br-ex' does not exist
2013-05-28 20:54:46    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 20:54:46    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 20:54:47    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 20:54:47    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 20:54:50    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-28 20:54:50    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-28 20:54:55    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds.
2013-05-28 20:54:55    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds.
2013-05-28 23:56:22    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 23:56:22    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 23:56:23    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 23:56:23    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 23:56:31    ERROR [quantum.agent.l3_agent] Failed synchronizing routers
    raise RuntimeError(m)
RuntimeError:
Stderr: '2013-05-28 23:56:31    ERROR [quantum.agent.linux.daemon] Pidfile /var/lib/quantum/external/pids/b0c375df-dcb3-407c-804e-7adceac255f5.pid already exist. Daemon already running?\n'
2013-05-28 23:57:26    ERROR [quantum.agent.l3_agent] Failed reporting state!
    yield self.ensure(_error_callback, _consume)
    error_callback(e)
  File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/impl_kombu.py", line 629, in _error_callback
2013-05-29 00:52:13    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-29 00:52:13    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-29 00:52:14    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-29 00:52:14    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-29 00:52:17    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-29 00:52:17    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-28 18:30:15    ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'del-flows', 'br-int']. Exception:
2013-05-28 18:30:15    ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'add-flow', 'br-int', 'hard_timeout=0,idle_timeout=0,priority=1,actions=normal']. Exception:
2013-05-28 19:58:16    ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated!
2013-05-28 20:42:09    ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated!
2013-05-28 20:54:47    ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated!
2013-05-28 23:32:47    ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated!
2013-05-28 23:56:23    ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated!
2013-05-29 00:14:46    ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated!
2013-05-29 00:52:15    ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-vsctl', '--timeout=2', '--', '--if-exists', 'del-port', 'br-int', 'patch-tun']. Exception:
2013-05-29 00:52:15    ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'del-flows', 'br-int']. Exception:
2013-05-29 00:52:15    ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-ofctl', 'add-flow', 'br-int', 'hard_timeout=0,idle_timeout=0,priority=1,actions=normal']. Exception:
2013-05-29 00:52:16    ERROR [quantum.agent.linux.ovs_lib] Unable to execute ['ovs-vsctl', '--timeout=2', 'add-port', 'br-int', 'patch-tun']. Exception:
2013-05-29 00:52:17    ERROR [quantum.plugins.openvswitch.agent.ovs_quantum_agent] Failed to create OVS patch port. Cannot have tunneling enabled on this agent, since this version of OVS does not support tunnels or patch ports. Agent terminated!
2013-05-28 23:56:31    ERROR [quantum.agent.linux.daemon] Pidfile /var/lib/quantum/external/pids/b0c375df-dcb3-407c-804e-7adceac255f5.pid already exist. Daemon already running?
2013-05-28 18:29:32    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 18:29:33    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 18:29:36    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 5 seconds.
2013-05-28 18:29:41    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 7 seconds.
2013-05-28 18:29:48    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 9 seconds.
2013-05-28 18:29:57    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 11 seconds.
2013-05-28 18:30:08    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 13 seconds.
2013-05-28 19:58:15    ERROR [quantum.openstack.common.rpc.amqp] Exception during message handling
2013-05-28 19:58:15    ERROR [quantum.openstack.common.rpc.common] Returning exception Agent with agent_type=DHCP agent and host=ubuntu150 could not be found to caller
2013-05-28 19:58:15    ERROR [quantum.openstack.common.rpc.common] ['Traceback (most recent call last):\n', '  File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/amqp.py", line 430, in _process_data\n    rval = self.proxy.dispatch(ctxt, version, method, *args)\n', **args)\n', '  File "/usr/lib/python2.7/dist-packages/quantum/common/rpc.py", line 43, in dispatch\n    quantum_ctxt, version, method, *kwargs)\n', **kwargs)\n', '  File "/usr/lib/python2.7/dist-packages/quantum/openstack/common/rpc/dispatcher.py", line 133, in dispatch\n    return getattr(proxyobj, method)(ctxt, **kwargs)\n', '  File "/usr/lib/python2.7/dist-packages/quantum/db/dhcp_rpc_base.py", line 42, in get_active_networks\n    context, host)\n', '  File "/usr/lib/python2.7/dist-packages/quantum/db/agentschedulers_db.py", line 137, in list_active_networks_on_active_dhcp_agent\n    context, constants.AGENT_TYPE_DHCP, host)\n', '  File "/usr/lib/python2.7/dist-packages/quantum/db/agents_db.py", line 125, in _get_agent_by_type_and_host\n    host=host)\n', 'AgentNotFoundByTypeHost: Agent with agent_type=DHCP agent and host=ubuntu150 could not be found\n']
2013-05-28 20:35:12    ERROR [keystoneclient.common.cms] Verify error: Error opening certificate file /var/lib/quantum/keystone-signing/signing_cert.pem
140661410227904:error:02001002:system library:fopen:No such file or directory:bss_file.c:169:fopen('/var/lib/quantum/keystone-signing/signing_cert.pem','r')
140661410227904:error:2006D080:BIO routines:BIO_new_file:no such file:bss_file.c:172:
2013-05-28 20:35:12    ERROR [keystoneclient.common.cms] Verify error: Error loading file /var/lib/quantum/keystone-signing/cacert.pem
139860540176064:error:02001002:system library:fopen:No such file or directory:bss_file.c:169:fopen('/var/lib/quantum/keystone-signing/cacert.pem','r')
139860540176064:error:2006D080:BIO routines:BIO_new_file:no such file:bss_file.c:172:
139860540176064:error:0B084002:x509 certificate routines:X509_load_cert_crl_file:system lib:by_file.c:274:
2013-05-28 20:54:54    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds.
2013-05-28 20:54:55    ERROR [quantum.openstack.common.rpc.common] AMQP server on localhost:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 3 seconds.
2013-05-28 22:22:53    ERROR [quantum.api.v2.resource] update failed
2013-05-28 22:23:42    ERROR [quantum.api.v2.resource] update failed
2013-05-28 22:26:00    ERROR [quantum.api.v2.resource] delete failed
2013-05-28 22:30:28    ERROR [quantum.api.v2.resource] update failed
2013-05-28 22:30:45    ERROR [quantum.api.v2.resource] update failed
2013-05-28 22:30:54    ERROR [quantum.api.v2.resource] update failed
2013-05-28 22:36:59    ERROR [quantum.api.v2.resource] update failed
2013-05-28 22:37:13    ERROR [quantum.api.v2.resource] delete failed
2013-05-28 22:38:16    ERROR [quantum.api.v2.resource] show failed
2013-05-28 22:39:00    ERROR [quantum.api.v2.resource] update failed
2013-05-28 22:39:25    ERROR [quantum.api.v2.resource] delete failed
2013-05-28 22:57:21    ERROR [quantum.api.v2.resource] delete failed

root@ubuntu150:/var#

failed