Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

same problem here, any fix/news?

same problem here, any fix/news?fix/news? Here is the relevant part in nova.log on my compute node:

2015-02-11 10:48:43.091 1067 ERROR oslo.messaging.rpc.dispatcher [req-9be93728-5e5e-423d-9797-ceef24994330 ] Exception during message handling: Timed out waiting for a reply to message ID 9b98aebd76dc43b3a0239daa68673050
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher Traceback (most recent call last):
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/dispatcher.py", line 134, in _dispatch_and_reply
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     incoming.message))
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/dispatcher.py", line 177, in _dispatch
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return self._do_dispatch(endpoint, method, ctxt, args)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/dispatcher.py", line 123, in _do_dispatch
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     result = getattr(endpoint, method)(ctxt, **new_args)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/exception.py", line 88, in wrapped
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     payload)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/exception.py", line 71, in wrapped
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return f(self, context, *args, **kw)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 298, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     pass
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 284, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return function(self, context, *args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 348, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return function(self, context, *args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 326, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     kwargs['instance'], e, sys.exc_info())
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 314, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return function(self, context, *args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2522, in terminate_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     do_terminate_instance(instance, bdms)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/lockutils.py", line 272, in inner
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return f(*args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2520, in do_terminate_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._set_instance_error_state(context, instance)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2510, in do_terminate_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._delete_instance(context, instance, bdms, quotas)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/hooks.py", line 131, in inner
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     rv = f(*args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2479, in _delete_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     quotas.rollback()
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2456, in _delete_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._shutdown_instance(context, instance, bdms)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2389, in _shutdown_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._try_deallocate_network(context, instance, requested_networks)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2319, in _try_deallocate_network
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._set_instance_error_state(context, instance)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2314, in _try_deallocate_network
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._deallocate_network(context, instance, requested_networks)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 1918, in _deallocate_network
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     context, instance, requested_networks=requested_networks)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/network/api.py", line 48, in wrapped
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return func(self, context, *args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/network/api.py", line 299, in deallocate_for_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     requested_networks=requested_networks)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/network/rpcapi.py", line 210, in deallocate_for_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return cctxt.call(ctxt, 'deallocate_for_instance', **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/client.py", line 389, in call
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return self.prepare().call(ctxt, method, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/client.py", line 152, in call
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     retry=self.retry)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/transport.py", line 90, in _send
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     timeout=timeout, retry=retry)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 408, in send
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     retry=retry)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 397, in _send
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     result = self._waiter.wait(msg_id, timeout)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 298, in wait
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     reply, ending, trylock = self._poll_queue(msg_id, timeout)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 238, in _poll_queue
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     message = self.waiters.get(msg_id, timeout)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 144, in get
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     'to message ID %s' % msg_id)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher MessagingTimeout: Timed out waiting for a reply to message ID 9b98aebd76dc43b3a0239daa68673050
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher

same problem here, any fix/news? Here is the relevant part in nova.log on my compute node:

2015-02-11 10:48:43.091 1067 ERROR oslo.messaging.rpc.dispatcher [req-9be93728-5e5e-423d-9797-ceef24994330 ] Exception during message handling: Timed out waiting for a reply to message ID 9b98aebd76dc43b3a0239daa68673050
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher Traceback (most recent call last):
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/dispatcher.py", line 134, in _dispatch_and_reply
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     incoming.message))
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/dispatcher.py", line 177, in _dispatch
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return self._do_dispatch(endpoint, method, ctxt, args)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/dispatcher.py", line 123, in _do_dispatch
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     result = getattr(endpoint, method)(ctxt, **new_args)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/exception.py", line 88, in wrapped
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     payload)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/exception.py", line 71, in wrapped
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return f(self, context, *args, **kw)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 298, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     pass
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 284, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return function(self, context, *args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 348, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return function(self, context, *args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 326, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     kwargs['instance'], e, sys.exc_info())
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 314, in decorated_function
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return function(self, context, *args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2522, in terminate_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     do_terminate_instance(instance, bdms)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/lockutils.py", line 272, in inner
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return f(*args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2520, in do_terminate_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._set_instance_error_state(context, instance)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2510, in do_terminate_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._delete_instance(context, instance, bdms, quotas)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/hooks.py", line 131, in inner
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     rv = f(*args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2479, in _delete_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     quotas.rollback()
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2456, in _delete_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._shutdown_instance(context, instance, bdms)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2389, in _shutdown_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._try_deallocate_network(context, instance, requested_networks)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2319, in _try_deallocate_network
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._set_instance_error_state(context, instance)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     six.reraise(self.type_, self.value, self.tb)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 2314, in _try_deallocate_network
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     self._deallocate_network(context, instance, requested_networks)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/compute/manager.py", line 1918, in _deallocate_network
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     context, instance, requested_networks=requested_networks)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/network/api.py", line 48, in wrapped
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return func(self, context, *args, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/network/api.py", line 299, in deallocate_for_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     requested_networks=requested_networks)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/nova/network/rpcapi.py", line 210, in deallocate_for_instance
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return cctxt.call(ctxt, 'deallocate_for_instance', **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/client.py", line 389, in call
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     return self.prepare().call(ctxt, method, **kwargs)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/rpc/client.py", line 152, in call
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     retry=self.retry)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/transport.py", line 90, in _send
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     timeout=timeout, retry=retry)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 408, in send
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     retry=retry)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 397, in _send
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     result = self._waiter.wait(msg_id, timeout)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 298, in wait
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     reply, ending, trylock = self._poll_queue(msg_id, timeout)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 238, in _poll_queue
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     message = self.waiters.get(msg_id, timeout)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher   File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 144, in get
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher     'to message ID %s' % msg_id)
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher MessagingTimeout: Timed out waiting for a reply to message ID 9b98aebd76dc43b3a0239daa68673050
2015-02-11 10:48:43.091 1067 TRACE oslo.messaging.rpc.dispatcher

I have restarted rabbit-mq server, same problem.