Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

heat stack-list - Unexpected error occurred serving API: Timed out waiting for a reply to message ID 7daf49ca17cd4070bcfa1041890f84ae

We came across an issue when execute 'heat stack-list' and by investigating the log of Heat api we saw the error message like below "Unexpected error occurred serving API: Timed out waiting for a reply to message ID 7daf49ca17cd4070bcfa1041890f84ae" ---->

#

2016-06-22 14:14:33.868 15417 DEBUG heat.common.serializers [req-d8ee1b0a-6568-4482-a177-e8b76356d375 - 69a1bf21012b4311b01d3f998ca95934-0c61356e-f17d-4b8d-ac33-64279d0] JSON response : {"explanation": "The server has ei ther erred or is incapable of performing the requested operation.", "code": 500, "error": {"message": "Timed out waiting for a reply to message ID 39cea53fd8de46b2a8332c32e047df59", "traceback": "Traceback (most recent c all last):\n File \"/usr/lib/python2.7/site-packages/heat/api/middleware/fault.py\", line 156, in process_request\n return req.get_response(self.application)\n File \"/usr/lib/python2.7/site-packages/webob/request.p y\", line 1296, in send\n application, catch_exc_info=False)\n File \"/usr/lib/python2.7/site-packages/webob/request.py\", line 1260, in call_application\n app_iter = application(self.environ, start_response)\n F ile \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 130, in __call__\n resp = self.call_func(req, args, *self.kwargs)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 195, in call_func\n re turn self.func(req, args, *kwargs)\n File \"/usr/lib/python2.7/site-packages/heat/common/wsgi.py\", line 390, in __call__\n response = req.get_response(self.application)\n File \"/usr/lib/python2.7/site-packages/w ebob/request.py\", line 1296, in send\n application, catch_exc_info=False)\n File \"/usr/lib/python2.7/site-packages/webob/request.py\", line 1260, in call_application\n app_iter = application(self.environ, start_ response)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 130, in __call__\n resp = self.call_func(req, args, *self.kwargs)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 195, in cal l_func\n return self.func(req, args, *kwargs)\n File \"/usr/lib/python2.7/site-packages/heat/common/wsgi.py\", line 390, in __call__\n response = req.get_response(self.application)\n File \"/usr/lib/python2.7/s ite-packages/webob/request.py\", line 1296, in send\n application, catch_exc_info=False)\n File \"/usr/lib/python2.7/site-packages/webob/request.py\", line 1260, in call_application\n app_iter = application(self.e nviron, start_response)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 130, in __call__\n resp = self.call_func(req, args, *self.kwargs)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", li ne 195, in call_func\n return self.func(req, args, *kwargs)\n File \"/usr/lib/python2.7/site-packages/osprofiler/web.py\", line 99, in __call__\n return request.get_response(self.application)\n File \"/usr/lib/ python2.7/site-packages/webob/request.py\", line 1296, in send\n application, catch_exc_info=False)\n File \"/usr/lib/python2.7/site-packages/webob/request.py\", line 1260, in call_application\n app_iter = applica tion(self.environ, start_response)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 130, in __call__\n resp = self.call_func(req, args, *self.kwargs)\n File \"/usr/lib/python2.7/site-packages/webob/d ec.py\", line 195, in call_func\n return self.func(req, args, *kwargs)\n File \"/usr/lib/python2.7/site-packages/heat/common/wsgi.py\", line 390, in __call__\n response = req.get_response(self.application)\n Fi le \"/usr/lib/python2.7/site-packages/webob/request.py\", line 1296, in send\n application, catch_exc_info=False)\n File \"/usr/lib/python2.7/site-packages/webob/request.py\", line 1260, in call_application\n app_ iter = application(self.environ, start_response)\n File \"/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py\", line 634, in __call__\n return self._call_app(env, start_response)\n File \"/us r/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py\", line 554, in _call_app\n return self._app(env, _fake_start_response)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 130, in _ _call__\n resp = self.call_func(req, args, *self.kwargs)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 195, in call_func\n return self.func(req, args, *kwargs)\n File \"/usr/lib/python2.7/sit e-packages/heat/common/wsgi.py\", line 390, in __call__\n response = req.get_response(self.application)\n File \"/usr/lib/python2.7/site-packages/webob/request.py\", line 1296, in send\n application, catch_exc_inf o=False)\n File \"/usr/lib/python2.7/site-packages/webob/request.py\", line 1260, in call_application\n app_iter = application(self.environ, start_response)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 144, in __call__\n return resp(environ, start_response)\n File \"/usr/lib/python2.7/site-packages/routes/middleware.py\", line 131, in __call__\n response = self.app(environ, start_response)\n File \"/usr/li b/python2.7/site-packages/webob/dec.py\", line 144, in __call__\n return resp(environ, start_response)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 130, in __call__\n resp = self.call_func(req, * args, *self.kwargs)\n File \"/usr/lib/python2.7/site-packages/webob/dec.py\", line 195, in call_func\n return self.func(req, *args, *kwargs)\n File \"/usr/lib/python2.7/site-packages/heat/common/wsgi.py\", line 68 2, in __call__\n raise translate_exception(err, request.best_match_language())\nMessagingTimeout: Timed out waiting for a reply to message ID 39cea53fd8de46b2a8332c32e047df59\n", "type": "MessagingTimeout"}, "title": "Internal Server Error"} to_json /usr/lib/python2.7/site-packages/heat/common/serializers.py:42 2016-06-22 14:14:33.869 15417 INFO eventlet.wsgi.server [req-d8ee1b0a-6568-4482-a177-e8b76356d375 - 69a1bf21012b4311b01d3f998ca95934-0c61356e-f17d-4b8d-ac33-64279d0] 172.18.34.32 - - [22/Jun/2016 14:14:33] "GET /v1/92333 d69318b42acb4913a4ccdbc3bbe/stacks/B-WASMULTI-RITMxxxx012-TASKxxxx102-server_group-kozo7us4rlux-0-ctuzizzqjz2b%2F8f3ffff3-591b-46a8-82ec-e6545f57ea5d/resources/server/metadata HTTP/1.1" 500 5572 60.092561 2016-06-22 14:14:34.026 15418 ERROR heat.common.wsgi [req-8f18d6d4-2877-4a0b-9be0-d7cfbfa9ba2d - 69a1bf21012b4311b01d3f998ca95934-9422635d-25e6-4b26-bc4c-d5b0949] Unexpected error occurred serving API: Timed out waiting for a reply to message ID 7daf49ca17cd4070bcfa1041890f84ae 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi Traceback (most recent call last): 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/heat/common/wsgi.py", line 650, in __call__ 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi request, *action_args) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/heat/common/wsgi.py", line 724, in dispatch 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi return method(args, *kwargs) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/heat/api/openstack/v1/util.py", line 38, in handle_stack_method 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi return handler(controller, req, *kwargs) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/heat/api/openstack/v1/util.py", line 54, in handle_stack_method 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi return handler(controller, req, dict(stack_identity), *kwargs) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/heat/api/openstack/v1/resources.py", line 119, in metadata 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi resource_name) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/heat/rpc/client.py", line 351, in describe_stack_resource2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi version='1.2') 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/heat/rpc/client.py", line 51, in call 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi return client.call(ctxt, method, *kwargs) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/client.py", line 156, in call 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi retry=self.retry) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/oslo_messaging/transport.py", line 90, in _send 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi timeout=timeout, retry=retry) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 350, in send 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi retry=retry) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 339, in _send 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi result = self._waiter.wait(msg_id, timeout) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 243, in wait 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi message = self.waiters.get(msg_id, timeout=timeout) 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 149, in get 2016-06-22 14:14:34.026 15418 TRACE heat.common.wsgi 'to message ID %s' % msg_id)

#

Then we go to the rabbitmq log and found that there are some message like this ---->

#

=ERROR REPORT==== 22-Jun-2016::09:47:07 === Channel error on connection <0.5707.62> (146.89.148.249:38333 -> 146.89.148.249:5671, vhost: '/', user: 'rabbitclient'), channel 2: {amqp_error,not_found, "no exchange 'reply_13459769fa4c4c118bffe6b51f4ae12e' in vhost '/'", 'exchange.declare'}

=ERROR REPORT==== 22-Jun-2016::09:47:07 === Channel error on connection <0.4176.62> (146.89.148.249:38317 -> 146.89.148.249:5671, vhost: '/', user: 'rabbitclient'), channel 2: {amqp_error,not_found, "no exchange 'reply_cc995e2739c8403abf700ed2938570ba' in vhost '/'", 'exchange.declare'}

=ERROR REPORT==== 22-Jun-2016::09:47:07 === Channel error on connection <0.5233.62> (146.89.148.249:38175 -> 146.89.148.249:5671, vhost: '/', user: 'rabbitclient'), channel 2: {amqp_error,not_found, "no exchange 'reply_cc995e2739c8403abf700ed2938570ba' in vhost '/'", 'exchange.declare'}

=ERROR REPORT==== 22-Jun-2016::09:47:07 === Channel error on connection <0.5583.62> (146.89.148.249:38311 -> 146.89.148.249:5671, vhost: '/', user: 'rabbitclient'), channel 2: {amqp_error,not_found, "no exchange 'reply_13459769fa4c4c118bffe6b51f4ae12e' in vhost '/'", 'exchange.declare'}

=ERROR REPORT==== 22-Jun-2016::09:47:07 === Channel error on connection <0.5200.62> (146.89.148.249:38172 -> 146.89.148.249:5671, vhost: '/', user: 'rabbitclient'), channel 2: {amqp_error,not_found, "no exchange 'reply_cc995e2739c8403abf700ed2938570ba' in vhost '/'", 'exchange.declare'}

=ERROR REPORT==== 22-Jun-2016::09:47:07 === Channel error on connection <0.5737.62> (146.89.148.249:38337 -> 146.89.148.249:5671, vhost: '/', user: 'rabbitclient'), channel 2: {amqp_error,not_found, "no exchange 'reply_13459769fa4c4c118bffe6b51f4ae12e' in vhost '/'", 'exchange.declare'}

=ERROR REPORT==== 22-Jun-2016::09:47:07 === Channel error on connection <0.5551.62> (146.89.148.249:38308 -> 146.89.148.249:5671, vhost: '/', user: 'rabbitclient'), channel 2: {amqp_error,not_found, "no exchange 'reply_cc995e2739c8403abf700ed2938570ba' in vhost '/'", 'exchange.declare'}

#