Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

error instance get stuck in networking

so /var/log/nova/nova-network.log 2012-08-08 11:16:43 ERROR nova.rpc.common [-] Failed to consume message from queue: Socket closed 2012-08-08 11:16:43 TRACE nova.rpc.common Traceback (most recent call last): 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 490, in ensure 2012-08-08 11:16:43 TRACE nova.rpc.common return method(args, kwargs) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 567, in _consume 2012-08-08 11:16:43 TRACE nova.rpc.common return self.connection.drain_events(timeout=timeout) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 175, in drain_events 2012-08-08 11:16:43 TRACE nova.rpc.common return self.transport.drain_events(self.connection, *kwargs) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 238, in drain_events 2012-08-08 11:16:43 TRACE nova.rpc.common return connection.drain_events(kwargs) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 57, in drain_events 2012-08-08 11:16:43 TRACE nova.rpc.common return self.wait_multi(self.channels.values(), timeout=timeout) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 63, in wait_multi 2012-08-08 11:16:43 TRACE nova.rpc.common chanmap.keys(), allowed_methods, timeout=timeout) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 120, in _wait_multiple 2012-08-08 11:16:43 TRACE nova.rpc.common channel, method_sig, args, content = read_timeout(timeout) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 88, in read_timeout 2012-08-08 11:16:43 TRACE nova.rpc.common return self.method_reader.read_method() 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/amqplib/client_0_8/method_framing.py", line 221, in read_method 2012-08-08 11:16:43 TRACE nova.rpc.common raise m 2012-08-08 11:16:43 TRACE nova.rpc.common IOError: Socket closed 2012-08-08 11:16:43 TRACE nova.rpc.common 2012-08-08 11:16:43 INFO nova.rpc.common [-] Reconnecting to AMQP server on 172.18.32.7:5672 2012-08-08 11:16:43 ERROR nova.rpc.common [-] AMQP server on 172.18.32.7:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2012-08-08 11:16:43 TRACE nova.rpc.common Traceback (most recent call last): 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 446, in reconnect 2012-08-08 11:16:43 TRACE nova.rpc.common self._connect() 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 423, in _connect 2012-08-08 11:16:43 TRACE nova.rpc.common self.connection.connect() 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 154, in connect 2012-08-08 11:16:43 TRACE nova.rpc.common return self.connection 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 560, in connection 2012-08-08 11:16:43 TRACE nova.rpc.common self._connection = self._establish_connection() 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 521, in _establish_connection 2012-08-08 11:16:43 TRACE nova.rpc.common conn = self.transport.establish_connection() 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 255, in establish_connection 2012-08-08 11:16:43 TRACE nova.rpc.common connect_timeout=conninfo.connect_timeout) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 52, in __init__ 2012-08-08 11:16:43 TRACE nova.rpc.common super(Connection, self).__init__(args, kwargs) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/amqplib/client_0_8/connection.py", line 129, in __init__ 2012-08-08 11:16:43 TRACE nova.rpc.common self.transport = create_transport(host, connect_timeout, ssl) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/amqplib/client_0_8/transport.py", line 281, in create_transport 2012-08-08 11:16:43 TRACE nova.rpc.common return TCPTransport(host, connect_timeout) 2012-08-08 11:16:43 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/amqplib/client_0_8/transport.py", line 85, in __init__ 2012-08-08 11:16:43 TRACE nova.rpc.common raise socket.error, msg 2012-08-08 11:16:43 TRACE nova.rpc.common error: [Errno 111] ECONNREFUSED 2012-08-08 11:16:43 TRACE nova.rpc.common 2012-08-08 11:16:46 AUDIT nova.service [-] Starting network node (version 2012.1-LOCALBRANCH:LOCALREVISION) 2012-08-08 11:16:48 INFO nova.rpc.common [-] Connected to AMQP server on 172.18.32.7:5672 2012-08-08 11:17:25 INFO nova.rpc.common [req-4a044830-2d44-4d24-9322-4e8211545c3c None None] Connected to AMQP server on 172.18.32.7:5672 2012-08-08 11:20:37 ERROR nova.rpc.common [-] Failed to consume message from queue: Socket closed 2012-08-08 11:20:37 TRACE nova.rpc.common Traceback (most recent call last): :$
2012-08-08 13:22:58 TRACE nova.rpc.common return self.method_reader.read_method() 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/amqplib/client_0_8/method_framing.py", line 221, in read_method 2012-08-08 13:22:58 TRACE nova.rpc.common raise m 2012-08-08 13:22:58 TRACE nova.rpc.common IOError: Socket closed 2012-08-08 13:22:58 TRACE nova.rpc.common 2012-08-08 13:22:58 INFO nova.rpc.common [-] Reconnecting to AMQP server on 172.18.32.7:5672 2012-08-08 13:22:58 ERROR nova.rpc.common [-] AMQP server on 172.18.32.7:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 1 seconds. 2012-08-08 13:22:58 TRACE nova.rpc.common Traceback (most recent call last): 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 446, in reconnect 2012-08-08 13:22:58 TRACE nova.rpc.common self._connect() 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 423, in _connect 2012-08-08 13:22:58 TRACE nova.rpc.common self.connection.connect() 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 154, in connect 2012-08-08 13:22:58 TRACE nova.rpc.common return self.connection 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 560, in connection 2012-08-08 13:22:58 TRACE nova.rpc.common self._connection = self._establish_connection() 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 521, in _establish_connection 2012-08-08 13:22:58 TRACE nova.rpc.common conn = self.transport.establish_connection() 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 255, in establish_connection 2012-08-08 13:22:58 TRACE nova.rpc.common connect_timeout=conninfo.connect_timeout) 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 52, in __init__ 2012-08-08 13:22:58 TRACE nova.rpc.common super(Connection, self).__init__(
args, *kwargs) 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/amqplib/client_0_8/connection.py", line 129, in __init__ 2012-08-08 13:22:58 TRACE nova.rpc.common self.transport = create_transport(host, connect_timeout, ssl) 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/amqplib/client_0_8/transport.py", line 281, in create_transport 2012-08-08 13:22:58 TRACE nova.rpc.common return TCPTransport(host, connect_timeout) 2012-08-08 13:22:58 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/amqplib/client_0_8/transport.py", line 85, in __init__ 2012-08-08 13:22:58 TRACE nova.rpc.common raise socket.error, msg 2012-08-08 13:22:58 TRACE nova.rpc.common error: [Errno 111] ECONNREFUSED 2012-08-08 13:22:58 TRACE nova.rpc.common 2012-08-08 13:23:01 AUDIT nova.service [-] Starting network node (version 2012.1-LOCALBRANCH:LOCALREVISION) 2012-08-08 13:23:03 INFO nova.rpc.common [-] Connected to AMQP server on 172.18.32.7:5672 2012-08-08 13:23:04 INFO nova.rpc.common [req-549a4990-f9bd-4210-ad54-66bc02301369 None None] Connected to AMQP server on 172.18.32.7:5672 2012-08-08 13:24:43 INFO nova.rpc.common [req-c258f321-a53c-4e95-a33f-b8fefa56f567 None None] Connected to AMQP server on 172.18.32.7:5672 2012-08-08 13:24:56 ERROR nova.rpc.common [-] Timed out waiting for RPC response: timed out 2012-08-08 13:24:56 TRACE nova.rpc.common Traceback (most recent call last): 2012-08-08 13:24:56 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 490, in ensure 2012-08-08 13:24:56 TRACE nova.rpc.common return method(args, *kwargs) 2012-08-08 13:24:56 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/nova/rpc/impl_kombu.py", line 567, in _consume 2012-08-08 13:24:56 TRACE nova.rpc.common return self.connection.drain_events(timeout=timeout) 2012-08-08 13:24:56 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 175, in drain_events 2012-08-08 13:24:56 TRACE nova.rpc.common return self.transport.drain_events(self.connection, *kwargs) 2012-08-08 13:24:56 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 238, in drain_events 2012-08-08 13:24:56 TRACE nova.rpc.common return connection.drain_events(kwargs) 2012-08-08 13:24:56 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 57, in drain_events 2012-08-08 13:24:56 TRACE nova.rpc.common return self.wait_multi(self.channels.values(), timeout=timeout) 2012-08-08 13:24:56 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 63, in wait_multi 2012-08-08 13:24:56 TRACE nova.rpc.common chanmap.keys(), allowed_methods, timeout=timeout) 2012-08-08 13:24:56 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 120, in _wait_multiple 2012-08-08 13:24:56 TRACE nova.rpc.common channel, method_sig, args, content = read_timeout(timeout) 2012-08-08 13:24:56 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/kombu/transport/pyamqplib.py", line 94, in read_timeout 2012-08-08 13:24:56 TRACE nova.rpc.common return self.method_reader.read_method() 2012-08-08 13:24:56 TRACE nova.rpc.common File "/usr/lib/python2.7/dist-packages/amqplib/client_0_8/method_framing.py", line 221, in read_method 2012-08-08 13:24:56 TRACE nova.rpc.common raise m 2012-08-08 13:24:56 TRACE nova.rpc.common timeout: timed out 2012-08-08 13:24:56 TRACE nova.rpc.common