Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

The state of instances are active but cluster state goes in error state.

The savanna log is as follows as:

127.0.0.1 - - [08/Aug/2013 20:18:30] "GET /v1.0/fd6e0af3983444bbaa41124740f373d9/clusters/1935935b-b8a2-4525-b2ed-000baf723c22 HTTP/1.1" 200 1790 0.012766 (23643) accepted ('127.0.0.1', 59464) 127.0.0.1 - - [08/Aug/2013 20:19:01] "GET /v1.0/fd6e0af3983444bbaa41124740f373d9/clusters/1935935b-b8a2-4525-b2ed-000baf723c22 HTTP/1.1" 200 1790 0.040173 (23643) accepted ('127.0.0.1', 59781) 127.0.0.1 - - [08/Aug/2013 20:19:31] "GET /v1.0/fd6e0af3983444bbaa41124740f373d9/clusters/1935935b-b8a2-4525-b2ed-000baf723c22 HTTP/1.1" 200 1790 0.034558 (23643) accepted ('127.0.0.1', 60114) 127.0.0.1 - - [08/Aug/2013 20:20:02] "GET /v1.0/fd6e0af3983444bbaa41124740f373d9/clusters/1935935b-b8a2-4525-b2ed-000baf723c22 HTTP/1.1" 200 1790 0.018801 (23643) accepted ('127.0.0.1', 60432) 127.0.0.1 - - [08/Aug/2013 20:20:32] "GET /v1.0/fd6e0af3983444bbaa41124740f373d9/clusters/1935935b-b8a2-4525-b2ed-000baf723c22 HTTP/1.1" 200 1790 0.025333 (23643) accepted ('127.0.0.1', 60756) 127.0.0.1 - - [08/Aug/2013 20:21:02] "GET /v1.0/fd6e0af3983444bbaa41124740f373d9/clusters/1935935b-b8a2-4525-b2ed-000baf723c22 HTTP/1.1" 200 1790 0.015351 (23643) accepted ('127.0.0.1', 32849) 127.0.0.1 - - [08/Aug/2013 20:21:33] "GET /v1.0/fd6e0af3983444bbaa41124740f373d9/clusters/1935935b-b8a2-4525-b2ed-000baf723c22 HTTP/1.1" 200 1790 0.018000 (23643) accepted ('127.0.0.1', 33179) 127.0.0.1 - - [08/Aug/2013 20:22:04] "GET /v1.0/fd6e0af3983444bbaa41124740f373d9/clusters/1935935b-b8a2-4525-b2ed-000baf723c22 HTTP/1.1" 200 1790 0.015437 (23643) accepted ('127.0.0.1', 33501) Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/eventlet/greenpool.py", line 80, in _spawn_n_impl func(args, *kwargs) File "/usr/local/lib/python2.7/dist-packages/eventlet/wsgi.py", line 584, in process_request proto.__init__(socket, address, self) File "/usr/lib/python2.7/SocketServer.py", line 638, in __init__ self.handle() File "/usr/lib/python2.7/BaseHTTPServer.py", line 340, in handle self.handle_one_request() File "/usr/local/lib/python2.7/dist-packages/eventlet/wsgi.py", line 226, in handle_one_request self.raw_requestline = self.rfile.readline(self.server.url_length_limit) File "/usr/lib/python2.7/socket.py", line 476, in readline data = self._sock.recv(self._rbufsize) File "/usr/local/lib/python2.7/dist-packages/eventlet/greenio.py", line 262, in recv timeout_exc=socket.timeout("timed out")) File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/__init__.py", line 151, in trampoline listener = hub.add(hub.READ, fileno, current.switch) File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/epolls.py", line 48, in add listener = BaseHub.add(self, evtype, fileno, cb) File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 126, in add evtype, fileno, evtype)) RuntimeError: Second simultaneous read on fileno 23 detected. Unless you really know what you're doing, make sure that only one greenthread can read any particular socket. Consider using a pools.Pool. If you do know what you're doing and want to disable this error, call eventlet.debug.hub_prevent_multiple_readers(False) 2013-08-09 09:51:23.123 23643 WARNING savanna.service.instances [-] Can't start cluster 'aai-hadoop' (reason: Unauthorized (HTTP 401)) 2013-08-09 09:51:23.548 23643 ERROR root [-] Original exception being dropped: ['Traceback (most recent call last):\n', ' File "/usr/local/lib/python2.7/dist-packages/savanna/service/instances.py", line 39, in create_cluster\n _await_instances(cluster)\n', ' File "/usr/local/lib/python2.7/dist-packages/savanna/service/instances.py", line 208, in _await_instances\n if not _check_if_up(instance):\n', ' File "/usr/local/lib/python2.7/dist-packages/savanna/service/instances.py", line 217, in _check_if_up\n server = nova.get_instance_info(instance)\n', ' File "/usr/local/lib/python2.7/dist-packages/savanna/utils/openstack/nova.py", line 69, in get_instance_info\n return client().servers.get(instance.instance_id)\n', ' File "/usr/local/lib/python2.7/dist-packages/novaclient/v1_1/servers.py", line 365, in get\n return self._get("/servers/%s" % base.getid(server), "server")\n', ' File "/usr/local/lib/python2.7/dist-packages/novaclient/base.py", line 139, in _get\n _resp, body = self.api.client.get(url)\n', ' File "/usr/local/lib/python2.7/dist-packages/novaclient/client.py", line 229, in get\n return self._cs_request(url, \'GET\', *kwargs)\n', ' File "/usr/local/lib/python2.7/dist-packages/novaclient/client.py", line 226, in _cs_request\n raise e\n', 'Unauthorized: Unauthorized (HTTP 401)\n'] Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/poll.py", line 97, in wait readers.get(fileno, noop).cb(fileno) File "/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 194, in main result = function(args, *kwargs) File "/usr/local/lib/python2.7/dist-packages/savanna/context.py", line 132, in wrapper func(args, *kwargs) File "/usr/local/lib/python2.7/dist-packages/savanna/service/api.py", line 111, in _provision_cluster i.create_cluster(cluster) File "/usr/local/lib/python2.7/dist-packages/savanna/service/instances.py", line 52, in create_cluster _rollback_cluster_creation(cluster, ex) File "/usr/local/lib/python2.7/dist-packages/savanna/service/instances.py", line 277, in _rollback_cluster_creation _shutdown_instances(cluster, True) File "/usr/local/lib/python2.7/dist-packages/savanna/service/instances.py", line 306, in _shutdown_instances _shutdown_instance(instance) File "/usr/local/lib/python2.7/dist-packages/savanna/service/instances.py", line 312, in _shutdown_instance nova.client().servers.delete(instance.instance_id) File "/usr/local/lib/python2.7/dist-packages/novaclient/v1_1/servers.py", line 668, in delete self._delete("/servers/%s" % base.getid(server)) File "/usr/local/lib/python2.7/dist-packages/novaclient/base.py", line 153, in _delete _resp, _body = self.api.client.delete(url) File "/usr/local/lib/python2.7/dist-packages/novaclient/client.py", line 238, in delete return self._cs_request(url, 'DELETE', *kwargs) File "/usr/local/lib/python2.7/dist-packages/novaclient/client.py", line 226, in _cs_request raise e Unauthorized: Unauthorized (HTTP 401) Removing descriptor: 16 (23643) accepted ('127.0.0.1', 39164) 127.0.0.1 - - [12/Aug/2013 09:48:35] "GET /v1.0/fd6e0af3983444bbaa41124740f373d9/clusters HTTP/1.1" 200 1962 0.284868