Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

nova hypervisor-list shows down

we have a new setup deployed and I found nova-schedule reports no available hosts, and hypervisor-list shows down. But I could not figure out what might be the cause. I tried to restart nova-compute and the issue persist. I found error messages during the above test. I’m wondering why nova-compute appears down here?

nova-common                            2:12.0.1-0ubuntu1~cloud0.1contrail   46

nova hypervisor-list: down

root@controller:/var/log# nova --debug hypervisor-list
DEBUG (session:198) REQ: curl -g -i -X GET http://192.168.15.192:5000/v2.0/ -H "Accept: application/json" -H "User-Agent: python-keystoneclient"
INFO (connectionpool:205) Starting new HTTP connection (1): 192.168.15.192
DEBUG (connectionpool:385) "GET /v2.0/ HTTP/1.1" 200 340
DEBUG (session:215) RESP: [200] Content-Length: 340 Vary: X-Auth-Token X-Distribution: Ubuntu Connection: keep-alive Date: Fri, 09 Feb 2018 20:19:00 GMT Content-Type: application/json X-Openstack-Request-Id: req-f4226725-309c-48fe-afd2-4f56b1229e70
RESP BODY: {"version": {"status": "stable", "updated": "2014-04-17T00:00:00Z", "media-types": [{"base": "application/json", "type": "application/vnd.openstack.identity-v2.0+json"}], "id": "v2.0", "links": [{"href": "http://192.168.15.192:5000/v2.0/", "rel": "self"}, {"href": "http://docs.openstack.org/", "type": "text/html", "rel": "describedby"}]}}

DEBUG (v2:86) Making authentication request to http://192.168.15.192:5000/v2.0/tokens
DEBUG (connectionpool:385) "POST /v2.0/tokens HTTP/1.1" 200 3598
DEBUG (session:198) REQ: curl -g -i -X GET http://192.168.15.192:8774/v1.1/ -H "User-Agent: python-novaclient" -H "Accept: application/json" -H "X-Auth-Token: {SHA1}464b0577aef57014880c82e8f7c853c4123ae297"
INFO (connectionpool:205) Starting new HTTP connection (1): 192.168.15.192
DEBUG (connectionpool:385) "GET /v1.1/ HTTP/1.1" 200 381
2018-02-09 15:19:01.213 193781 INFO nova.osapi_compute.wsgi.server [req-3dfbe105-6ec1-4b0e-bca4-268b016aa3ba 95bf0fce1d884b7f93386b3ad850f29c c8160e007e7446168e968ee63bfa850e - - -] 192.168.15.192 "GET /v1.1/ HTTP/1.1" status: 200 len: 578 time: 0.2799881
DEBUG (session:215) RESP: [200] Date: Fri, 09 Feb 2018 20:19:01 GMT Connection: keep-alive Content-Type: application/json Content-Length: 381 X-Compute-Request-Id: req-3dfbe105-6ec1-4b0e-bca4-268b016aa3ba
RESP BODY: {"version": {"status": "SUPPORTED", "updated": "2011-01-21T11:33:21Z", "links": [{"href": "http://192.168.15.192:8774/v1.1/", "rel": "self"}, {"href": "http://docs.openstack.org/", "type": "text/html", "rel": "describedby"}], "min_version": "", "version": "", "media-types": [{"base": "application/json", "type": "application/vnd.openstack.compute+json;version=2"}], "id": "v2.0"}}

DEBUG (session:198) REQ: curl -g -i -X GET http://192.168.15.192:8774/v1.1/c8160e007e7446168e968ee63bfa850e/os-hypervisors -H "User-Agent: python-novaclient" -H "Accept: application/json" -H "X-Auth-Token: {SHA1}464b0577aef57014880c82e8f7c853c4123ae297"
DEBUG (connectionpool:385) "GET /v1.1/c8160e007e7446168e968ee63bfa850e/os-hypervisors HTTP/1.1" 200 185
2018-02-09 15:19:01.483 193781 INFO nova.osapi_compute.wsgi.server [req-84b5e03c-bc20-4717-8208-18fc6107f85e 95bf0fce1d884b7f93386b3ad850f29c c8160e007e7446168e968ee63bfa850e - - -] 192.168.15.192 "GET /v1.1/c8160e007e7446168e968ee63bfa850e/os-hypervisors HTTP/1.1" status: 200 len: 382 time: 0.0459030
DEBUG (session:215) RESP: [200] Date: Fri, 09 Feb 2018 20:19:01 GMT Connection: keep-alive Content-Type: application/json Content-Length: 185 X-Compute-Request-Id: req-84b5e03c-bc20-4717-8208-18fc6107f85e
RESP BODY: {"hypervisors": [{"status": "enabled", "state": "down", "id": 1, "hypervisor_hostname": "compute1"}, {"status": "enabled", "state": "down", "id": 2, "hypervisor_hostname": "compute2"}]}

+----+---------------------+-------+---------+
| ID | Hypervisor hostname | State | Status  |
+----+---------------------+-------+---------+
| 1  | compute1            | down  | enabled |
| 2  | compute2            | down  | enabled |
+----+---------------------+-------+---------+

errors when restarting nova-compute

==> nova-conductor.log <==
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters [req-4a19d7d1-a152-4600-a57c-9e29d679be8b - - - - -] DBAPIError exception wrapped from (pymysql.err.InternalError) (1241, u'Operand should contain 1 column(s)') [SQL: u'SELECT migrations.created_at AS migrations_created_at, migrations.updated_at AS migrations_updated_at, migrations.deleted_at AS migrations_deleted_at, migrations.deleted AS migrations_deleted, migrations.id AS migrations_id, migrations.source_compute AS migrations_source_compute, migrations.dest_compute AS migrations_dest_compute, migrations.source_node AS migrations_source_node, migrations.dest_node AS migrations_dest_node, migrations.dest_host AS migrations_dest_host, migrations.old_instance_type_id AS migrations_old_instance_type_id, migrations.new_instance_type_id AS migrations_new_instance_type_id, migrations.instance_uuid AS migrations_instance_uuid, migrations.status AS migrations_status, migrations.migration_type AS migrations_migration_type, migrations.hidden AS migra2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters [req-4a19d7d1-a152-4600-a57c-9e29d679be8b - - - - -] DBAPIError exception wrapped from (pymysql.err.InternalError) (1241, u'Operand should contain 1 column(s)') [SQL: u'SELECT migrations.created_at AS migrations_created_at, migrations.updated_at AS migrations_updated_at, migrations.deleted_at AS migrations_deleted_at, migrations.deleted AS migrations_deleted, migrations.id AS migrations_id, migrations.source_compute AS migrations_source_compute, migrations.dest_compute AS migrations_dest_compute, migrations.source_node AS migrations_source_node, migrations.dest_node AS migrations_dest_node, migrations.dest_host AS migrations_dest_host, migrations.old_instance_type_id AS migrations_old_instance_type_id, migrations.new_instance_type_id AS migrations_new_instance_type_id, migrations.instance_uuid AS migrations_instance_uuid, migrations.status AS migrations_status, migrations.migration_type AS migrations_migration_type, migrations.hidden AS migrations_hidden \nFROM migrations \nWHERE migrations.deleted = %s AND migrations.status = %s AND migrations.source_compute = %s AND migrations.migration_type = %s'] [parameters: (0, [u'accepted', u'done'], u'compute1', u'evacuation')]
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters Traceback (most recent call last):
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     context)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 450, in do_execute
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     cursor.execute(statement, parameters)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/cursors.py", line 132, in execute
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     result = self._query(query)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/cursors.py", line 271, in _query
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     conn.query(q)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 726, in query
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     self._affected_rows = self._read_query_result(unbuffered=unbuffered)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 861, in _read_query_result
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     result.read()
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 1064, in read
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     first_packet = self.connection._read_packet()
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 826, in _read_packet
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     packet.check_error()
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 370, in check_error
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     raise_mysql_exception(self._data)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/err.py", line 116, in raise_mysql_exception
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     _check_mysql_exception(errinfo)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/err.py", ltions_hidden \nFROM migrations \nWHERE migrations.deleted = %s AND migrations.status = %s AND migrations.source_compute = %s AND migrations.migration_type = %s'] [parameters: (0, [u'accepted', u'done'], u'compute1', u'evacuation')]
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters Traceback (most recent call last):
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     context)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 450, in do_execute
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     cursor.execute(statement, parameters)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/cursors.py", line 132, in execute
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     result = self._query(query)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/cursors.py", line 271, in _query
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     conn.query(q)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 726, in query
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     self._affected_rows = self._read_query_result(unbuffered=unbuffered)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 861, in _read_query_result
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     result.read()
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 1064, in read
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     first_packet = self.connection._read_packet()
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 826, in _read_packet
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     packet.check_error()
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/connections.py", line 370, in check_error
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     raise_mysql_exception(self._data)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dist-packages/pymysql/err.py", line 116, in raise_mysql_exception
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters     _check_mysql_exception(errinfo)
2018-02-13 13:42:59.429 193946 ERROR oslo_db.sqlalchemy.exc_filters   File "/usr/lib/python2.7/dis


root@controller: /log/nova# openstack-status | grep -v disabled 
== Nova services ==                                             
openstack-nova-api:           active                            
openstack-nova-scheduler:     active                            
openstack-nova-conductor:     active                            
== Glance services ==                                           
openstack-glance-api:         active                            
openstack-glance-registry:    active                            
== Keystone service ==                                          
openstack-keystone:           active                            
== Cinder services ==                                           
openstack-cinder-api:         active                            
openstack-cinder-scheduler:   active                            
== Heat services ==                                             
heat-api:                     active                            
heat-api-cfn:                 active                            
heat-engine:                  active                            
== Support services ==                                          
rabbitmq-server:              active                            
== Keystone users ==                                            
Warning keystonerc not sourced                                  


root@compute1:~# openstack-status | grep -v disable
dpkg-query: no packages found matching heat-common
== Nova services ==
openstack-nova-compute:       active
== Support services ==
libvirt-bin:                  active
Warning novarc not sourced