Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

Instance launch fails with nova/oslo errors

Since a few days ago every attempt to launch an instance fails on my devstack VM (same attempts worked fine before). I sync my local with repository updates regularly.

Here are the errors I see in Nova sceduler logs:

2014-02-11 11:58:28.076 ERROR nova.scheduler.filter_scheduler [req-71ab0a13-ead3-4e60-90c4-322a0fb96646 admin demo] [instance: 290ca314-c2c6-42e4-b8ab-0bda2bfa0780] Error from last host: devstackready02 (node devstackready02): [u'Traceback (most recent call last):\n', u'  File "/opt/stack/nova/nova/compute/manager.py", line 1072, in _build_instance\n    set_access_ip=set_access_ip)\n', u'  File "/opt/stack/nova/nova/compute/manager.py", line 357, in decorated_function\n    return function(self, context, *args, **kwargs)\n', u'  File "/opt/stack/nova/nova/compute/manager.py", line 1480, in _spawn\n    LOG.exception(_(\'Instance failed to spawn\'), instance=instance)\n', u'  File "/opt/stack/nova/nova/openstack/common/excutils.py", line 68, in __exit__\n    six.reraise(self.type_, self.value, self.tb)\n', u'  File "/opt/stack/nova/nova/compute/manager.py", line 1477, in _spawn\n    block_device_info)\n', u'  File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 2205, in spawn\n    admin_pass=admin_password)\n', u'  File "/opt/stack/nova/nova/virt/libvirt/driver.py", line 2570, in _create_image\n    content=files, extra_md=extra_md, network_info=network_info)\n', u'  File "/opt/stack/nova/nova/api/metadata/base.py", line 163, in __init__\n    ec2utils.get_ip_info_for_instance_from_nw_info(network_info)\n', u'  File "/opt/stack/nova/nova/api/ec2/ec2utils.py", line 147, in get_ip_info_for_instance_from_nw_info\n    fixed_ips = nw_info.fixed_ips()\n', u'  File "/opt/stack/nova/nova/network/model.py", line 366, in _sync_wrapper\n    self.wait()\n', u'  File "/opt/stack/nova/nova/network/model.py", line 398, in wait\n    self[:] = self._gt.wait()\n', u'  File "/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 168, in wait\n    return self._exit_event.wait()\n', u'  File "/usr/local/lib/python2.7/dist-packages/eventlet/event.py", line 120, in wait\n    current.throw(*self._exc)\n', u'  File "/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py", line 194, in main\n    result = function(*args, **kwargs)\n', u'  File "/opt/stack/nova/nova/compute/manager.py", line 1258, in _allocate_network_async\n    dhcp_options=dhcp_options)\n', u'  File "/opt/stack/nova/nova/network/api.py", line 94, in wrapped\n    return func(self, context, *args, **kwargs)\n', u'  File "/opt/stack/nova/nova/network/api.py", line 48, in wrapper\n    res = f(self, context, *args, **kwargs)\n', u'  File "/opt/stack/nova/nova/network/api.py", line 302, in allocate_for_instance\n    nw_info = self.network_rpcapi.allocate_for_instance(context, **args)\n', u'  File "/opt/stack/nova/nova/network/rpcapi.py", line 167, in allocate_for_instance\n    macs=jsonutils.to_primitive(macs))\n', u'  File "/opt/stack/oslo.messaging/oslo/messaging/rpc/client.py", line 150, in call\n    wait_for_reply=True, timeout=timeout)\n', u'  File "/opt/stack/oslo.messaging/oslo/messaging/transport.py", line 87, in _send\n    timeout=timeout)\n', u'  File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/amqpdriver.py", line 390, in send\n    return self._send(target, ctxt, message, wait_for_reply, timeout)\n', u'  File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/amqpdriver.py", line 383, in _send\n    raise result\n', u'OSError: [Errno 2] No such file or directory\n']

2014-02-11 11:58:28.139 ERROR oslo.messaging.notify._impl_messaging [-] Could not send notification to notifications. Payload={'_context_roles': [u'admin'], '_context_request_id': u'req-71ab0a13-ead3-4e60-90c4-322a0fb96646', '_context_quota_class': None, 'event_type': 'scheduler.run_instance', '_context_service_catalog': [{u'endpoints': [{u'adminURL': u'http://9.30.182.117:8776/v1/d3f943c834bc4a7fbddbfcd3a235f6b2', u'region': u'RegionOne', u'id': u'2431f016fbf14e2b94e938e3c9c6d00a', u'internalURL': u'http://9.30.182.117:8776/v1/d3f943c834bc4a7fbddbfcd3a235f6b2', u'publicURL': u'http://9.30.182.117:8776/v1/d3f943c834bc4a7fbddbfcd3a235f6b2'}], u'endpoints_links': [], u'type': u'volume', u'name': u'cinder'}], 'timestamp': '2014-02-11 19:58:28.137398', '_context_user': u'4b495863369c49338026498407486e0c', '_unique_id': 'b7be121ea7d94a74b25c0a4495382488', '_context_instance_lock_checked': False, '_context_user_id': u'4b495863369c49338026498407486e0c', 'payload': {'instance_id': u'290ca314-c2c6-42e4-b8ab-0bda2bfa0780', 'state': 'error', 'request_spec': {u'num_instances': 1, u'block_device_mapping': [{u'instance_uuid': u'290ca314-c2c6-42e4-b8ab-0bda2bfa0780', u'guest_format': None, u'boot_index': 0, u'no_device': None, u'connection_info': None, u'image_id': u'fc999aa5-3419-4451-88dc-3942803ab357', u'snapshot_id': None, u'device_name': None, u'disk_bus': None, u'volume_size': None, u'source_type': u'image', u'device_type': u'disk', u'volume_id': None, u'destination_type': u'local', u'delete_on_termination': True}], u'image': {u'status': u'active', u'name': u'precise-server-cloudimg-amd64-disk1', u'deleted': False, u'container_format': u'bare', u'created_at': u'2014-02-11T19:21:24.000000', u'disk_format': u'qcow2', u'updated_at': u'2014-02-11T19:21:26.000000', u'id': u'fc999aa5-3419-4451-88dc-3942803ab357', u'owner': u'91f41068662c413988d8279e5a8f09ec', u'min_ram': 0, u'checksum': u'6c9dc6d8c3fd2f5e3f42fd8e7ea182b4', u'min_disk': 0, u'is_public': True, u'deleted_at': None, u'properties': {}, u'size': 255393792}, u'instance_type': {u'root_gb': 40, u'name': u'm1.medium', u'ephemeral_gb': 0, u'memory_mb': 4096, u'vcpus': 2, u'extra_specs': {}, u'swap': 0, u'rxtx_factor': 1.0, u'flavorid': u'3', u'vcpu_weight': None, u'id': 1}, u'instance_properties': {u'vm_state': u'building', u'availability_zone': u'nova', u'terminated_at': None, u'ephemeral_gb': 0, u'instance_type_id': 1, u'user_data': None, u'cleaned': False, u'vm_mode': None, u'deleted_at': None, u'reservation_id': u'r-4tm7602l', u'id': 3, u'security_groups': [{u'project_id': u'd3f943c834bc4a7fbddbfcd3a235f6b2', u'user_id': u'4b495863369c49338026498407486e0c', u'description': u'default', u'deleted': False, u'created_at': u'2014-02-11T19:22:28.000000', u'updated_at': None, u'deleted_at': None, u'id': 1, u'name': u'default'}], u'disable_terminate': False, u'display_name': u'ins2', u'uuid': u'290ca314-c2c6-42e4-b8ab-0bda2bfa0780', u'default_swap_device': None, u'info_cache': {u'instance_uuid': u'290ca314-c2c6-42e4-b8ab-0bda2bfa0780', u'deleted': False, u'created_at': u'2014-02-11T19:58:11.000000', u'updated_at': None, u'network_info': [], u'deleted_at': None}, u'hostname': u'ins2', u'launched_on': None, u'display_description': u'ins2', u'key_data': None, u'kernel_id': u'', u'power_state': 0, u'default_ephemeral_device': None, u'progress': 0, u'project_id': u'd3f943c834bc4a7fbddbfcd3a235f6b2', u'launched_at': None, u'config_drive': u'', u'node': None, u'ramdisk_id': u'', u'access_ip_v6': None, u'access_ip_v4': None, u'deleted': False, u'key_name': None, u'updated_at': None, u'host': None, u'ephemeral_key_uuid': None, u'architecture': None, u'user_id': u'4b495863369c49338026498407486e0c', u'system_metadata': {u'image_min_disk': u'40', u'instance_type_memory_mb': u'4096', u'instance_type_swap': u'0', u'instance_type_vcpu_weight': None, u'instance_type_root_gb': u'40', u'instance_type_id': u'1', u'instance_type_name': u'm1.medium', u'instance_type_ephemeral_gb': u'0', u'instance_type_rxtx_factor': u'1.0', u'instance_type_flavorid': u'3', u'image_container_format': u'bare', u'instance_type_vcpus': u'2', u'image_min_ram': u'0', u'image_disk_format': u'qcow2', u'image_base_image_ref': u'fc999aa5-3419-4451-88dc-3942803ab357'}, u'task_state': u'scheduling', u'shutdown_terminate': False, u'cell_name': None, u'root_gb': 40, u'locked': False, u'name': u'instance-00000003', u'created_at': u'2014-02-11T19:58:11.000000', u'locked_by': None, u'launch_index': 0, u'metadata': {}, u'memory_mb': 4096, u'vcpus': 2, u'image_ref': u'fc999aa5-3419-4451-88dc-3942803ab357', u'root_device_name': None, u'auto_disk_config': False, u'os_type': None, u'scheduled_at': None}, u'security_group': [u'default']}, 'instance_properties': {u'vm_state': u'building', u'availability_zone': u'nova', u'terminated_at': None, u'ephemeral_gb': 0, u'instance_type_id': 1, u'user_data': None, u'cleaned': False, u'vm_mode': None, u'deleted_at': None, u'reservation_id': u'r-4tm7602l', u'id': 3, u'security_groups': [{u'project_id': u'd3f943c834bc4a7fbddbfcd3a235f6b2', u'user_id': u'4b495863369c49338026498407486e0c', u'description': u'default', u'deleted': False, u'created_at': u'2014-02-11T19:22:28.000000', u'updated_at': None, u'deleted_at': None, u'id': 1, u'name': u'default'}], u'disable_terminate': False, u'display_name': u'ins2', u'uuid': u'290ca314-c2c6-42e4-b8ab-0bda2bfa0780', u'default_swap_device': None, u'info_cache': {u'instance_uuid': u'290ca314-c2c6-42e4-b8ab-0bda2bfa0780', u'deleted': False, u'created_at': u'2014-02-11T19:58:11.000000', u'updated_at': None, u'network_info': [], u'deleted_at': None}, u'hostname': u'ins2', u'launched_on': None, u'display_description': u'ins2', u'key_data': None, u'kernel_id': u'', u'power_state': 0, u'default_ephemeral_device': None, u'progress': 0, u'project_id': u'd3f943c834bc4a7fbddbfcd3a235f6b2', u'launched_at': None, u'config_drive': u'', u'node': None, u'ramdisk_id': u'', u'access_ip_v6': None, u'access_ip_v4': None, u'deleted': False, u'key_name': None, u'updated_at': None, u'host': None, u'ephemeral_key_uuid': None, u'architecture': None, u'user_id': u'4b495863369c49338026498407486e0c', u'system_metadata': {u'image_min_disk': u'40', u'instance_type_memory_mb': u'4096', u'instance_type_swap': u'0', u'instance_type_vcpu_weight': None, u'instance_type_root_gb': u'40', u'instance_type_id': u'1', u'instance_type_name': u'm1.medium', u'instance_type_ephemeral_gb': u'0', u'instance_type_rxtx_factor': u'1.0', u'instance_type_flavorid': u'3', u'image_container_format': u'bare', u'instance_type_vcpus': u'2', u'image_min_ram': u'0', u'image_disk_format': u'qcow2', u'image_base_image_ref': u'fc999aa5-3419-4451-88dc-3942803ab357'}, u'task_state': u'scheduling', u'shutdown_terminate': False, u'cell_name': None, u'root_gb': 40, u'locked': False, u'name': u'instance-00000003', u'created_at': u'2014-02-11T19:58:11.000000', u'locked_by': None, u'launch_index': 0, u'metadata': {}, u'memory_mb': 4096, u'vcpus': 2, u'image_ref': u'fc999aa5-3419-4451-88dc-3942803ab357', u'root_device_name': None, u'auto_disk_config': False, u'os_type': None, u'scheduled_at': None}, 'reason': NoValidHost(u'No valid host was found. ',), 'method': 'run_instance'}, '_context_project_name': u'demo', '_context_read_deleted': u'no', '_context_auth_token': u'e0642e0499024f74b21a4aa809ce0aa4', '_context_tenant': u'd3f943c834bc4a7fbddbfcd3a235f6b2', 'priority': 'ERROR', '_context_is_admin': True, '_context_project_id': u'd3f943c834bc4a7fbddbfcd3a235f6b2', '_context_timestamp': '2014-02-11T19:58:11.771496', '_context_user_name': u'admin', 'publisher_id': 'scheduler.devstackready02', 'message_id': '58de4622-941c-49ad-ae0e-6b5ffe11fcae', '_context_remote_address': u'9.30.182.117'}
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging Traceback (most recent call last):
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/oslo.messaging/oslo/messaging/notify/_impl_messaging.py", line 47, in notify
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     version=self.version)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/oslo.messaging/oslo/messaging/transport.py", line 93, in _send_notification
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     self._driver.send_notification(target, ctxt, message, version)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/amqpdriver.py", line 393, in send_notification
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     return self._send(target, ctxt, message, envelope=(version == 2.0))
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/amqpdriver.py", line 378, in _send
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     conn.topic_send(topic, msg, timeout=timeout)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 701, in topic_send
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     self.publisher_send(TopicPublisher, topic, msg, timeout)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 673, in publisher_send
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     self.ensure(_error_callback, _publish)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 577, in ensure
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     return method(*args, **kwargs)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 671, in _publish
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     publisher.send(msg, timeout)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 328, in send
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     self.producer.publish(msg)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/usr/local/lib/python2.7/dist-packages/kombu/messaging.py", line 159, in publish
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     compression, headers)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/usr/local/lib/python2.7/dist-packages/kombu/messaging.py", line 235, in _prepare
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     body) = dumps(body, serializer=serializer)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/usr/local/lib/python2.7/dist-packages/kombu/serialization.py", line 147, in dumps
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     payload = encoder(data)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/usr/local/lib/python2.7/dist-packages/anyjson/__init__.py", line 141, in dumps
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     return implementation.dumps(value)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/usr/local/lib/python2.7/dist-packages/anyjson/__init__.py", line 87, in dumps
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     return self._encode(data)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/opt/stack/python-neutronclient/neutronclient/openstack/common/jsonutils.py", line 155, in dumps
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     return json.dumps(value, default=default, **kwargs)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/usr/lib/python2.7/json/__init__.py", line 238, in dumps
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     **kw).encode(obj)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/usr/lib/python2.7/json/encoder.py", line 201, in encode
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     chunks = self.iterencode(o, _one_shot=True)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging   File "/usr/lib/python2.7/json/encoder.py", line 264, in iterencode
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging     return _iterencode(o, 0)
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging ValueError: Circular reference detected
2014-02-11 11:58:28.139 TRACE oslo.messaging.notify._impl_messaging

And this is my localrc content:

FLOATING_RANGE=127.0.1.224/27
FIXED_RANGE=10.34.102.0/24
FIXED_NETWORK_SIZE=256
FLAT_INTERFACE=eth0
ADMIN_PASSWORD=****
MYSQL_PASSWORD=****
RABBIT_PASSWORD=****
SERVICE_PASSWORD=****
# Enable Logging
LOGFILE=$DEST/logs/stack.sh.log
VERBOSE=True
LOG_COLOR=True
SCREEN_LOGDIR=$DEST/logs/screen
LOGDAYS=1
## For Keystone
KEYSTONE_TOKEN_FORMAT=UUID
# Horizon (Dashboard UI) - (always use the trunk)
ENABLED_SERVICES+=,horizon
HORIZON_REPO=https://github.com/openstack/horizon
HORIZON_BRANCH=master
# Cinder - Block Device Service
ENABLED_SERVICES+=,cinder,c-api,c-vol,c-sch
# Heat - Orchestration Service
#ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng
IMAGE_URLS+=",http://fedorapeople.org/groups/heat/prebuilt-jeos-images/F17-x86_64-cfntools.qcow2"
IMAGE_URLS+=",http://download.cirros-cloud.net/0.3.1/cirros-0.3.1-x86_64-disk.img"
IMAGE_URLS+=",http://uec-images.ubuntu.com/precise/current/precise-server-cloudimg-amd64-disk1.img"
# Ceilometer - Metering Service (metering + alarming)
ENABLED_SERVICES+=,ceilometer-acompute,ceilometer-acentral,ceilometer-collector,ceilometer-api
ENABLED_SERVICES+=,ceilometer-alarm-notify,ceilometer-alarm-eval
# Apache fronted for WSGI
APACHE_ENABLED_SERVICES+=keystone,swift
#EXTRA_OPTS=(scheduler_default_filters=AllHostsFilter)
SERVICE_TOKEN=****

I was able to launch instances with the same config before.

And just in case, I already have gotten the patch for https://bugs.launchpad.net/nova/+bug/1275771.

Any help is very much appreciated.

Thanks.