Ask Your Question

Revision history [back]

nova detach iscsi error

Hi guys,

i get a strange problem on all my compute hosts with volume attachments by Cinder. All volumes are attached with multipath iSCSI.

  • Ubuntu 12.04
  • Icehouse
  • KVM

how to reproduce: * Attach volume * mount volume and fill with files * umount * detach volume

root@comp8:~# multipath -ll
36006016013503c00a91d1364f910e511 dm-1 DGC,VRAID
size=3.1T features='1 queue_if_no_path' hwhandler='1 emc' wp=rw
|-+- policy='round-robin 0' prio=1 status=active
| `- 13:0:0:7 sdg 8:96 active ready  running
`-+- policy='round-robin 0' prio=0 status=enabled
  `- 12:0:0:7 sdf 8:80 active ready  running

Nova Compute log:

2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher   File
"/usr/lib/python2.7/dist-packages/nova/openstack/common/excutils.py",
line 68, in __exit__
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher    
six.reraise(self.type_, self.value, self.tb)
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher   File
"/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 307, in
decorated_function
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher    
return function(self, context, *args, **kwargs)
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher   File
"/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 4296,
in detach_volume
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher    
self._detach_volume(context, instance, bdm)
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher   File
"/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 4243,
in _detach_volume
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher    
connection_info = jsonutils.loads(bdm.connection_info)
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher   File
"/usr/lib/python2.7/dist-packages/nova/openstack/common/jsonutils.py",
line 164, in loads
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher    
return json.loads(s)
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher   File
"/usr/lib/python2.7/json/__init__.py", line 326, in loads
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher    
return _default_decoder.decode(s)
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher   File
"/usr/lib/python2.7/json/decoder.py", line 366, in decode
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher    
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher
TypeError: expected string or buffer
2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher
2015-06-26 08:17:01.999 13059 ERROR oslo.messaging._drivers.common [-]
Returning exception expected string or buffer to caller
2015-06-26 08:17:02.000 13059 ERROR oslo.messaging._drivers.common [-]
['Traceback (most recent call last):\n', '  File
"/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py",
line 133, in _dispatch_and_reply\n    incoming.message))\n', '  File
"/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py",
line 176, in _dispatch\n    return self._do_dispatch(endpoint, method,
ctxt, args)\n', '  File
"/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py",
line 122, in _do_dispatch\n    result = getattr(endpoint, method)(ctxt,
**new_args)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/exception.py", line 88, in
wrapped\n    payload)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/openstack/common/excutils.py",
line 68, in __exit__\n    six.reraise(self.type_, self.value,
self.tb)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/exception.py", line 71, in
wrapped\n    return f(self, context, *args, **kw)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 291, in
decorated_function\n    LOG.warning(msg, e,
instance_uuid=instance_uuid)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/openstack/common/excutils.py",
line 68, in __exit__\n    six.reraise(self.type_, self.value,
self.tb)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 268, in
decorated_function\n    return function(self, context, *args,
**kwargs)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 320, in
decorated_function\n    e, sys.exc_info())\n', '  File
"/usr/lib/python2.7/dist-packages/nova/openstack/common/excutils.py",
line 68, in __exit__\n    six.reraise(self.type_, self.value,
self.tb)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 307, in
decorated_function\n    return function(self, context, *args,
**kwargs)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 4296,
in detach_volume\n    self._detach_volume(context, instance, bdm)\n', ' 
File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line
4243, in _detach_volume\n    connection_info =
jsonutils.loads(bdm.connection_info)\n', '  File
"/usr/lib/python2.7/dist-packages/nova/openstack/common/jsonutils.py",
line 164, in loads\n    return json.loads(s)\n', '  File
"/usr/lib/python2.7/json/__init__.py", line 326, in loads\n    return
_default_decoder.decode(s)\n', '  File
"/usr/lib/python2.7/json/decoder.py", line 366, in decode\n    obj, end
= self.raw_decode(s, idx=_w(s, 0).end())\n', 'TypeError: expected string
or buffer\n']

Nova-Compute.conf:

[DEFAULT]
libvirt_type=kvm
libvirt_ovs_bridge=br-int
libvirt_vif_type=ethernet
libvirt_vif_driver=nova.virt.libvirt.vif.LibvirtGenericVIFDriver
vif_plugging_is_fatal=True
libvirt_nonblocking = True
libvirt_use_virtio_for_bridges=True
libvirt_cpu_mode=host-passthrough
disk_cachemodes="file=writeback,block=none"
running_deleted_instance_action=reap
libvirt_inject_partition = -2
vif_plugging_timeout = 0
compute_driver=nova.virt.libvirt.LibvirtDriver
libvirt_iscsi_use_multipath=True

Anynone has seen this error before ? Is there any solution ?

Cheers and thanks Heiko