Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

Cinder-volume going down from up

I am deploying openstack newton on Ubuntu 16.04.

My cinder volume service suddenly goes down from up state.

If I restart services again it is show up for few second then goes to down. I have checked time service and message queue services. But I could not find out the proper reason.

Time sychronization output:

root@hpc-cluster-host3:/home/hpchost3# chronyc sources

210 Number of sources = 1
MS Name/IP address         Stratum Poll Reach LastRx Last sample

^? controller                    0  10     0   10y     +0ns[   +0ns] +/-    0ns

root@controller:/home/hpchost1# rabbitmqctl status

Status of node rabbit@controller ...
[{pid,2366},

 {running_applications,[{rabbit,"RabbitMQ","3.5.7"},

                        {mnesia,"MNESIA  CXC 138 12","4.13.3"},
                        {xmerl,"XML parser","1.3.10"},
                        {os_mon,"CPO  CXC 138 46","2.4"},
                        {sasl,"SASL  CXC 138 11","2.7"},
                        {stdlib,"ERTS  CXC 138 10","2.8"},
                        {kernel,"ERTS  CXC 138 10","4.2"}]},
 {os,{unix,linux}},

 {erlang_version,"Erlang/OTP 18 [erts-7.3] [source] [64-bit] [smp:32:32] [async-threads:64] [kernel-poll:true]\n"},
 {memory,[{total,690022024},

          {connection_readers,2326320},
          {connection_writers,621008},
          {connection_channels,2070648},
          {connection_other,7142160},
          {queue_procs,5266552},
          {queue_slave_procs,0},
          {plugins,0},
          {other_proc,15665616},
          {mnesia,947344},
          {mgmt_db,0},
          {msg_index,202288},
          {other_ets,1163008},
          {binary,607076944},
          {code,16972695},
          {atom,654217},
          {other_system,29913224}]},
 {alarms,[]},
 {listeners,[{clustering,25672,"::"},{amqp,5672,"::"}]},
 {vm_memory_high_watermark,0.4},
 {vm_memory_limit,40444759244},
 {disk_free_limit,50000000},
 {disk_free,833303617536},
 {file_descriptors,[{total_limit,65436},
                    {total_used,246},
                    {sockets_limit,58890},
                    {sockets_used,244}]},
 {processes,[{limit,1048576},{used,3414}]},
 {run_queue,0},
 {uptime,536692}]

root@controller:/home/hpchost1# ps -ef | grep 2366

rabbitmq  2366  1597  5 Nov14 ?        07:28:21 /usr/lib/erlang/erts-7.3/bin/beam.smp -W w -A 64 -P 1048576 -K true -B i -- -root /usr/lib/erlang -progname erl -- -home /var/lib/rabbitmq -- -pa /usr/lib/rabbitmq/lib/rabbitmq_server-3.5.7/sbin/../ebin -noshell -noinput -s rabbit boot -sname rabbit@controller -boot start_sasl -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/var/log/rabbitmq/rabbit@controller.log"} -rabbit sasl_error_logger {file,"/var/log/rabbitmq/rabbit@controller-sasl.log"} -rabbit enabled_plugins_file "/etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/lib/rabbitmq/lib/rabbitmq_server-3.5.7/sbin/../plugins" -rabbit plugins_expand_dir "/var/lib/rabbitmq/mnesia/rabbit@controller-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/rabbitmq/mnesia/rabbit@controller" -kernel inet_dist_listen_min 25672 -kernel inet_dist_listen_max 25672
rabbitmq  2639  2366  0 Nov14 ?        00:00:06 inet_gethost 4
root     18126 19972  0 18:30 pts/0    00:00:00 grep --color=auto 2366

root@hpc-cluster-host3:/home/hpchost3# lsof -i :5672|grep hpc-cluster-host3

cinder-vo 5872 cinder    9u  IPv4  23304      0t0  TCP hpc-cluster-host3:36232->controller:amqp (ESTABLISHED)

Cinder Service list on controller Node:

hpchost1@controller:~$ cinder service-list

+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+
| Binary           | Host                  | Zone | Status  | State | Updated_at                 | Disabled Reason |
+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+
| cinder-scheduler | controller            | nova | enabled | up    | 2016-11-20T11:09:54.000000 | -               |
| cinder-volume    | hpc-cluster-host3@lvm | nova | enabled | up    | 2016-11-20T11:09:27.000000 | -               |
+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+

hpchost1@controller:~$ cinder service-list

+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+
| Binary           | Host                  | Zone | Status  | State | Updated_at                 | Disabled Reason |
+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+
| cinder-scheduler | controller            | nova | enabled | up    | 2016-11-20T11:11:15.000000 | -               |
| cinder-volume    | hpc-cluster-host3@lvm | nova | enabled | down  | 2016-11-20T11:09:27.000000 | -               |
+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+

cinder-volume log file

2016-11-20 16:39:27.724 5859 WARNING oslo_reports.guru_meditation_report [-] Guru meditation now registers SIGUSR1 and SIGUSR2 by default for backward compatibility. SIGUSR1 will no longer be registered in a future release, so please use SIGUSR2 to generate reports.
2016-11-20 16:39:27.782 5859 INFO root [-] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
2016-11-20 16:39:27.802 5859 INFO root [-] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
2016-11-20 16:39:27.970 5859 INFO cinder.rpc [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Automatically selected cinder-volume objects version 1.11 as minimum service version.
2016-11-20 16:39:27.984 5859 INFO cinder.rpc [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Automatically selected cinder-scheduler objects version 1.11 as minimum service version.
2016-11-20 16:39:27.989 5859 INFO cinder.rpc [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Automatically selected cinder-scheduler RPC version 3.0 as minimum service version.
2016-11-20 16:39:28.045 5859 INFO cinder.volume.manager [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Determined volume DB was empty at startup.
2016-11-20 16:39:28.050 5859 WARNING cinder.keymgr.conf_key_mgr [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] This key manager is insecure and is not recommended for production deployments
2016-11-20 16:39:28.058 5859 INFO cinder.volume.manager [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Image-volume cache disabled for host hpc-cluster-host3@lvm.
2016-11-20 16:39:28.058 5859 INFO oslo_service.service [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Starting 1 workers
2016-11-20 16:39:28.069 5859 WARNING oslo_config.cfg [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Option "rabbit_host" from group "oslo_messaging_rabbit" is deprecated for removal.  Its value may be silently ignored in the future.
2016-11-20 16:39:28.069 5872 INFO cinder.service [-] Starting cinder-volume node (version 9.0.0)
2016-11-20 16:39:28.071 5859 WARNING oslo_config.cfg [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Option "rabbit_password" from group "oslo_messaging_rabbit" is deprecated for removal.  Its value may be silently ignored in the future.
2016-11-20 16:39:28.072 5859 WARNING oslo_config.cfg [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Option "rabbit_userid" from group "oslo_messaging_rabbit" is deprecated for removal.  Its value may be silently ignored in the future.
2016-11-20 16:39:28.090 5872 INFO cinder.coordination [-] Coordination backend started successfully.
2016-11-20 16:39:28.091 5872 INFO cinder.volume.manager [req-2b17cbff-e761-45b5-aeaf-44d6b2589fb4 - - - - -] Starting volume driver LVMVolumeDriver (3.0.0)
2016-11-20 16:39:31.089 5872 WARNING oslo_config.cfg [req-2b17cbff-e761-45b5-aeaf-44d6b2589fb4 - - - - -] Option "rabbit_host" from group "oslo_messaging_rabbit" is deprecated for removal.  Its value may be silently ignored in the future.
2016-11-20 16:39:31.090 5872 WARNING oslo_config.cfg [req-2b17cbff-e761-45b5-aeaf-44d6b2589fb4 - - - - -] Option "rabbit_userid" from group "oslo_messaging_rabbit" is deprecated for removal.  Its value may be silently ignored in the future.
2016-11-20 16:39:31.090 5872 WARNING oslo_config.cfg [req-2b17cbff-e761-45b5-aeaf-44d6b2589fb4 - - - - -] Option "rabbit_password" from group "oslo_messaging_rabbit" is deprecated for removal.  Its value may be silently ignored in the future.
2016-11-20 16:39:31.110 5872 INFO cinder.volume.manager [req-2b17cbff-e761-45b5-aeaf-44d6b2589fb4 - - - - -] Driver initialization completed successfully.

2016-11-20 16:39:31.137 5872 ERROR oslo.messaging._drivers.impl_rabbit [req-2b17cbff-e761-45b5-aeaf-44d6b2589fb4 - - - - -] Failed to declare consumer for topic 'cinder-volume': Exchange.declare: (406) PRECONDITION_FAILED - inequivalent arg 'durable' for exchange 'openstack' in vhost '/': received 'false' but current is 'true'

2016-11-20 16:39:31.139 5872 ERROR oslo_service.service [req-2b17cbff-e761-45b5-aeaf-44d6b2589fb4 - - - - -] Error starting thread.
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service Traceback (most recent call last):
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_service/service.py", line 708, in run_service
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     service.start()
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/cinder/service.py", line 247, in start
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     self.rpcserver.start()
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/server.py", line 268, in wrapper
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     log_after, timeout_timer)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/server.py", line 188, in run_once
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     post_fn = fn()
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/server.py", line 267, in <lambda>
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     states[state].run_once(lambda: fn(self, *args, **kwargs),
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/server.py", line 420, in start
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     self.listener = self._create_listener()
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/server.py", line 121, in _create_listener
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     return self.transport._listen(self._target, 1, None)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/transport.py", line 112, in _listen
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     batch_timeout)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 477, in listen
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     callback=listener)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/impl_rabbit.py", line 1127, in declare_topic_consumer
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     self.declare_consumer(consumer)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/impl_rabbit.py", line 1015, in declare_consumer
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     error_callback=_connect_error)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/impl_rabbit.py", line 791, in ensure
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     ret, channel = autoretry_method()
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 436, in _ensured
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     return fun(*args, **kwargs)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/kombu/connection.py", line 508, in __call__
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     return fun(*args, channel=channels[0], **kwargs), channels[0]
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/impl_rabbit.py", line 767, in execute_method
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     method()
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/impl_rabbit.py", line 1003, in _declare_consumer
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     consumer.declare(self)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/impl_rabbit.py", line 300, in declare
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     self.queue.declare()
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/kombu/entity.py", line 521, in declare
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     self.exchange.declare(nowait)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/kombu/entity.py", line 174, in declare
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     nowait=nowait, passive=passive,
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/amqp/channel.py", line 619, in exchange_declare
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     (40, 11),  # Channel.exchange_declare_ok
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/amqp/abstract_channel.py", line 69, in wait
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     return self.dispatch_method(method_sig, args, content)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/amqp/abstract_channel.py", line 87, in dispatch_method
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     return amqp_method(self, args)
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service   File "/usr/lib/python2.7/dist-packages/amqp/channel.py", line 243, in _close
2016-11-20 16:39:31.139 5872 ERROR oslo_service.service     reply_code, reply_text, (class_id, method_id), ChannelError,
**2016-11-20 16:39:31.139 5872 ERROR oslo_service.service PreconditionFailed: Exchange.declare: (406) PRECONDITION_FAILED - inequivalent arg 'durable' for exchange 'openstack' in vhost '/': received 'false' but current is 'true'**

2016-11-20 16:39:31.139 5872 ERROR oslo_service.service
2016-11-20 16:40:01.145 5872 WARNING oslo_messaging.server [req-3b441400-4019-4ddb-8fb3-32c9131dadc1 - - - - -] Possible hang: wait is waiting for stop to complete

I am stuck for last few days. Please help me.

Any kind of help will be appreciated.

Thanks in advance.