Ask Your Question
0

Cinder-volume going down from up

asked 2016-11-20 07:45:02 -0600

Mutty Putty gravatar image

I am deploying openstack newton on Ubuntu 16.04.

My cinder volume service suddenly goes down from up state.

If I restart services again it is show up for few second then goes to down. I have checked time service and message queue services. But I could not find out the proper reason.

Time sychronization output:

root@hpc-cluster-host3:/home/hpchost3# chronyc sources

210 Number of sources = 1
MS Name/IP address         Stratum Poll Reach LastRx Last sample

^? controller                    0  10     0   10y     +0ns[   +0ns] +/-    0ns

root@controller:/home/hpchost1# rabbitmqctl status

Status of node rabbit@controller ...
[{pid,2366},

 {running_applications,[{rabbit,"RabbitMQ","3.5.7"},

                        {mnesia,"MNESIA  CXC 138 12","4.13.3"},
                        {xmerl,"XML parser","1.3.10"},
                        {os_mon,"CPO  CXC 138 46","2.4"},
                        {sasl,"SASL  CXC 138 11","2.7"},
                        {stdlib,"ERTS  CXC 138 10","2.8"},
                        {kernel,"ERTS  CXC 138 10","4.2"}]},
 {os,{unix,linux}},

 {erlang_version,"Erlang/OTP 18 [erts-7.3] [source] [64-bit] [smp:32:32] [async-threads:64] [kernel-poll:true]\n"},
 {memory,[{total,690022024},

          {connection_readers,2326320},
          {connection_writers,621008},
          {connection_channels,2070648},
          {connection_other,7142160},
          {queue_procs,5266552},
          {queue_slave_procs,0},
          {plugins,0},
          {other_proc,15665616},
          {mnesia,947344},
          {mgmt_db,0},
          {msg_index,202288},
          {other_ets,1163008},
          {binary,607076944},
          {code,16972695},
          {atom,654217},
          {other_system,29913224}]},
 {alarms,[]},
 {listeners,[{clustering,25672,"::"},{amqp,5672,"::"}]},
 {vm_memory_high_watermark,0.4},
 {vm_memory_limit,40444759244},
 {disk_free_limit,50000000},
 {disk_free,833303617536},
 {file_descriptors,[{total_limit,65436},
                    {total_used,246},
                    {sockets_limit,58890},
                    {sockets_used,244}]},
 {processes,[{limit,1048576},{used,3414}]},
 {run_queue,0},
 {uptime,536692}]

root@controller:/home/hpchost1# ps -ef | grep 2366

rabbitmq  2366  1597  5 Nov14 ?        07:28:21 /usr/lib/erlang/erts-7.3/bin/beam.smp -W w -A 64 -P 1048576 -K true -B i -- -root /usr/lib/erlang -progname erl -- -home /var/lib/rabbitmq -- -pa /usr/lib/rabbitmq/lib/rabbitmq_server-3.5.7/sbin/../ebin -noshell -noinput -s rabbit boot -sname rabbit@controller -boot start_sasl -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/var/log/rabbitmq/rabbit@controller.log"} -rabbit sasl_error_logger {file,"/var/log/rabbitmq/rabbit@controller-sasl.log"} -rabbit enabled_plugins_file "/etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/lib/rabbitmq/lib/rabbitmq_server-3.5.7/sbin/../plugins" -rabbit plugins_expand_dir "/var/lib/rabbitmq/mnesia/rabbit@controller-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/rabbitmq/mnesia/rabbit@controller" -kernel inet_dist_listen_min 25672 -kernel inet_dist_listen_max 25672
rabbitmq  2639  2366  0 Nov14 ?        00:00:06 inet_gethost 4
root     18126 19972  0 18:30 pts/0    00:00:00 grep --color=auto 2366

root@hpc-cluster-host3:/home/hpchost3# lsof -i :5672|grep hpc-cluster-host3

cinder-vo 5872 cinder    9u  IPv4  23304      0t0  TCP hpc-cluster-host3:36232->controller:amqp (ESTABLISHED)

Cinder Service list on controller Node:

hpchost1@controller:~$ cinder service-list

+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+
| Binary           | Host                  | Zone | Status  | State | Updated_at                 | Disabled Reason |
+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+
| cinder-scheduler | controller            | nova | enabled | up    | 2016-11-20T11:09:54.000000 | -               |
| cinder-volume    | hpc-cluster-host3@lvm | nova | enabled | up    | 2016-11-20T11:09:27.000000 | -               |
+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+

hpchost1@controller:~$ cinder service-list

+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+
| Binary           | Host                  | Zone | Status  | State | Updated_at                 | Disabled Reason |
+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+
| cinder-scheduler | controller            | nova | enabled | up    | 2016-11-20T11:11:15.000000 | -               |
| cinder-volume    | hpc-cluster-host3@lvm | nova | enabled | down  | 2016-11-20T11:09:27.000000 | -               |
+------------------+-----------------------+------+---------+-------+----------------------------+-----------------+

cinder-volume log file

2016-11-20 16:39:27.724 5859 WARNING oslo_reports.guru_meditation_report [-] Guru ...
(more)
edit retag flag offensive close merge delete

Comments

I found this by googling for the error message: https://fatmin.com/2015/06/08/opensta.... Perhaps you configured durable queues in one Cinder config file and non-durable queues in another (not that I pretend to know what durable queues are).

Bernd Bausch gravatar imageBernd Bausch ( 2016-11-20 09:16:00 -0600 )edit

Thank you so much @Bernd Bausch. I saw the mentioned blog earlier also. I don't know about durable queues in rabbit message queue. So I thought that if I change the rabbitmq configuration setting that will impact my overall environment. That's why I was afraid of changing. Any way I will give a try.

Mutty Putty gravatar imageMutty Putty ( 2016-11-21 09:56:23 -0600 )edit

Just check if you have conflicting configurations in your various config files.

Bernd Bausch gravatar imageBernd Bausch ( 2016-11-21 23:59:37 -0600 )edit

thanks @Bernd Bausch. I have made rabbitmq durable queue to true. That's solved my problem.

Mutty Putty gravatar imageMutty Putty ( 2016-11-23 01:22:36 -0600 )edit

1 answer

Sort by ยป oldest newest most voted
0

answered 2016-11-23 10:35:54 -0600

Mutty Putty gravatar image
I have made rabbitmq durable queue to true. That's solved my problem.
edit flag offensive delete link more

Your Answer

Please start posting anonymously - your entry will be published after you log in or create a new account.

Add Answer

Get to know Ask OpenStack

Resources for moderators

Question Tools

1 follower

Stats

Asked: 2016-11-20 07:45:02 -0600

Seen: 350 times

Last updated: Nov 23 '16