Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

Constantly seeing the "CRITICAL nova [req-b4004c91-3651-4230-9283-7ed9b4998ad1 None] OperationalError: (OperationalError) (1040, 'Too many connections') None None"

Constantly seeing the below mentioned errors on my openstack JUNO (all in one) system.

When this happens the system stops working and restarting various nova-services rectifies these issues but after some time the system runs into same state again (error + stops working).

Any idea the reason behind this and how to prevent this ?

Command failed, please check log for more info 2015-01-08 18:49:11.056 9097 CRITICAL nova [req-b4004c91-3651-4230-9283-7ed9b4998ad1 None] OperationalError: (OperationalError) (1040, 'Too many connections') None None 2015-01-08 18:49:11.056 9097 TRACE nova Traceback (most recent call last): 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/bin/nova-manage", line 10, in <module> 2015-01-08 18:49:11.056 9097 TRACE nova sys.exit(main()) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/nova/cmd/manage.py", line 1401, in main 2015-01-08 18:49:11.056 9097 TRACE nova ret = fn(fn_args, *fn_kwargs) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/nova/cmd/manage.py", line 704, in list 2015-01-08 18:49:11.056 9097 TRACE nova services = db.service_get_all(ctxt) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/nova/db/api.py", line 112, in service_get_all 2015-01-08 18:49:11.056 9097 TRACE nova return IMPL.service_get_all(context, disabled) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 125, in wrapper 2015-01-08 18:49:11.056 9097 TRACE nova return f(args, *kwargs) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 441, in service_get_all 2015-01-08 18:49:11.056 9097 TRACE nova query = model_query(context, models.Service) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 213, in model_query 2015-01-08 18:49:11.056 9097 TRACE nova session = kwargs.get('session') or get_session(use_slave=use_slave) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 101, in get_session 2015-01-08 18:49:11.056 9097 TRACE nova facade = _create_facade_lazily() 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 91, in _create_facade_lazily 2015-01-08 18:49:11.056 9097 TRACE nova _ENGINE_FACADE = db_session.EngineFacade.from_config(CONF) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/session.py", line 795, in from_config 2015-01-08 18:49:11.056 9097 TRACE nova retry_interval=conf.database.retry_interval) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/session.py", line 711, in __init__ 2015-01-08 18:49:11.056 9097 TRACE nova *engine_kwargs) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/session.py", line 386, in create_engine 2015-01-08 18:49:11.056 9097 TRACE nova connection_trace=connection_trace 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/utils.py", line 890, in __call__ 2015-01-08 18:49:11.056 9097 TRACE nova self._url_from_target(target), target, arg, kw) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/utils.py", line 927, in _dispatch_on 2015-01-08 18:49:11.056 9097 TRACE nova return self._dispatch_on_db_driver(dbname, driver, arg, kw) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/utils.py", line 981, in _dispatch_on_db_driver 2015-01-08 18:49:11.056 9097 TRACE nova if self._invoke_fn(fn, arg, kw) is not None: 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/utils.py", line 930, in _invoke_fn 2015-01-08 18:49:11.056 9097 TRACE nova return fn(arg, *kw) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/session.py", line 456, in _init_events 2015-01-08 18:49:11.056 9097 TRACE nova realmode = engine.execute("SHOW VARIABLES LIKE 'sql_mode'").fetchone() 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1751, in execute 2015-01-08 18:49:11.056 9097 TRACE nova connection = self.contextual_connect(close_with_result=True) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1799, in contextual_connect 2015-01-08 18:49:11.056 9097 TRACE nova self.pool.connect(), 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 338, in connect 2015-01-08 18:49:11.056 9097 TRACE nova return _ConnectionFairy._checkout(self) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 641, in _checkout 2015-01-08 18:49:11.056 9097 TRACE nova fairy = _ConnectionRecord.checkout(pool) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 440, in checkout 2015-01-08 18:49:11.056 9097 TRACE nova rec = pool._do_get() 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 961, in _do_get 2015-01-08 18:49:11.056 9097 TRACE nova return self._create_connection() 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 285, in _create_connection 2015-01-08 18:49:11.056 9097 TRACE nova return _ConnectionRecord(self) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 411, in __init__ 2015-01-08 18:49:11.056 9097 TRACE nova self.connection = self.__connect() 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 537, in __connect 2015-01-08 18:49:11.056 9097 TRACE nova connection = self.__pool._creator() 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/strategies.py", line 96, in connect 2015-01-08 18:49:11.056 9097 TRACE nova connection_invalidated=invalidated 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause 2015-01-08 18:49:11.056 9097 TRACE nova reraise(type(exception), exception, tb=exc_tb) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/strategies.py", line 90, in connect 2015-01-08 18:49:11.056 9097 TRACE nova return dialect.connect(cargs, *cparams) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 377, in connect 2015-01-08 18:49:11.056 9097 TRACE nova return self.dbapi.connect(cargs, *cparams) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/MySQLdb/__init__.py", line 81, in Connect 2015-01-08 18:49:11.056 9097 TRACE nova return Connection(args, *kwargs) 2015-01-08 18:49:11.056 9097 TRACE nova File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 187, in __init__ 2015-01-08 18:49:11.056 9097 TRACE nova super(Connection, self).__init__(args, **kwargs2) 2015-01-08 18:49:11.056 9097 TRACE nova OperationalError: (OperationalError) (1040, 'Too many connections') None None 2015-01-08 18:49:11.056 9097 TRACE nova

Constantly seeing the "CRITICAL nova [req-b4004c91-3651-4230-9283-7ed9b4998ad1 None] OperationalError: (OperationalError) (1040, 'Too many connections') None None"

Constantly seeing the below mentioned errors on my openstack JUNO (all in one) system.

When this happens the system stops working and restarting various nova-services rectifies these issues but after some time the system runs into same state again (error + stops working).

Any idea the reason behind this and how to prevent this ?

Command failed, please check log for more info
2015-01-08 18:49:11.056 9097 CRITICAL nova [req-b4004c91-3651-4230-9283-7ed9b4998ad1 None] OperationalError: (OperationalError) (1040, 'Too many connections') None None
2015-01-08 18:49:11.056 9097 TRACE nova Traceback (most recent call last):
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/bin/nova-manage", line 10, in <module>
2015-01-08 18:49:11.056 9097 TRACE nova     sys.exit(main())
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/nova/cmd/manage.py", line 1401, in main
2015-01-08 18:49:11.056 9097 TRACE nova     ret = fn(fn_args, *fn_kwargs)
fn(*fn_args, **fn_kwargs)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/nova/cmd/manage.py", line 704, in list
2015-01-08 18:49:11.056 9097 TRACE nova     services = db.service_get_all(ctxt)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/nova/db/api.py", line 112, in service_get_all
2015-01-08 18:49:11.056 9097 TRACE nova     return IMPL.service_get_all(context, disabled)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 125, in wrapper
2015-01-08 18:49:11.056 9097 TRACE nova     return f(args, *kwargs)
f(*args, **kwargs)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 441, in service_get_all
2015-01-08 18:49:11.056 9097 TRACE nova     query = model_query(context, models.Service)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 213, in model_query
2015-01-08 18:49:11.056 9097 TRACE nova     session = kwargs.get('session') or get_session(use_slave=use_slave)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 101, in get_session
2015-01-08 18:49:11.056 9097 TRACE nova     facade = _create_facade_lazily()
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/nova/db/sqlalchemy/api.py", line 91, in _create_facade_lazily
2015-01-08 18:49:11.056 9097 TRACE nova     _ENGINE_FACADE = db_session.EngineFacade.from_config(CONF)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/session.py", line 795, in from_config
2015-01-08 18:49:11.056 9097 TRACE nova     retry_interval=conf.database.retry_interval)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/session.py", line 711, in __init__
2015-01-08 18:49:11.056 9097 TRACE nova     *engine_kwargs)
**engine_kwargs)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/session.py", line 386, in create_engine
2015-01-08 18:49:11.056 9097 TRACE nova     connection_trace=connection_trace
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/utils.py", line 890, in __call__
2015-01-08 18:49:11.056 9097 TRACE nova     self._url_from_target(target), target, arg, kw)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/utils.py", line 927, in _dispatch_on
2015-01-08 18:49:11.056 9097 TRACE nova     return self._dispatch_on_db_driver(dbname, driver, arg, kw)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/utils.py", line 981, in _dispatch_on_db_driver
2015-01-08 18:49:11.056 9097 TRACE nova     if self._invoke_fn(fn, arg, kw) is not None:
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/utils.py", line 930, in _invoke_fn
2015-01-08 18:49:11.056 9097 TRACE nova     return fn(arg, *kw)
fn(*arg, **kw)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/oslo/db/sqlalchemy/session.py", line 456, in _init_events
2015-01-08 18:49:11.056 9097 TRACE nova     realmode = engine.execute("SHOW VARIABLES LIKE 'sql_mode'").fetchone()
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1751, in execute
2015-01-08 18:49:11.056 9097 TRACE nova     connection = self.contextual_connect(close_with_result=True)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1799, in contextual_connect
2015-01-08 18:49:11.056 9097 TRACE nova     self.pool.connect(),
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 338, in connect
2015-01-08 18:49:11.056 9097 TRACE nova     return _ConnectionFairy._checkout(self)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 641, in _checkout
2015-01-08 18:49:11.056 9097 TRACE nova     fairy = _ConnectionRecord.checkout(pool)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 440, in checkout
2015-01-08 18:49:11.056 9097 TRACE nova     rec = pool._do_get()
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 961, in _do_get
2015-01-08 18:49:11.056 9097 TRACE nova     return self._create_connection()
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 285, in _create_connection
2015-01-08 18:49:11.056 9097 TRACE nova     return _ConnectionRecord(self)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 411, in __init__
2015-01-08 18:49:11.056 9097 TRACE nova     self.connection = self.__connect()
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 537, in __connect
2015-01-08 18:49:11.056 9097 TRACE nova     connection = self.__pool._creator()
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/strategies.py", line 96, in connect
2015-01-08 18:49:11.056 9097 TRACE nova     connection_invalidated=invalidated
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
2015-01-08 18:49:11.056 9097 TRACE nova     reraise(type(exception), exception, tb=exc_tb)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/strategies.py", line 90, in connect
2015-01-08 18:49:11.056 9097 TRACE nova     return dialect.connect(cargs, *cparams)
dialect.connect(*cargs, **cparams)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 377, in connect
2015-01-08 18:49:11.056 9097 TRACE nova     return self.dbapi.connect(cargs, *cparams)
self.dbapi.connect(*cargs, **cparams)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/MySQLdb/__init__.py", line 81, in Connect
2015-01-08 18:49:11.056 9097 TRACE nova     return Connection(args, *kwargs)
Connection(*args, **kwargs)
2015-01-08 18:49:11.056 9097 TRACE nova   File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 187, in __init__
2015-01-08 18:49:11.056 9097 TRACE nova     super(Connection, self).__init__(args, self).__init__(*args, **kwargs2)
2015-01-08 18:49:11.056 9097 TRACE nova OperationalError: (OperationalError) (1040, 'Too many connections') None None
2015-01-08 18:49:11.056 9097 TRACE nova