Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

error in nova verification and in neutron

After setting up the cloud environment using openstack and creating a lot of instances. Now i'm not able to create any new instance or to access to the cloud.

root@controller:/var/log/neutron# openstack server list                                               Failed to discover available identity versions when contacting http://192.168.1.23:5000/v3. Attempting to parse version from URL.                                                                             Unable to establish connection to http://192.168.1.23:5000/v3/auth/tokens: HTTPConnectionPool(host='192.168.1.23', port=5000): Max retries exceeded with url: /v3/auth/tokens (Caused by NewConnectionError("<urllib3.connection.HTTPConnection object at 0x7f32b20fe990>: Failed to establish a new connection: [Errno 113] Aucun chemin d'acc\xc3\xa8s pour atteindre l'h\xc3\xb4te cible",))



root@controller:~# nova-status upgrade check  SQL connection failed. 10 attempts left.

SQL connection failed. 9 attempts left. SQL connection failed. 8 attempts left. SQL connection failed. 7 attempts left. SQL connection failed. 6 attempts left. SQL connection failed. 5 attempts left. SQL connection failed. 4 attempts left. SQL connection failed. 3 attempts left. SQL connection failed. 2 attempts left. SQL connection failed. 1 attempts left. Error:                                                                                                                                                 Traceback (most recent call last): File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 920, in connect **kwargs) File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 69, in create_connection raise err File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 59, in create_connection sock.connect(sa) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 256, in connect socket_checkerr(fd) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 46, in socket_checkerr raise socket.error(err, errno.errorcode[err]) OSError: [Errno 113] EHOSTUNREAC  During handling of the above exception, another exception occurred:                                                                      Traceback (most recent call last): File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 2158, in _wrap_pool_connect return fn() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 345, in unique_connection return _ConnectionFairy._checkout(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 791, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 532, in checkout rec = pool._do_get() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1196, in _do_get self._dec_overflow() File "/usr/lib/python3/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in exit compat.reraise(exc_type, exc_value, exc_tb) File "/usr/lib/python3/dist-packages/sqlalchemy/util/compat.py", line 187, in reraise raise value File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1193, in _do_get return self._create_connection() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 350, in _create_connection return _ConnectionRecord(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 477, in init self.connect(first_connect_check=True) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 674, in __connect connection = pool._invoke_creator(self) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/strategies.py", line 106, in connect return dialect.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/default.py", line 411, in connect return self.dbapi.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/pymysql/__init.py", line 90, in Connect return Connection(*args, **kwargs) File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 699, in init self.connect() File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 967, in connect raise exc pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") The above exception was the direct cause of the following exception:                                                                                                                            Traceback (most recent call last): File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 774, in main ret = fn(*fn_args, **fn_kwargs) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 704, in check result = func(self) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 146, in _check_cellsv2 meta.bind = db_session.get_api_engine() File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 156, in get_api_engine return api_context_manager.writer.get_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 814, in get_engine return self._factory.get_writer_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 363, in get_writer_engine self._start() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 496, in _start engine_args, maker_args) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 520, in _setup_for_connection sql_connection=sql_connection, **engine_kwargs) File "/usr/lib/python3/dist-packages/debtcollector/renames.py", line 43, in decorator return wrapped(*args, **kwargs) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 202, in create_engine test_conn = _test_connection(engine, max_retries, retry_interval) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 388, in _test_connection six.reraise(type(de_ref), de_ref) File "/usr/lib/python3/dist-packages/six.py", line 692, in reraise raise value.with_traceback(tb) oslo_db.exception.DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-server.log

2019-01-26 15:31:27.639 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:39.928 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 9 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:52.215 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 8 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:04.154 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 7 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:16.440 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 6 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:28.727 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 5 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:39.223 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 4 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:51.511 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 3 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:03.799 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 2 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:16.088 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 1 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:26.091 8938 DEBUG neutron_lib.db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Retry wrapper got retriable exception: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) wrapped /usr/lib/python2.7/dist-packages/neutron_lib/db/api.py:183 2019-01-26 15:33:26.092 8938 DEBUG oslo_db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Performing DB retry for function neutron.plugins.ml2.drivers.type_vlan._sync_vlan_allocations wrapper /usr/lib/python2.7/dist-packages/oslo_db/api.py:156 2019-01-26 15:33:31.451 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-dhcp-agent.log

2019-01-26 15:43:59.034 3353 ERROR oslo.messaging._drivers.impl_rabbit [-] [a20207f2-abed-4b5c-91a4-65ba4d229132] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH 2019-01-26 15:44:08.247 3353 WARNING oslo.messaging._drivers.impl_rabbit [-] Unexpected error during heartbeart thread processing, retrying...: error: [Errno 113] EHOSTUNREACH

nano /var/log/neutron

2019-01-26 15:47:32.151 3344 ERROR oslo.messaging._drivers.impl_rabbit [-] [6208e08b-ebf4-41c7-b393-1d72ee80c0b9] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH

error in nova verification and in neutron

After setting up the cloud environment using openstack and creating a lot of instances. Now i'm not able to create any new instance or to access to the cloud.

root@controller:/var/log/neutron# openstack server list                                               Failed to discover available identity versions when contacting http://192.168.1.23:5000/v3. Attempting to parse version from URL.                                                                             Unable to establish connection to http://192.168.1.23:5000/v3/auth/tokens: HTTPConnectionPool(host='192.168.1.23', port=5000): Max retries exceeded with url: /v3/auth/tokens (Caused by NewConnectionError("<urllib3.connection.HTTPConnection object at 0x7f32b20fe990>: Failed to establish a new connection: [Errno 113] Aucun chemin d'acc\xc3\xa8s pour atteindre l'h\xc3\xb4te cible",))



root@controller:~# nova-status upgrade check  SQL  connection failed. 10 attempts left.
 SQL connection failed. 9 attempts left.  SQL connection failed. 8 attempts left.  SQL connection failed. 7 attempts left.  SQL connection failed. 6 attempts left.  SQL connection failed. 5 attempts left.  SQL connection failed. 4 attempts left.  SQL connection failed. 3 attempts left.  SQL connection failed. 2 attempts left.  SQL connection failed. 1 attempts left.  Error:                                                                                                                                                 Traceback (most recent call last): File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 920, in connect **kwargs) File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 69, in create_connection raise err File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 59, in create_connection sock.connect(sa) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 256, in connect socket_checkerr(fd) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 46, in socket_checkerr raise socket.error(err, errno.errorcode[err]) OSError: [Errno 113] EHOSTUNREAC  During handling of the above exception, another exception occurred:                                                                      Traceback (most recent call last): File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 2158, in _wrap_pool_connect return fn() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 345, in unique_connection return _ConnectionFairy._checkout(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 791, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 532, in checkout rec = pool._do_get() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1196, in _do_get self._dec_overflow() File "/usr/lib/python3/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in exit compat.reraise(exc_type, exc_value, exc_tb) File "/usr/lib/python3/dist-packages/sqlalchemy/util/compat.py", line 187, in reraise raise value File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1193, in _do_get return self._create_connection() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 350, in _create_connection return _ConnectionRecord(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 477, in init self.connect(first_connect_check=True) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 674, in __connect connection = pool._invoke_creator(self) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/strategies.py", line 106, in connect return dialect.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/default.py", line 411, in connect return self.dbapi.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/pymysql/__init.py", line 90, in Connect return Connection(*args, **kwargs) File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 699, in init self.connect() File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 967, in connect raise exc pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") The above exception was the direct cause of the following exception:                                                                                                                            Traceback (most recent call last): File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 774, in main ret = fn(*fn_args, **fn_kwargs) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 704, in check result = func(self) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 146, in _check_cellsv2 meta.bind = db_session.get_api_engine() File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 156, in get_api_engine return api_context_manager.writer.get_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 814, in get_engine return self._factory.get_writer_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 363, in get_writer_engine self._start() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 496, in _start engine_args, maker_args) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 520, in _setup_for_connection sql_connection=sql_connection, **engine_kwargs) File "/usr/lib/python3/dist-packages/debtcollector/renames.py", line 43, in decorator return wrapped(*args, **kwargs) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 202, in create_engine test_conn = _test_connection(engine, max_retries, retry_interval) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 388, in _test_connection six.reraise(type(de_ref), de_ref) File "/usr/lib/python3/dist-packages/six.py", line 692, in reraise raise value.with_traceback(tb) oslo_db.exception.DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-server.log

2019-01-26 15:31:27.639 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:39.928 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 9 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:52.215 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 8 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:04.154 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 7 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:16.440 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 6 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:28.727 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 5 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:39.223 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 4 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:51.511 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 3 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:03.799 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 2 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:16.088 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 1 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:26.091 8938 DEBUG neutron_lib.db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Retry wrapper got retriable exception: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) wrapped /usr/lib/python2.7/dist-packages/neutron_lib/db/api.py:183 2019-01-26 15:33:26.092 8938 DEBUG oslo_db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Performing DB retry for function neutron.plugins.ml2.drivers.type_vlan._sync_vlan_allocations wrapper /usr/lib/python2.7/dist-packages/oslo_db/api.py:156 2019-01-26 15:33:31.451 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-dhcp-agent.log

2019-01-26 15:43:59.034 3353 ERROR oslo.messaging._drivers.impl_rabbit [-] [a20207f2-abed-4b5c-91a4-65ba4d229132] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH 2019-01-26 15:44:08.247 3353 WARNING oslo.messaging._drivers.impl_rabbit [-] Unexpected error during heartbeart thread processing, retrying...: error: [Errno 113] EHOSTUNREACH

nano /var/log/neutron

2019-01-26 15:47:32.151 3344 ERROR oslo.messaging._drivers.impl_rabbit [-] [6208e08b-ebf4-41c7-b393-1d72ee80c0b9] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH

error in nova verification and in neutron

After setting up the cloud environment using openstack and creating a lot of instances. Now i'm not able to create any new instance or to access to the cloud.

root@controller:/var/log/neutron# openstack server list                                               Failed to discover available identity versions when contacting http://192.168.1.23:5000/v3. Attempting to parse version from URL.                                                                             Unable to establish connection to http://192.168.1.23:5000/v3/auth/tokens: HTTPConnectionPool(host='192.168.1.23', port=5000): Max retries exceeded with url: /v3/auth/tokens (Caused by NewConnectionError("<urllib3.connection.HTTPConnection object at 0x7f32b20fe990>: Failed to establish a new connection: [Errno 113] Aucun chemin d'acc\xc3\xa8s pour atteindre l'h\xc3\xb4te cible",))



root@controller:~# nova-status upgrade check  SQL                                               connection failed. 10 attempts left.
SQL connection failed. 9 attempts left.                                                       SQL connection failed. 8 attempts left.                                                      SQL connection failed. 7 attempts left.                                                               SQL connection failed. 6 attempts left.                                                                          SQL connection failed. 5 attempts left.                                                                SQL connection failed. 4 attempts left.                                                                                 SQL connection failed. 3 attempts left.                                                         SQL connection failed. 2 attempts left.                                                       SQL connection failed. 1 attempts left.                                                                                       Error:                                                                                                                                                 Traceback (most recent call last): File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 920, in connect **kwargs) File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 69, in create_connection raise err File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 59, in create_connection sock.connect(sa) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 256, in connect socket_checkerr(fd) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 46, in socket_checkerr raise socket.error(err, errno.errorcode[err]) OSError: [Errno 113] EHOSTUNREAC  During handling of the above exception, another exception occurred:                                                                      Traceback (most recent call last): File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 2158, in _wrap_pool_connect return fn() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 345, in unique_connection return _ConnectionFairy._checkout(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 791, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 532, in checkout rec = pool._do_get() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1196, in _do_get self._dec_overflow() File "/usr/lib/python3/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in exit compat.reraise(exc_type, exc_value, exc_tb) File "/usr/lib/python3/dist-packages/sqlalchemy/util/compat.py", line 187, in reraise raise value File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1193, in _do_get return self._create_connection() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 350, in _create_connection return _ConnectionRecord(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 477, in init self.connect(first_connect_check=True) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 674, in __connect connection = pool._invoke_creator(self) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/strategies.py", line 106, in connect return dialect.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/default.py", line 411, in connect return self.dbapi.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/pymysql/__init.py", line 90, in Connect return Connection(*args, **kwargs) File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 699, in init self.connect() File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 967, in connect raise exc pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") The above exception was the direct cause of the following exception:                                                                                                                            Traceback (most recent call last): File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 774, in main ret = fn(*fn_args, **fn_kwargs) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 704, in check result = func(self) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 146, in _check_cellsv2 meta.bind = db_session.get_api_engine() File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 156, in get_api_engine return api_context_manager.writer.get_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 814, in get_engine return self._factory.get_writer_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 363, in get_writer_engine self._start() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 496, in _start engine_args, maker_args) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 520, in _setup_for_connection sql_connection=sql_connection, **engine_kwargs) File "/usr/lib/python3/dist-packages/debtcollector/renames.py", line 43, in decorator return wrapped(*args, **kwargs) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 202, in create_engine test_conn = _test_connection(engine, max_retries, retry_interval) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 388, in _test_connection six.reraise(type(de_ref), de_ref) File "/usr/lib/python3/dist-packages/six.py", line 692, in reraise raise value.with_traceback(tb) oslo_db.exception.DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-server.log

2019-01-26 15:31:27.639 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:39.928 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 9 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:52.215 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 8 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:04.154 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 7 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:16.440 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 6 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:28.727 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 5 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:39.223 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 4 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:51.511 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 3 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:03.799 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 2 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:16.088 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 1 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:26.091 8938 DEBUG neutron_lib.db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Retry wrapper got retriable exception: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) wrapped /usr/lib/python2.7/dist-packages/neutron_lib/db/api.py:183 2019-01-26 15:33:26.092 8938 DEBUG oslo_db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Performing DB retry for function neutron.plugins.ml2.drivers.type_vlan._sync_vlan_allocations wrapper /usr/lib/python2.7/dist-packages/oslo_db/api.py:156 2019-01-26 15:33:31.451 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-dhcp-agent.log

2019-01-26 15:43:59.034 3353 ERROR oslo.messaging._drivers.impl_rabbit [-] [a20207f2-abed-4b5c-91a4-65ba4d229132] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH 2019-01-26 15:44:08.247 3353 WARNING oslo.messaging._drivers.impl_rabbit [-] Unexpected error during heartbeart thread processing, retrying...: error: [Errno 113] EHOSTUNREACH

nano /var/log/neutron

2019-01-26 15:47:32.151 3344 ERROR oslo.messaging._drivers.impl_rabbit [-] [6208e08b-ebf4-41c7-b393-1d72ee80c0b9] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH

error in nova verification and in neutron

After setting up the cloud environment using openstack and creating a lot of instances. Now i'm not able to create any new instance or to access to the cloud.

root@controller:/var/log/neutron# openstack server list                                               Failed to discover available identity versions when contacting http://192.168.1.23:5000/v3. Attempting to parse version from URL.                                                                             Unable to establish connection to http://192.168.1.23:5000/v3/auth/tokens: HTTPConnectionPool(host='192.168.1.23', port=5000): Max retries exceeded with url: /v3/auth/tokens (Caused by NewConnectionError("<urllib3.connection.HTTPConnection object at 0x7f32b20fe990>: Failed to establish a new connection: [Errno 113] Aucun chemin d'acc\xc3\xa8s pour atteindre l'h\xc3\xb4te cible",))



root@controller:~# nova-status upgrade check  SQL                                               connection failed. 10 attempts left.
SQL connection failed. 9 attempts left.                                                       SQL connection failed. 8 attempts left.                                                        SQL connection failed. 7 attempts left.                                                               SQL connection failed. 6 attempts left.                                                                          SQL connection failed. 5 attempts left.                                                                SQL connection failed. 4 attempts left.                                                                                 SQL connection failed. 3 attempts left.                                                         SQL connection failed. 2 attempts left.                                                       SQL connection failed. 1 attempts left.                                                                                       Error:                                                                                                                                                 Traceback (most recent call last): File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 920, in connect **kwargs) File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 69, in create_connection raise err File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 59, in create_connection sock.connect(sa) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 256, in connect socket_checkerr(fd) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 46, in socket_checkerr raise socket.error(err, errno.errorcode[err]) OSError: [Errno 113] EHOSTUNREAC  During handling of the above exception, another exception occurred:                                                                      Traceback (most recent call last): File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 2158, in _wrap_pool_connect return fn() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 345, in unique_connection return _ConnectionFairy._checkout(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 791, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 532, in checkout rec = pool._do_get() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1196, in _do_get self._dec_overflow() File "/usr/lib/python3/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in exit compat.reraise(exc_type, exc_value, exc_tb) File "/usr/lib/python3/dist-packages/sqlalchemy/util/compat.py", line 187, in reraise raise value File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1193, in _do_get return self._create_connection() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 350, in _create_connection return _ConnectionRecord(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 477, in init self.connect(first_connect_check=True) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 674, in __connect connection = pool._invoke_creator(self) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/strategies.py", line 106, in connect return dialect.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/default.py", line 411, in connect return self.dbapi.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/pymysql/__init.py", line 90, in Connect return Connection(*args, **kwargs) File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 699, in init self.connect() File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 967, in connect raise exc pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") The above exception was the direct cause of the following exception:                                                                                                                            Traceback (most recent call last): File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 774, in main ret = fn(*fn_args, **fn_kwargs) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 704, in check result = func(self) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 146, in _check_cellsv2 meta.bind = db_session.get_api_engine() File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 156, in get_api_engine return api_context_manager.writer.get_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 814, in get_engine return self._factory.get_writer_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 363, in get_writer_engine self._start() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 496, in _start engine_args, maker_args) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 520, in _setup_for_connection sql_connection=sql_connection, **engine_kwargs) File "/usr/lib/python3/dist-packages/debtcollector/renames.py", line 43, in decorator return wrapped(*args, **kwargs) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 202, in create_engine test_conn = _test_connection(engine, max_retries, retry_interval) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 388, in _test_connection six.reraise(type(de_ref), de_ref) File "/usr/lib/python3/dist-packages/six.py", line 692, in reraise raise value.with_traceback(tb) oslo_db.exception.DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-server.log

2019-01-26 15:31:27.639 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:39.928 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 9 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:52.215 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 8 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:04.154 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 7 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:16.440 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 6 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:28.727 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 5 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:39.223 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 4 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:51.511 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 3 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:03.799 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 2 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:16.088 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 1 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:26.091 8938 DEBUG neutron_lib.db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Retry wrapper got retriable exception: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) wrapped /usr/lib/python2.7/dist-packages/neutron_lib/db/api.py:183 2019-01-26 15:33:26.092 8938 DEBUG oslo_db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Performing DB retry for function neutron.plugins.ml2.drivers.type_vlan._sync_vlan_allocations wrapper /usr/lib/python2.7/dist-packages/oslo_db/api.py:156 2019-01-26 15:33:31.451 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-dhcp-agent.log

2019-01-26 15:43:59.034 3353 ERROR oslo.messaging._drivers.impl_rabbit [-] [a20207f2-abed-4b5c-91a4-65ba4d229132] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH 2019-01-26 15:44:08.247 3353 WARNING oslo.messaging._drivers.impl_rabbit [-] Unexpected error during heartbeart thread processing, retrying...: error: [Errno 113] EHOSTUNREACH

nano /var/log/neutron

2019-01-26 `2019-01-26 15:47:32.151 3344 ERROR oslo.messaging._drivers.impl_rabbit [-] [6208e08b-ebf4-41c7-b393-1d72ee80c0b9] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH EHOSTUNREACH

root@controller:~# ifconfig                                                                                           brq6c053128-c4: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
    inet 192.168.1.23  netmask 255.255.255.0  broadcast 192.168.1.255
    inet6 fe80::6033:feff:fef6:f058  prefixlen 64  scopeid 0x20<link>
    ether 70:85:c2:74:c4:84  txqueuelen 1000  (Ethernet)
    RX packets 54843  bytes 3733555 (3.7 MB)
    RX errors 0  dropped 0  overruns 0  frame 0
    TX packets 9551  bytes 1248222 (1.2 MB)
    TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0                                         enp5s0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
    inet6 fe80::7285:c2ff:fe74:c484  prefixlen 64  scopeid 0x20<link>
    ether 70:85:c2:74:c4:84  txqueuelen 1000  (Ethernet)
    RX packets 59204  bytes 7626879 (7.6 MB)
    RX errors 0  dropped 2  overruns 0  frame 0
    TX packets 10725  bytes 1360730 (1.3 MB)
    TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0                                               lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
    inet 127.0.0.1  netmask 255.0.0.0
    inet6 ::1  prefixlen 128  scopeid 0x10<host>
    loop  txqueuelen 1000  (Boucle locale)
    RX packets 278278  bytes 58043196 (58.0 MB)
    RX errors 0  dropped 0  overruns 0  frame 0
    TX packets 278278  bytes 58043196 (58.0 MB)
    TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0                                                          tap32a7710d-e6: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
    ether fa:81:ed:2d:5c:07  txqueuelen 1000  (Ethernet)
    RX packets 9  bytes 940 (940.0 B)
    RX errors 0  dropped 0  overruns 0  frame 0
    TX packets 44889  bytes 3549935 (3.5 MB)
    TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

error in nova verification and in neutron

After setting up the cloud environment using openstack and creating a lot of instances. Now i'm not able to create any new instance or to access to the cloud.

root@controller:/var/log/neutron# openstack server list                                               Failed to discover available identity versions when contacting http://192.168.1.23:5000/v3. Attempting to parse version from URL.                                                                             Unable to establish connection to http://192.168.1.23:5000/v3/auth/tokens: HTTPConnectionPool(host='192.168.1.23', port=5000): Max retries exceeded with url: /v3/auth/tokens (Caused by NewConnectionError("<urllib3.connection.HTTPConnection object at 0x7f32b20fe990>: Failed to establish a new connection: [Errno 113] Aucun chemin d'acc\xc3\xa8s pour atteindre l'h\xc3\xb4te cible",))



root@controller:~# nova-status upgrade check  SQL                                               connection failed. 10 attempts left.
SQL connection failed. 9 attempts left.                                                       SQL connection failed. 8 attempts left.                                                        SQL connection failed. 7 attempts left.                                                               SQL connection failed. 6 attempts left.                                                                          SQL connection failed. 5 attempts left.                                                                SQL connection failed. 4 attempts left.                                                                                 SQL connection failed. 3 attempts left.                                                         SQL connection failed. 2 attempts left.                                                       SQL connection failed. 1 attempts left.                                                                                       Error:                                                                                                                                                 Traceback (most recent call last): File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 920, in connect **kwargs) File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 69, in create_connection raise err File "/usr/lib/python3/dist-packages/eventlet/green/socket.py", line 59, in create_connection sock.connect(sa) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 256, in connect socket_checkerr(fd) File "/usr/lib/python3/dist-packages/eventlet/greenio/base.py", line 46, in socket_checkerr raise socket.error(err, errno.errorcode[err]) OSError: [Errno 113] EHOSTUNREAC  During handling of the above exception, another exception occurred:                                                                      Traceback (most recent call last): File "/usr/lib/python3/dist-packages/sqlalchemy/engine/base.py", line 2158, in _wrap_pool_connect return fn() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 345, in unique_connection return _ConnectionFairy._checkout(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 791, in _checkout fairy = _ConnectionRecord.checkout(pool) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 532, in checkout rec = pool._do_get() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1196, in _do_get self._dec_overflow() File "/usr/lib/python3/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in exit compat.reraise(exc_type, exc_value, exc_tb) File "/usr/lib/python3/dist-packages/sqlalchemy/util/compat.py", line 187, in reraise raise value File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 1193, in _do_get return self._create_connection() File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 350, in _create_connection return _ConnectionRecord(self) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 477, in init self.connect(first_connect_check=True) File "/usr/lib/python3/dist-packages/sqlalchemy/pool.py", line 674, in __connect connection = pool._invoke_creator(self) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/strategies.py", line 106, in connect return dialect.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/sqlalchemy/engine/default.py", line 411, in connect return self.dbapi.connect(*cargs, **cparams) File "/usr/lib/python3/dist-packages/pymysql/__init.py", line 90, in Connect return Connection(*args, **kwargs) File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 699, in init self.connect() File "/usr/lib/python3/dist-packages/pymysql/connections.py", line 967, in connect raise exc pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") The above exception was the direct cause of the following exception:                                                                                                                            Traceback (most recent call last): File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 774, in main ret = fn(*fn_args, **fn_kwargs) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 704, in check result = func(self) File "/usr/lib/python3/dist-packages/nova/cmd/status.py", line 146, in _check_cellsv2 meta.bind = db_session.get_api_engine() File "/usr/lib/python3/dist-packages/nova/db/sqlalchemy/api.py", line 156, in get_api_engine return api_context_manager.writer.get_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 814, in get_engine return self._factory.get_writer_engine() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 363, in get_writer_engine self._start() File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 496, in _start engine_args, maker_args) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/enginefacade.py", line 520, in _setup_for_connection sql_connection=sql_connection, **engine_kwargs) File "/usr/lib/python3/dist-packages/debtcollector/renames.py", line 43, in decorator return wrapped(*args, **kwargs) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 202, in create_engine test_conn = _test_connection(engine, max_retries, retry_interval) File "/usr/lib/python3/dist-packages/oslo_db/sqlalchemy/engines.py", line 388, in _test_connection six.reraise(type(de_ref), de_ref) File "/usr/lib/python3/dist-packages/six.py", line 692, in reraise raise value.with_traceback(tb) oslo_db.exception.DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-server.log

2019-01-26 15:31:27.639 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:39.928 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 9 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:31:52.215 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 8 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:04.154 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 7 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:16.440 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 6 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:28.727 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 5 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:39.223 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 4 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:32:51.511 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 3 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:03.799 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 2 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:16.088 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 1 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) 2019-01-26 15:33:26.091 8938 DEBUG neutron_lib.db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Retry wrapper got retriable exception: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8) wrapped /usr/lib/python2.7/dist-packages/neutron_lib/db/api.py:183 2019-01-26 15:33:26.092 8938 DEBUG oslo_db.api [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] Performing DB retry for function neutron.plugins.ml2.drivers.type_vlan._sync_vlan_allocations wrapper /usr/lib/python2.7/dist-packages/oslo_db/api.py:156 2019-01-26 15:33:31.451 8938 WARNING oslo_db.sqlalchemy.engines [req-b538b9c5-9ef6-4d66-a475-eec5b03cd4d0 - - - - -] SQL connection failed. 10 attempts left.: DBConnectionError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on '192.168.1.23' ([Errno 113] EHOSTUNREACH)") (Background on this error at: http://sqlalche.me/e/e3q8)

nano /var/log/neutron/neutron-dhcp-agent.log

2019-01-26 15:43:59.034 3353 ERROR oslo.messaging._drivers.impl_rabbit [-] [a20207f2-abed-4b5c-91a4-65ba4d229132] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH 2019-01-26 15:44:08.247 3353 WARNING oslo.messaging._drivers.impl_rabbit [-] Unexpected error during heartbeart thread processing, retrying...: error: [Errno 113] EHOSTUNREACH

nano /var/log/neutron

`2019-01-26 15:47:32.151 3344 ERROR oslo.messaging._drivers.impl_rabbit [-] [6208e08b-ebf4-41c7-b393-1d72ee80c0b9] AMQP server on 192.168.1.23:5672 is unreachable: [Errno 113] EHOSTUNREACH. Trying again in 32 seconds.: error: [Errno 113] EHOSTUNREACH

root@controller:~# ifconfig                                                                                           brq6c053128-c4: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
    inet 192.168.1.23  netmask 255.255.255.0  broadcast 192.168.1.255
    inet6 fe80::6033:feff:fef6:f058  prefixlen 64  scopeid 0x20<link>
    ether 70:85:c2:74:c4:84  txqueuelen 1000  (Ethernet)
    RX packets 54843  bytes 3733555 (3.7 MB)
    RX errors 0  dropped 0  overruns 0  frame 0
    TX packets 9551  bytes 1248222 (1.2 MB)
    TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0                                         enp5s0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
    inet6 fe80::7285:c2ff:fe74:c484  prefixlen 64  scopeid 0x20<link>
    ether 70:85:c2:74:c4:84  txqueuelen 1000  (Ethernet)
    RX packets 59204  bytes 7626879 (7.6 MB)
    RX errors 0  dropped 2  overruns 0  frame 0
    TX packets 10725  bytes 1360730 (1.3 MB)
    TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0                                               lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
    inet 127.0.0.1  netmask 255.0.0.0
    inet6 ::1  prefixlen 128  scopeid 0x10<host>
    loop  txqueuelen 1000  (Boucle locale)
    RX packets 278278  bytes 58043196 (58.0 MB)
    RX errors 0  dropped 0  overruns 0  frame 0
    TX packets 278278  bytes 58043196 (58.0 MB)
    TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0                                                          tap32a7710d-e6: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
    ether fa:81:ed:2d:5c:07  txqueuelen 1000  (Ethernet)
    RX packets 9  bytes 940 (940.0 B)
    RX errors 0  dropped 0  overruns 0  frame 0
    TX packets 44889  bytes 3549935 (3.5 MB)
    TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0









#The management interface                                                                    auto enp5s0:0                                                                                iface enp5s0:0 inet static
  address 192.168.1.23
  netmask 255.255.255.0
  broadcast 192.168.1.255
  dns-nameservers 8.8.8.8
  dns-nameservers 8.8.4.4                                                                               #The provider interface                                                                               auto enp5s0                                                                                                                     iface enp5s0 inet manual                                                                    up ip link set dev enp5s0 up                                                                                      down ip link set dev enp5s0 down