Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

neutron-server returns 'Error:503' after update to queens

Hi folks,

I updated my openstack environment from newton to queens. Because i didn't found a best practice solution for an upgrade progress, i used apt to upgrade the packages to queens. I encountered serveral errors from keystone, glance and nova and could all fix them with some changes in the database and the config files (specialy i changed the auth. method from v2 to v3). But unfortunately i didn't found out the problem with my neutron configuration. Everytime when i try to list my networks i receive the following error:

******@ctrl:~$ openstack network list
HttpException: 503: Server Error for url: http://ctrl:9696/v2.0/networks, Service Unavailable

/var/log/neutron-server.log

2019-02-05 09:37:09.263 21087 WARNING keystonemiddleware.auth_token [-] Identity response: {"error": {"message": "The request you have made requires authentication.", "code": 401, "title": "Unauthorized"}}: Unauthorized: The request you have made requires authentication. (HTTP 401) (Request-ID: req-a7c96551-ca90-48e8-af95-297acc82bd23)
    2019-02-05 09:37:09.287 21087 WARNING keystonemiddleware.auth_token [-] Identity response: {"error": {"message": "The request you have made requires authentication.", "code": 401, "title": "Unauthorized"}}: Unauthorized: The request you have made requires authentication. (HTTP 401) (Request-ID: req-beed4c2e-1c65-4adf-bb3c-17b6b7170e64)
    2019-02-05 09:37:09.288 21087 CRITICAL keystonemiddleware.auth_token [-] Unable to validate token: Identity server rejected authorization necessary to fetch token data: ServiceError: Identity server rejected authorization necessary to fetch token data
    2019-02-05 09:37:09.289 21087 INFO neutron.wsgi [-] 192.168.0.2 "GET /v2.0/ports?tenant_id=885ea3ee676b41c6a2ef1d15b76e1ff0&device_id=286b1b13-cba6-45f7-8934-421f675778d4 HTTP/1.1" status: 503  len: 386 time: 0.0506890

/var/log/apache2/keystone_access.log

192.168.0.1 - - [05/Feb/2019:09:37:09 +0000] "POST /v3/auth/tokens HTTP/1.1" 401 482 "-"
"neutron/12.0.5 keystonemiddleware.auth_token/4.21.0 keystoneauth1/3.4.0 python-requests/
2.18.4 CPython/2.7.12"
192.168.0.1 - - [05/Feb/2019:09:37:09 +0000] "POST /v3/auth/tokens HTTP/1.1" 401 481 "-"
"neutron/12.0.5 keystonemiddleware.auth_token/4.21.0 keystoneauth1/3.4.0 python-requests/
2.18.4 CPython/2.7.12"
192.168.0.1 - - [05/Feb/2019:09:37:11 +0000] "GET /v3/auth/tokens HTTP/1.1" 200 4982 "-"
"python-keystoneclient"
192.168.0.1 - - [05/Feb/2019:09:37:20 +0000] "POST /v3/auth/tokens HTTP/1.1" 401 482 "-"
"neutron/12.0.5 keystonemiddleware.auth_token/4.21.0 keystoneauth1/3.4.0 python-requests/
2.18.4 CPython/2.7.12"
192.168.0.1 - - [05/Feb/2019:09:37:20 +0000] "POST /v3/auth/tokens HTTP/1.1" 401 481 "-"
"neutron/12.0.5 keystonemiddleware.auth_token/4.21.0 keystoneauth1/3.4.0 python-requests/
2.18.4 CPython/2.7.12"

I check the configured password in /etc/neutron and /etc/nova for the [keystone_auth] several times and it's correct!

My configurations files for neutron: /etc/neutron/neutron.conf

  [DEFAULT]
# ...
core_plugin = ml2
service_plugins = router
allow_overlapping_ips = true
transport_url = rabbit://openstack:<passwordRemoved>@ctrl
auth_strategy = keystone
notify_nova_on_port_status_changes = true
notify_nova_on_port_data_changes = true

[keystone_authtoken]
# ...
auth_uri = http://ctrl:5000
auth_url = http://ctrl:5000
memcached_servers = ctrl:11211
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = neutron
password = <passwordRemoved>

[database]
# ...
connection = mysql+pymysql://neutron:<passwordRemoved>@ctrl/neutron

[nova]
# ...
auth_url = http://ctrl:5000
auth_type = password
project_domain_name = default
user_domain_name = default
region_name = RegionOne
project_name = service
username = nova
password = <passwordRemoved>

/etc/neutron/api-paste.ini

[composite:neutron]
use = egg:Paste#urlmap
/: neutronversions_composite
/v2.0: neutronapi_v2_0

[composite:neutronapi_v2_0]
use = call:neutron.auth:pipeline_factory
noauth = cors http_proxy_to_wsgi request_id catch_errors extensions neutronapiapp_v2_0
keystone = cors http_proxy_to_wsgi request_id catch_errors authtoken keystonecontext extensions neutronapiapp_v2_0

[composite:neutronversions_composite]
use = call:neutron.auth:pipeline_factory
noauth = cors http_proxy_to_wsgi neutronversions
keystone = cors http_proxy_to_wsgi neutronversions

[filter:request_id]
paste.filter_factory = oslo_middleware:RequestId.factory

[filter:catch_errors]
paste.filter_factory = oslo_middleware:CatchErrors.factory

[filter:cors]
paste.filter_factory = oslo_middleware.cors:filter_factory
oslo_config_project = neutron

[filter:http_proxy_to_wsgi]
paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory

[filter:keystonecontext]
paste.filter_factory = neutron.auth:NeutronKeystoneContext.factory

[filter:authtoken]
paste.filter_factory = keystonemiddleware.auth_token:filter_factory

auth_plugin = password
auth_url = http://ctrl:5000
username = neutron
password = <passwordRemoved>
project_name = services
user_domain_name = Default
project_domain_name = Default
auth_uri = http://ctrl:5000


[filter:extensions]
paste.filter_factory = neutron.api.extensions:plugin_aware_extension_middleware_factory

[app:neutronversions]
paste.app_factory = neutron.pecan_wsgi.app:versions_factory

[app:neutronapiapp_v2_0]
paste.app_factory = neutron.api.v2.router:APIRouter.factory

[filter:osprofiler]
paste.filter_factory = osprofiler.web:WsgiMiddleware.factory

/etc/neutron/plugins/ml2/ml2_conf.ini

[DEFAULT]
core_plugin = ml2
service_plugins =
transport_url = rabbit://openstack:<passwordRemoved>@ctrl
auth_strategy = keystone
notify_nova_on_port_status_changes = true
notify_nova_on_port_data_changes = true

[database]
# ...
connection = mysql+pymysql://neutron:<passwordRemoved>@ctrl/neutron

[keystone_authtoken]
# ...
auth_uri = http://ctrl:5000
auth_url = http://ctrl:5000
memcached_servers = ctrl:11211
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = neutron
password = <passwordRemoved>

[nova]
# ...
auth_url = http://ctrl:5000
auth_type = password
project_domain_name = default
user_domain_name = default
region_name = RegionOne
project_name = service
username = nova
password = <passwordRemoved>
<---------</snipped>--------->

Status for Neutron-server and rabbitmq-server:

*******@ctrl:~$ sudo service neutron-server status
● neutron-server.service - OpenStack Neutron Server
   Loaded: loaded (/lib/systemd/system/neutron-server.service; enabled; vendor preset: ena
   Active: active (running) since Tue 2019-02-05 08:56:50 UTC; 1h 7min ago
 Main PID: 21070 (neutron-server)
    Tasks: 8
   Memory: 349.7M
      CPU: 20.526s
   CGroup: /system.slice/neutron-server.service
           ├─21070 /usr/bin/python2 /usr/bin/neutron-server --config-file=/etc/neutron/neu
           ├─21085 /usr/bin/python2 /usr/bin/neutron-server --config-file=/etc/neutron/neu
           ├─21086 /usr/bin/python2 /usr/bin/neutron-server --config-file=/etc/neutron/neu
           ├─21087 /usr/bin/python2 /usr/bin/neutron-server --config-file=/etc/neutron/neu
           ├─21088 /usr/bin/python2 /usr/bin/neutron-server --config-file=/etc/neutron/neu
           ├─21089 /usr/bin/python2 /usr/bin/neutron-server --config-file=/etc/neutron/neu
           ├─21090 /usr/bin/python2 /usr/bin/neutron-server --config-file=/etc/neutron/neu
           └─21091 /usr/bin/python2 /usr/bin/neutron-server --config-file=/etc/neutron/neu

Feb 05 08:56:50 ctrl systemd[1]: Started OpenStack Neutron Server.

******@ctrl:~$ sudo service rabbitmq-server status
● rabbitmq-server.service - RabbitMQ Messaging Server
   Loaded: loaded (/lib/systemd/system/rabbitmq-server.service; enabled; vendor preset: en
   Active: active (running) since Fri 2019-02-01 12:20:31 UTC; 3 days ago
 Main PID: 1304 (rabbitmq-server)
    Tasks: 78
   Memory: 53.1M
      CPU: 54min 47.360s
   CGroup: /system.slice/rabbitmq-server.service
           ├─1304 /bin/sh /usr/sbin/rabbitmq-server
           ├─1337 /bin/sh /usr/lib/rabbitmq/bin/rabbitmq-server
           ├─1717 /usr/lib/erlang/erts-7.3/bin/epmd -daemon
           ├─2219 /usr/lib/erlang/erts-7.3/bin/beam.smp -W w -A 64 -P 1048576 -t 5000000 -
           ├─2482 inet_gethost 4
           └─2483 inet_gethost 4

Feb 01 12:20:04 ctrl systemd[1]: Starting RabbitMQ Messaging Server...
Feb 01 12:20:21 ctrl rabbitmq[1305]: Waiting for rabbit@ctrl
Feb 01 12:20:21 ctrl rabbitmq[1305]: pid is 1337
Feb 01 12:20:31 ctrl systemd[1]: Started RabbitMQ Messaging Server.

Could someone please send me some Feedback what else i clould check or if i missed some configuration parameters? thanks in advance and best regards Luke

UPDATE: debug output from openstack command.

norbert@ctrl:~$ openstack network list --debug
START with options: network list --debug
options: Namespace(access_key='', access_secret='***', access_token='***', access_token_endpoint='', access_token_type='', application_credential_id='', application_credential_name='', application_credential_secret='***', auth_type='', auth_url='http://ctrl:5000/v3', cacert=None, cert='', client_id='', client_secret='***', cloud='', code='', consumer_key='', consumer_secret='***', debug=True, default_domain='default', default_domain_id='', default_domain_name='', deferred_help=False, discovery_endpoint='', domain_id='', domain_name='', endpoint='', identity_provider='', identity_provider_url='', insecure=None, interface='public', key='', log_file=None, openid_scope='', os_beta_command=False, os_compute_api_version='', os_dns_api_version='2', os_identity_api_version='3', os_image_api_version='2', os_key_manager_api_version='1', os_network_api_version='', os_object_api_version='', os_orchestration_api_version='1', os_project_id=None, os_project_name=None, os_volume_api_version='', passcode='', password='***', profile='', project_domain_id='', project_domain_name='Default', project_id='', project_name='admin', protocol='', redirect_uri='', region_name='RegionOne', remote_project_domain_id='', remote_project_domain_name='', remote_project_id='', remote_project_name='', service_provider='', service_provider_endpoint='', service_provider_entity_id='', system_scope='', timing=False, token='***', trust_id='', url='', user_domain_id='', user_domain_name='Default', user_id='', username='admin', verbose_level=3, verify=None)
Auth plugin password selected
auth_config_hook(): {'auth_type': 'password', 'beta_command': False, 'orchestration_api_version': '1', 'timing': False, 'auth_url': 'http://ctrl:5000/v3', u'network_api_version': u'2', u'message': u'', u'image_format': u'qcow2', 'networks': [], 'image_api_version': '2', 'verify': True, 'dns_api_version': '2', u'object_store_api_version': u'1', u'status': u'active', 'verbose_level': 3, 'region_name': 'RegionOne', 'api_timeout': None, 'auth': {'user_domain_name': 'Default', 'project_name': 'admin', 'project_domain_name': 'Default'}, 'default_domain': 'default', u'image_api_use_tasks': False, u'floating_ip_source': u'neutron', 'key': None, 'cacert': None, 'password': '***', 'key_manager_api_version': '1', u'baremetal_status_code_retries': '5', 'identity_api_version': '3', 'username': 'admin', 'deferred_help': False, 'cert': None, u'secgroup_source': u'neutron', 'debug': True, u'interface': 'public', u'disable_vendor_agent': {}}
defaults: {u'auth_type': 'password', u'status': u'active', 'api_timeout': None, 'cacert': None, u'image_api_use_tasks': False, u'floating_ip_source': u'neutron', 'key': None, u'interface': u'public', u'network_api_version': u'2', u'message': u'', u'image_format': u'qcow2', u'baremetal_status_code_retries': 5, 'verify': True, 'cert': None, u'secgroup_source': u'neutron', u'object_store_api_version': u'1', u'disable_vendor_agent': {}}
cloud cfg: {'auth_type': 'password', 'beta_command': False, 'orchestration_api_version': '1', 'cacert': None, 'auth_url': 'http://ctrl:5000/v3', u'network_api_version': u'2', u'message': u'', u'image_format': u'qcow2', 'networks': [], 'image_api_version': '2', 'verify': True, 'dns_api_version': '2', u'object_store_api_version': u'1', u'status': u'active', 'verbose_level': 3, 'region_name': 'RegionOne', 'api_timeout': None, 'auth': {'user_domain_name': 'Default', 'project_name': 'admin', 'project_domain_name': 'Default'}, 'default_domain': 'default', u'image_api_use_tasks': False, u'floating_ip_source': u'neutron', 'key': None, 'timing': False, 'password': '***', 'key_manager_api_version': '1', u'baremetal_status_code_retries': '5', 'identity_api_version': '3', 'username': 'admin', 'deferred_help': False, 'cert': None, u'secgroup_source': u'neutron', 'debug': True, u'interface': 'public', u'disable_vendor_agent': {}}
compute API version 2.1, cmd group openstack.compute.v2
network API version 2, cmd group openstack.network.v2
image API version 2, cmd group openstack.image.v2
volume API version 2, cmd group openstack.volume.v2
identity API version 3, cmd group openstack.identity.v3
object_store API version 1, cmd group openstack.object_store.v1
neutronclient API version 2, cmd group openstack.neutronclient.v2
orchestration API version 1, cmd group openstack.orchestration.v1
dns API version 2, cmd group openstack.dns.v2
key_manager API version 1, cmd group openstack.key_manager.v1
Auth plugin password selected
auth_config_hook(): {'auth_type': 'password', 'beta_command': False, 'orchestration_api_version': '1', 'timing': False, 'auth_url': 'http://ctrl:5000/v3', u'network_api_version': u'2', u'message': u'', u'image_format': u'qcow2', 'networks': [], 'image_api_version': '2', 'verify': True, 'dns_api_version': '2', u'object_store_api_version': u'1', u'status': u'active', 'verbose_level': 3, 'region_name': 'RegionOne', 'api_timeout': None, 'auth': {'user_domain_name': 'Default', 'project_name': 'admin', 'project_domain_name': 'Default'}, 'default_domain': 'default', u'image_api_use_tasks': False, u'floating_ip_source': u'neutron', 'key': None, 'cacert': None, 'password': '***', 'key_manager_api_version': '1', u'baremetal_status_code_retries': '5', 'identity_api_version': '3', 'username': 'admin', 'deferred_help': False, 'cert': None, u'secgroup_source': u'neutron', 'debug': True, u'interface': 'public', u'disable_vendor_agent': {}}
Auth plugin password selected
auth_config_hook(): {'auth_type': 'password', 'beta_command': False, 'orchestration_api_version': '1', 'timing': False, 'auth_url': 'http://ctrl:5000/v3', u'network_api_version': u'2', u'message': u'', u'image_format': u'qcow2', 'networks': [], 'image_api_version': '2', 'verify': True, 'dns_api_version': '2', u'object_store_api_version': u'1', u'status': u'active', 'verbose_level': 3, 'region_name': 'RegionOne', 'api_timeout': None, 'auth': {'user_domain_name': 'Default', 'project_name': 'admin', 'project_domain_name': 'Default'}, 'default_domain': 'default', u'image_api_use_tasks': False, u'floating_ip_source': u'neutron', 'key': None, 'cacert': None, 'password': '***', 'key_manager_api_version': '1', u'baremetal_status_code_retries': '5', 'identity_api_version': '3', 'username': 'admin', 'deferred_help': False, 'cert': None, u'secgroup_source': u'neutron', 'debug': True, u'interface': 'public', u'disable_vendor_agent': {}}
command: network list -> openstackclient.network.v2.network.ListNetwork (auth=True)
Auth plugin password selected
auth_config_hook(): {'auth_type': 'password', 'beta_command': False, 'key': None, 'timing': False, 'auth_url': 'http://ctrl:5000/v3', 'additional_user_agent': [('osc-lib', '1.12.0')], u'network_api_version': u'2', u'message': u'', u'image_format': u'qcow2', 'networks': [], 'image_api_version': '2', 'verify': True, 'dns_api_version': '2', u'object_store_api_version': u'1', u'status': u'active', 'verbose_level': 3, 'region_name': 'RegionOne', 'api_timeout': None, 'auth': {'user_domain_name': 'Default', 'project_name': 'admin', 'project_domain_name': 'Default'}, 'default_domain': 'default', u'image_api_use_tasks': False, u'floating_ip_source': u'neutron', 'orchestration_api_version': '1', 'cacert': None, 'password': '***', 'key_manager_api_version': '1', u'baremetal_status_code_retries': '5', 'identity_api_version': '3', 'username': 'admin', 'deferred_help': False, 'cert': None, u'secgroup_source': u'neutron', 'debug': True, u'interface': 'public', u'disable_vendor_agent': {}}
Using auth plugin: password
Using parameters {'username': 'admin', 'project_name': 'admin', 'user_domain_name': 'Default', 'auth_url': 'http://ctrl:5000/v3', 'password': '***', 'project_domain_name': 'Default'}
Get auth_ref
REQ: curl -g -i -X GET http://ctrl:5000/v3 -H "Accept: application/json" -H "User-Agent: openstacksdk/0.23.0 keystoneauth1/3.11.2 python-requests/2.21.0 CPython/2.7.12"
Starting new HTTP connection (1): ctrl:5000
http://ctrl:5000 "GET /v3 HTTP/1.1" 200 244
RESP: [200] Connection: Keep-Alive Content-Length: 244 Content-Type: application/json Date: Tue, 05 Feb 2019 11:01:00 GMT Keep-Alive: timeout=5, max=100 Server: Apache/2.4.18 (Ubuntu) Vary: X-Auth-Token X-Distribution: Ubuntu x-openstack-request-id: req-6e6f0ceb-f2e2-46ae-86fe-76c8643aa840
RESP BODY: {"version": {"status": "stable", "updated": "2018-02-28T00:00:00Z", "media-types": [{"base": "application/json", "type": "application/vnd.openstack.identity-v3+json"}], "id": "v3.10", "links": [{"href": "http://ctrl:5000/v3/", "rel": "self"}]}}
GET call to http://ctrl:5000/v3 used request id req-6e6f0ceb-f2e2-46ae-86fe-76c8643aa840
Making authentication request to http://ctrl:5000/v3/auth/tokens
http://ctrl:5000 "POST /v3/auth/tokens HTTP/1.1" 201 4466
{"token": {"is_domain": false, "methods": ["password"], "roles": [{"id": "d6422e5a7cb2413a9aaecc927c0e4d22", "name": "admin"}], "expires_at": "2019-02-05T12:01:00.000000Z", "project": {"domain": {"id": "default", "name": "Default"}, "id": "885ea3ee676b41c6a2ef1d15b76e1ff0", "name": "admin"}, "catalog": [{"endpoints": [{"url": "http://ctrl:9696", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "3aff76a7e7bb4551a9626f7a7fa05282"}, {"url": "http://ctrl:9696", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "43753f1f5e3842bfb94ecfdecdf566bc"}, {"url": "http://ctrl:9696", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "cb818c1377644461a265ccc29162d20b"}], "type": "network", "id": "027c679b8b2c42d8b629f66e50b7613c", "name": "neutron"}, {"endpoints": [{"url": "http://ctrl:5000/v3", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "1be7a91a061f4de9ae30fa3388def9c1"}, {"url": "http://ctrl:5000/v3", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "335d0508149a428e8fb72059624d7979"}, {"url": "http://ctrl:5000/v3", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "ec34a62b98294ac29eac3a5b9f99cce6"}], "type": "identity", "id": "0c1e3afabc054de78482806d8e81f119", "name": "keystone"}, {"endpoints": [{"url": "http://ctrl:8776/v1/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "5793ee84988d4ecd967735a19723bcda"}, {"url": "http://ctrl:8776/v1/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "782c2a2cf62942d1b59fb76830d3e956"}, {"url": "http://ctrl:8776/v1/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "facd596d3e0a4583a3c8e4426a2e8871"}], "type": "volume", "id": "2229f02db26d4ae1b72f29da22033568", "name": "cinder"}, {"endpoints": [{"url": "http://ctrl:8776/v2/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "1ff12d4a8cce4018be3810aea5689892"}, {"url": "http://ctrl:8776/v2/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "2d68f209cb224834982abe7edf7ecc7d"}, {"url": "http://ctrl:8776/v2/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "403673ec1ad34e0a866d901dc1ee387a"}], "type": "volumev2", "id": "3b6b302d0be44be88e17639af3545984", "name": "cinderv2"}, {"endpoints": [{"url": "http://ctrl:8778", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "4a6e8e223b5146c49698a371155da277"}, {"url": "http://ctrl:8778", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "b9de8f53dc20483f949f1633673f7a45"}, {"url": "http://ctrl:8778", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "d053b29d03e247fd92fc241b373d5bc7"}], "type": "placement", "id": "585acd3f0626453fb28d56334baed556", "name": "placement"}, {"endpoints": [{"url": "http://ctrl:9292", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "a80d997f611d4f1b98aafaba35e0772f"}, {"url": "http://ctrl:9292", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "ca3cd0102d8545c89864016c164c7bcd"}, {"url": "http://ctrl:9292", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "e8dfa34db0a34708be8d11a568a1c472"}], "type": "image", "id": "8db306fef474493b9cab2a7cf9f19adf", "name": "glance"}, {"endpoints": [{"url": "http://ctrl:8774/v2.1", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "0c583de9691a4503bcbf48b85fe2f001"}, {"url": "http://ctrl:8774/v2.1", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "5ae617ead5e5411d87e26569d8b5d67a"}, {"url": "http://ctrl:8774/v2.1", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "88664cf572544d2dbde6ab9cffdfc470"}], "type": "compute", "id": "b0f1a120b93e48a2a2354d05d40a0fb6", "name": "nova"}], "user": {"domain": {"id": "default", "name": "Default"}, "password_expires_at": null, "name": "admin", "id": "39491615f4694b0ea8acc728f9d3f10d"}, "audit_ids": ["HlfuDjwzRVWhpDDbPeXDdw"], "issued_at": "2019-02-05T11:01:00.000000Z"}}
get_parser(openstack network list)
common parser: ArgumentParser(prog=u'openstack network list', usage=None, description=u'List networks', version=None, formatter_class=<class 'cliff.command._SmartHelpFormatter'>, conflict_handler='error', add_help=True)
network endpoint in service catalog
run(Namespace(agent_id=None, any_tags=None, columns=[], disable=False, enable=False, external=False, fit_width=False, formatter='table', internal=False, long=False, max_width=0, name=None, no_share=False, noindent=False, not_any_tags=None, not_tags=None, physical_network=None, print_empty=False, project=None, project_domain=None, provider_network_type=None, quote_mode='nonnumeric', segmentation_id=None, share=False, sort_columns=[], status=None, tags=None))
network endpoint in service catalog
Making authentication request to http://ctrl:5000/v3/auth/tokens
http://ctrl:5000 "POST /v3/auth/tokens HTTP/1.1" 201 4466
{"token": {"is_domain": false, "methods": ["password"], "roles": [{"id": "d6422e5a7cb2413a9aaecc927c0e4d22", "name": "admin"}], "expires_at": "2019-02-05T12:01:00.000000Z", "project": {"domain": {"id": "default", "name": "Default"}, "id": "885ea3ee676b41c6a2ef1d15b76e1ff0", "name": "admin"}, "catalog": [{"endpoints": [{"url": "http://ctrl:9696", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "3aff76a7e7bb4551a9626f7a7fa05282"}, {"url": "http://ctrl:9696", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "43753f1f5e3842bfb94ecfdecdf566bc"}, {"url": "http://ctrl:9696", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "cb818c1377644461a265ccc29162d20b"}], "type": "network", "id": "027c679b8b2c42d8b629f66e50b7613c", "name": "neutron"}, {"endpoints": [{"url": "http://ctrl:5000/v3", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "1be7a91a061f4de9ae30fa3388def9c1"}, {"url": "http://ctrl:5000/v3", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "335d0508149a428e8fb72059624d7979"}, {"url": "http://ctrl:5000/v3", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "ec34a62b98294ac29eac3a5b9f99cce6"}], "type": "identity", "id": "0c1e3afabc054de78482806d8e81f119", "name": "keystone"}, {"endpoints": [{"url": "http://ctrl:8776/v1/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "5793ee84988d4ecd967735a19723bcda"}, {"url": "http://ctrl:8776/v1/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "782c2a2cf62942d1b59fb76830d3e956"}, {"url": "http://ctrl:8776/v1/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "facd596d3e0a4583a3c8e4426a2e8871"}], "type": "volume", "id": "2229f02db26d4ae1b72f29da22033568", "name": "cinder"}, {"endpoints": [{"url": "http://ctrl:8776/v2/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "1ff12d4a8cce4018be3810aea5689892"}, {"url": "http://ctrl:8776/v2/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "2d68f209cb224834982abe7edf7ecc7d"}, {"url": "http://ctrl:8776/v2/885ea3ee676b41c6a2ef1d15b76e1ff0", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "403673ec1ad34e0a866d901dc1ee387a"}], "type": "volumev2", "id": "3b6b302d0be44be88e17639af3545984", "name": "cinderv2"}, {"endpoints": [{"url": "http://ctrl:8778", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "4a6e8e223b5146c49698a371155da277"}, {"url": "http://ctrl:8778", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "b9de8f53dc20483f949f1633673f7a45"}, {"url": "http://ctrl:8778", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "d053b29d03e247fd92fc241b373d5bc7"}], "type": "placement", "id": "585acd3f0626453fb28d56334baed556", "name": "placement"}, {"endpoints": [{"url": "http://ctrl:9292", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "a80d997f611d4f1b98aafaba35e0772f"}, {"url": "http://ctrl:9292", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "ca3cd0102d8545c89864016c164c7bcd"}, {"url": "http://ctrl:9292", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "e8dfa34db0a34708be8d11a568a1c472"}], "type": "image", "id": "8db306fef474493b9cab2a7cf9f19adf", "name": "glance"}, {"endpoints": [{"url": "http://ctrl:8774/v2.1", "interface": "admin", "region": "RegionOne", "region_id": "RegionOne", "id": "0c583de9691a4503bcbf48b85fe2f001"}, {"url": "http://ctrl:8774/v2.1", "interface": "internal", "region": "RegionOne", "region_id": "RegionOne", "id": "5ae617ead5e5411d87e26569d8b5d67a"}, {"url": "http://ctrl:8774/v2.1", "interface": "public", "region": "RegionOne", "region_id": "RegionOne", "id": "88664cf572544d2dbde6ab9cffdfc470"}], "type": "compute", "id": "b0f1a120b93e48a2a2354d05d40a0fb6", "name": "nova"}], "user": {"domain": {"id": "default", "name": "Default"}, "password_expires_at": null, "name": "admin", "id": "39491615f4694b0ea8acc728f9d3f10d"}, "audit_ids": ["Hhi1ZFAcRvSbdp_48w4qoQ"], "issued_at": "2019-02-05T11:01:00.000000Z"}}
Network client initialized using OpenStack SDK: <openstack.network.v2._proxy.Proxy object at 0x7fad209cf750>
Instantiating identity client: <class 'keystoneclient.v3.client.Client'>
Manager RegionOne running task network.GET.networks
TaskManager RegionOne queue size: 0)
REQ: curl -g -i -X GET http://ctrl:9696/v2.0/networks -H "Accept: application/json" -H "User-Agent: openstacksdk/0.23.0 keystoneauth1/3.11.2 python-requests/2.21.0 CPython/2.7.12" -H "X-Auth-Token: {SHA256}b3c34f4553ab4d7a94475d2e42560b75e0d01c48d43c68affbaa327ceaa9c4cc"
Manager RegionOne ran task network.GET.networks in 0.00100302696228s
Starting new HTTP connection (1): ctrl:9696
http://ctrl:9696 "GET /v2.0/networks HTTP/1.1" 503 170
RESP: [503] Connection: keep-alive Content-Length: 170 Content-Type: application/json Date: Tue, 05 Feb 2019 11:01:01 GMT X-Openstack-Request-Id: req-99e2c9c5-9c02-48a9-bf44-64df2ab8801a
RESP BODY: {"message": "The server is currently unavailable. Please try again at a later time.<br /><br />\n\n\n", "code": "503 Service Unavailable", "title": "Service Unavailable"}
GET call to network for http://ctrl:9696/v2.0/networks used request id req-99e2c9c5-9c02-48a9-bf44-64df2ab8801a
HttpException: 503: Server Error for url: http://ctrl:9696/v2.0/networks, Service Unavailable
Traceback (most recent call last):
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/app.py", line 401, in run_subcommand
    result = cmd.run(parsed_args)
  File "/home/norbert/.local/lib/python2.7/site-packages/osc_lib/command/command.py", line 41, in run
    return super(Command, self).run(parsed_args)
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/display.py", line 119, in run
    self.produce_output(parsed_args, column_names, data)
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/lister.py", line 82, in produce_output
    parsed_args,
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/formatters/table.py", line 101, in emit_list
    self.add_rows(x, column_names, data)
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/formatters/table.py", line 80, in add_rows
    first_row = next(data_iter)
  File "/home/norbert/.local/lib/python2.7/site-packages/openstackclient/network/v2/network.py", line 571, in <genexpr>
    (utils.get_item_properties(
  File "/home/norbert/.local/lib/python2.7/site-packages/openstack/resource.py", line 1317, in list
    exceptions.raise_from_response(response)
  File "/home/norbert/.local/lib/python2.7/site-packages/openstack/exceptions.py", line 229, in raise_from_response
    http_status=http_status, request_id=request_id
HttpException: HttpException: 503: Server Error for url: http://ctrl:9696/v2.0/networks, Service Unavailable
clean_up ListNetwork: HttpException: 503: Server Error for url: http://ctrl:9696/v2.0/networks, Service Unavailable
Traceback (most recent call last):
  File "/home/norbert/.local/lib/python2.7/site-packages/osc_lib/shell.py", line 136, in run
    ret_val = super(OpenStackShell, self).run(argv)
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/app.py", line 281, in run
    result = self.run_subcommand(remainder)
  File "/home/norbert/.local/lib/python2.7/site-packages/osc_lib/shell.py", line 176, in run_subcommand
    ret_value = super(OpenStackShell, self).run_subcommand(argv)
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/app.py", line 401, in run_subcommand
    result = cmd.run(parsed_args)
  File "/home/norbert/.local/lib/python2.7/site-packages/osc_lib/command/command.py", line 41, in run
    return super(Command, self).run(parsed_args)
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/display.py", line 119, in run
    self.produce_output(parsed_args, column_names, data)
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/lister.py", line 82, in produce_output
    parsed_args,
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/formatters/table.py", line 101, in emit_list
    self.add_rows(x, column_names, data)
  File "/home/norbert/.local/lib/python2.7/site-packages/cliff/formatters/table.py", line 80, in add_rows
    first_row = next(data_iter)
  File "/home/norbert/.local/lib/python2.7/site-packages/openstackclient/network/v2/network.py", line 571, in <genexpr>
    (utils.get_item_properties(
  File "/home/norbert/.local/lib/python2.7/site-packages/openstack/resource.py", line 1317, in list
    exceptions.raise_from_response(response)
  File "/home/norbert/.local/lib/python2.7/site-packages/openstack/exceptions.py", line 229, in raise_from_response
    http_status=http_status, request_id=request_id
HttpException: HttpException: 503: Server Error for url: http://ctrl:9696/v2.0/networks, Service Unavailable

END return value: 1