Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

"Deadlock found" and "DuplicateEntry" errors when trying to attach a neutron L3 HA router to an internal network

On a setup with 2 neutron nodes, L3 agent in HA (VRRP), we:

  • create the external network (vlan)
  • create an internal network (vxlan)
  • create a router
  • attach the router to the external network
  • attach the router to the internal network

After attaching the router to the internal network, the following happens:

a) some router _and_ dhcp ports of the external networks go from ACTIVE to BUILD status b) dhcp ports of the internal networks go from ACTIVE to BUILD c) two type of errors are found on the neutron log file: - a Deadlock found when trying to get a lock error, on one of the network nodes - multiple Duplicate entry ... for key PRIMARY on both network nodes

How we replicated the issue

root@kenny:~# neutron  net-create uzh-only \
  --shared --provider:network_type vlan \
  --provider:segmentation_id 842 \
  --provider:physical_network vlannet \
  --router:external 

root@kenny:~# neutron  subnet-create uzh-only \
    --name vlan842 \
    172.23.0.0/16 \
    --gateway 172.23.0.1 \
    --allocation-pool start=172.23.2.0,end=172.23.200.255 \
    --enable-dhcp \
    --dns-nameservers 130.60.128.3 130.60.64.51 \
    --   --host-routes type=dict list=true \
    destination=130.60.206.69/32,nexthop=172.23.0.5

root@kenny:~# neutron net-create test 
root@kenny:~# neutron subnet-create test --name subtest 10.0.0.0/24 
root@kenny:~# neutron router-create test
root@kenny:~# neutron router-gateway-set test uzh-only

root@kenny:~# neutron port-list -c id -c fixed_ips -c status
+--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
| id                                   | fixed_ips                                                                            | status | device_owner                |
+--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
| 1463b78c-f640-4e41-8831-927679be380b | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.3"}      | ACTIVE | network:dhcp                |
| 1e7a0ac2-268c-48b7-9c70-e7e03126d9ed | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.1"} | ACTIVE | network:router_ha_interface |
| 234ddb9b-16cc-48f2-b646-fee5f6d5def2 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.2"}    | BUILD  | network:router_gateway      |
| 3e755174-14b3-48e9-af84-489cce8985be | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.2"}      | ACTIVE | network:dhcp                |
| 4df3d3ab-1967-4bf1-bc3f-31ec1addd8d1 | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.1"}      | BUILD  | network:router_interface    |
| 5a2e6cbd-4c69-47e0-bfa6-92fe6242aba4 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.0"}    | ACTIVE | network:dhcp                |
| 881882f0-4ad7-4294-bec4-93f869cb40d8 | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.2"} | ACTIVE | network:router_ha_interface |
| ac0e1761-335e-431f-ae89-a2703f2bafa5 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.1"}    | ACTIVE | network:dhcp                |
+--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+

root@kenny:~# neutron router-interface-add test subtest

root@kenny:~# neutron port-list -c id -c fixed_ips -c status

root@kenny:~# neutron port-list -c id -c fixed_ips -c status -c device_owner
+--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
| id                                   | fixed_ips                                                                            | status | device_owner                |
+--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
| 1463b78c-f640-4e41-8831-927679be380b | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.3"}      | BUILD  | network:dhcp                |
| 1e7a0ac2-268c-48b7-9c70-e7e03126d9ed | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.1"} | ACTIVE | network:router_ha_interface |
| 234ddb9b-16cc-48f2-b646-fee5f6d5def2 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.2"}    | BUILD  | network:router_gateway      |
| 3e755174-14b3-48e9-af84-489cce8985be | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.2"}      | ACTIVE | network:dhcp                |
| 4df3d3ab-1967-4bf1-bc3f-31ec1addd8d1 | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.1"}      | BUILD  | network:router_interface    |
| 5a2e6cbd-4c69-47e0-bfa6-92fe6242aba4 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.0"}    | ACTIVE | network:dhcp                |
| 881882f0-4ad7-4294-bec4-93f869cb40d8 | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.2"} | BUILD  | network:router_ha_interface |
| ac0e1761-335e-431f-ae89-a2703f2bafa5 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.1"}    | BUILD  | network:dhcp                |
+--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+

excerpt from the logs on controller node A (only WARNING, ERROR and TRACE)

2015-07-30 15:50:59.322 251400 WARNING neutron.api.rpc.handlers.dhcp_rpc [req-203d6d37-4643-46e1-baa4-eb58c5f45750 ] Action create_port for network 9f7e9af9-72fc-44f1-89e3-be5cdf3d5bc5 could not complete successfully: (OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction') 'INSERT INTO ml2_port_binding_levels (port_id, host, level, driver, segment_id) VALUES (%s, %s, %s, %s, %s)' ('3e755174-14b3-48e9-af84-489cce8985be', 'cloud-l7-41', 0, 'openvswitch', '9a00e2d3-d842-4517-bec2-e3296ae711a5')
2015-07-30 15:51:46.687 228137 ERROR neutron.plugins.openvswitch.agent.ovs_neutron_agent [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] Error while processing VIF ports
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Traceback (most recent call last):
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1586, in rpc_loop
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     ovs_restarted)
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1362, in process_network_ports
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     devices_added_updated, ovs_restarted)
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1267, in treat_devices_added_or_updated
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     self.context, device, self.agent_id, cfg.CONF.host)
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/rpc.py", line 128, in update_device_up
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     agent_id=agent_id, host=host)
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/client.py", line 156, in call
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     retry=self.retry)
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/transport.py", line 90, in _send
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     timeout=timeout, retry=retry)
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 350, in send
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     retry=retry)
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 341, in _send
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     raise result
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent RemoteError: Remote error: DBDuplicateEntry (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2')
2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent [u'Traceback (most recent call last):\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 142, in _dispatch_and_reply\n    executor_callback))\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 186, in _dispatch\n    executor_callback)\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 130, in _do_dispatch\n    result = func(ctxt, **new_args)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/rpc.py", line 186, in update_device_up\n    host)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1373, in update_port_status\n    original_port[\'network_id\'])\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 658, in get_network\n    result = super(Ml2Plugin, self).get_network(context, id, None)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 1007, in get_network\n    network = self._get_network(context, id)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 91, in _get_network\n    network = self._get_by_id(context, models_v2.Network, id)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/db/common_db_mixin.py", line 130, in _get_by_id\n    return query.filter(model.id == id).one()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2395, in one\n    ret = list(self)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2437, in __iter__\n    self.session._autoflush()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1198, in _autoflush\n    self.flush()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush\n    self._flush(objects)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush\n    transaction.rollback(_capture_exception=True)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__\n    compat.reraise(exc_type, exc_value, exc_tb)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush\n    flush_context.execute()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute\n    rec.execute(self)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute\n    uow\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 60, in save_obj\n    mapper, table, update)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 518, in _emit_update_statements\n    execute(statement, params)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute\n    return meth(self, multiparams, params)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection\n    return connection._execute_clauseelement(self, multiparams, params)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement\n    compiled_sql, distilled_params\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context\n    context)\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception\n    e, statement, parameters, cursor, context)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1155, in _handle_dbapi_exception\n    util.raise_from_cause(newraise, exc_info)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause\n    reraise(type(exception), exception, tb=exc_tb)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context\n    context)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute\n    cursor.execute(statement, parameters)\n', u'  File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute\n    self.errorhandler(self, exc, value)\n', u'  File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler\n    raise errorclass, errorvalue\n', u'DBDuplicateEntry: (IntegrityError) (1062, "Duplicate entry \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\' for key \'PRIMARY\'") \'UPDATE ports SET status=%s WHERE ports.id = %s\' (\'ACTIVE\', \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\')\n'].

excerpt from the logs on controller node B (only WARNING, ERROR and TRACE)

2015-07-30 15:51:46.680 55894 ERROR oslo_messaging.rpc.dispatcher [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] Exception during message handling: (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2')
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher Traceback (most recent call last):
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 142, in _dispatch_and_reply
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     executor_callback))
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 186, in _dispatch
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     executor_callback)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 130, in _do_dispatch
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     result = func(ctxt, **new_args)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/rpc.py", line 186, in update_device_up
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     host)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1373, in update_port_status
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     original_port['network_id'])
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 658, in get_network
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     result = super(Ml2Plugin, self).get_network(context, id, None)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 1007, in get_network
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     network = self._get_network(context, id)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 91, in _get_network
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     network = self._get_by_id(context, models_v2.Network, id)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/db/common_db_mixin.py", line 130, in _get_by_id
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     return query.filter(model.id == id).one()
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2395, in one
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     ret = list(self)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2437, in __iter__
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self.session._autoflush()
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1198, in _autoflush
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self.flush()
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self._flush(objects)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     transaction.rollback(_capture_exception=True)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     compat.reraise(exc_type, exc_value, exc_tb)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     flush_context.execute()
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     rec.execute(self)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     uow
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 60, in save_obj
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     mapper, table, update)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 518, in _emit_update_statements
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     execute(statement, params)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     return meth(self, multiparams, params)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     return connection._execute_clauseelement(self, multiparams, params)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     compiled_sql, distilled_params
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     context)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     e, statement, parameters, cursor, context)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1155, in _handle_dbapi_exception
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     util.raise_from_cause(newraise, exc_info)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     reraise(type(exception), exception, tb=exc_tb)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     context)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     cursor.execute(statement, parameters)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self.errorhandler(self, exc, value)
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     raise errorclass, errorvalue
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher DBDuplicateEntry: (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2')
2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher 
2015-07-30 15:51:46.682 55894 ERROR oslo_messaging._drivers.common [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] Returning exception (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2') to caller
2015-07-30 15:51:46.682 55894 ERROR oslo_messaging._drivers.common [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] ['Traceback (most recent call last):\n', '  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 142, in _dispatch_and_reply\n    executor_callback))\n', '  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 186, in _dispatch\n    executor_callback)\n', '  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 130, in _do_dispatch\n    result = func(ctxt, **new_args)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/rpc.py", line 186, in update_device_up\n    host)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1373, in update_port_status\n    original_port[\'network_id\'])\n', '  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 658, in get_network\n    result = super(Ml2Plugin, self).get_network(context, id, None)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 1007, in get_network\n    network = self._get_network(context, id)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 91, in _get_network\n    network = self._get_by_id(context, models_v2.Network, id)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/db/common_db_mixin.py", line 130, in _get_by_id\n    return query.filter(model.id == id).one()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2395, in one\n    ret = list(self)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2437, in __iter__\n    self.session._autoflush()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1198, in _autoflush\n    self.flush()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush\n    self._flush(objects)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush\n    transaction.rollback(_capture_exception=True)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__\n    compat.reraise(exc_type, exc_value, exc_tb)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush\n    flush_context.execute()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute\n    rec.execute(self)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute\n    uow\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 60, in save_obj\n    mapper, table, update)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 518, in _emit_update_statements\n    execute(statement, params)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute\n    return meth(self, multiparams, params)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection\n    return connection._execute_clauseelement(self, multiparams, params)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement\n    compiled_sql, distilled_params\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context\n    context)\n', '  File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception\n    e, statement, parameters, cursor, context)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1155, in _handle_dbapi_exception\n    util.raise_from_cause(newraise, exc_info)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause\n    reraise(type(exception), exception, tb=exc_tb)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context\n    context)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute\n    cursor.execute(statement, parameters)\n', '  File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute\n    self.errorhandler(self, exc, value)\n', '  File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler\n    raise errorclass, errorvalue\n', 'DBDuplicateEntry: (IntegrityError) (1062, "Duplicate entry \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\' for key \'PRIMARY\'") \'UPDATE ports SET status=%s WHERE ports.id = %s\' (\'ACTIVE\', \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\')\n']

Some more information about our setup:

  • 3 nova/glance/cinder/keystone servers, API endpoints balanced using HAProxy
  • 3 MySQL Galera cluster, balanced using HAProxy, but in active/passive configuration (2 over 3 servers as backup in haproxy.cfg)
  • 3 RabbitMQ cluster
  • All nodes running Ubuntu Trusty and Kilo 2015.1.0 from ubuntu-cloud.archive.canonical.com

neutron.conf file:

[DEFAULT]
verbose = True
use_syslog = True
bind_host = cloud-k4-31.os.s3it.uzh.ch
core_plugin = ml2
service_plugins = router
allow_overlapping_ips = True
advertise_mtu = True
dhcp_agents_per_network = 2
l3_ha = True
max_l3_agents_per_router = 0
api_workers = 20
rpc_workers = 20
notify_nova_on_port_status_changes = True
notify_nova_on_port_data_changes = True
nova_url = http://10.129.17.1:8774/v2
nova_admin_username = tkunsz
nova_admin_tenant_name = service
nova_admin_password = 9.gXd6MpRH
nova_admin_auth_url = http://130.60.24.21:35357/v2.0
amqp_durable_queues=false
rabbit_hosts = cloud-l2-41.os.s3it.uzh.ch,cloud-l6-39.os.s3it.uzh.ch,cloud-k3-39.os.s3it.uzh.ch
rabbit_ha_queues=True
notification_driver = neutron.openstack.common.notifier.rpc_notifier
[matchmaker_redis]
[matchmaker_ring]
[quotas]
[agent]
root_helper = sudo
[keystone_authtoken]
identity_uri = http://130.60.24.21:35357
admin_tenant_name = service
admin_user = tkunsz
admin_password = 9.gXd6MpRH
auth_uri = http://130.60.24.21:5000/v2.0
[database]
connection=mysql://neutron:neutronqwerty@sc-ha.os.s3it.uzh.ch/neutron
max_pool_size = 50
max_overflow = 100
[nova]
region_name = RegionOne
[oslo_concurrency]
lock_path = $state_path/lock
[oslo_policy]
[oslo_messaging_amqp]
[oslo_messaging_qpid]
[oslo_messaging_rabbit]
amqp_auto_delete = True
rabbit_hosts = cloud-l2-41.os.s3it.uzh.ch,cloud-l6-39.os.s3it.uzh.ch,cloud-k3-39.os.s3it.uzh.ch
rabbit_userid=openstack
rabbit_password=rabbitqwerty
rabbit_ha_queues = True

l3_agent.ini file:

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
use_namespaces = True
external_network_bridge =
agent_mode = legacy

dhcp_agent.ini file:

[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
use_namespaces = True
enable_isolated_metadata = True

ml2_conf.ini file:

[ml2]
type_drivers = gre,vlan,vxlan
tenant_network_types = vxlan,gre
mechanism_drivers = openvswitch
physical_network_mtus = vlannet:9000
[ml2_type_flat]
[ml2_type_vlan]
network_vlan_ranges = vlannet:1:4000
[ml2_type_gre]
tunnel_id_ranges = 1:1000
[ml2_type_vxlan]
vni_ranges = 65537:69999
[securitygroup]
enable_security_group = True
enable_ipset = True
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
[ovs]
local_ip = 10.129.31.231
bridge_mappings = vlannet:br-vlan
enable_tunneling = True
tunnel_type = vxlan
tunnel_types = vxlan,gre
vxlan_udp_port = 4789
[agent]
tunnel_types = vxlan,gre
l2_population = False

P.S. I've also opened a bug on launchpad but I'm not sure if it's a bug or a configuration issue: https://bugs.launchpad.net/neutron/+bug/1479818