Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

DCHPOFFER not leaving neutron network node

I have setup openstack on 3 nodes as follows:

  • 10.2.4.129: controller
  • 10.2.4.122: neutron network node (compute-122)
  • 10.2.4.125: compute node (compute-125)
root@compute-122:~# neutron agent-list
+--------------------------------------+--------------------+-------------+-------+----------------+
| id                                   | agent_type         | host        | alive | admin_state_up |
+--------------------------------------+--------------------+-------------+-------+----------------+
| 3c3d9e60-6b7b-4dfc-962b-cf818a6559e8 | DHCP agent         | compute-122 | :-)   | True           |
| 5bda115f-8f78-493f-a9d1-c8eb3e3e843d | Open vSwitch agent | compute-125 | :-)   | True           |
| 8fffa808-acd5-4bac-9ba1-fdb10db785c5 | L3 agent           | compute-122 | :-)   | True           |
| 970003e7-b0f7-4513-bf79-3e406a3a91d1 | Metadata agent     | compute-122 | :-)   | True           |
| 9b4b4e8f-d0d2-413d-b594-7456a68f3d6c | Open vSwitch agent | compute-122 | :-)   | True           |
+--------------------------------------+--------------------+-------------+-------+----------------+

When I try spawning a VM, it doesn't get an ip. So, I traced the DHCP/BOOTP packet and saw that it was reaching the dnsmasq service on the network node.

root@compute-122:~# strace -p 28005 -e network,write -s 4096

recvmsg(4, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"D\0\0\0\24\0\2\0\312h\1\0cm\0\0\2\10\200\376\1\0\0\0\10\0\1\0\177\0\0\1\10\0\2\0\ 177\0\0\1\7\0\3\0lo\0\0\24\0\6\0\377\377\377\377\377\377\377\377l\31\33\0l\31\33\0X\0\0\0\24\0\2\0\312h\1\0cm\0\0\2\30\200\0\17\0\0\0\10\0\1\0\300\250\1e\10\0\2\0\300\2 50\1e\10\0\4\0\300\250\1\377\23\0\3\0tap25c0651b-89\0\0\24\0\6\0\377\377\377\377\377\377\377\377\215\31\33\0\215\31\33\0X\0\0\0\24\0\2\0\312h\1\0cm\0\0\2\20\200\0\17\0\ 0\0\10\0\1\0\251\376\251\376\10\0\2\0\251\376\251\376\10\0\4\0\251\376\377\377\23\0\3\0tap25c0651b-89\0\0\24\0\6\0\377\377\377\377\377\377\377\377\311\334\"\0\311\334\" \0", 244}], msg_controllen=0, msg_flags=0}, 0) = 244 recvmsg(4, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\3\0\2\0\312h\1\0cm\0\0\0\0\0\0", 244}], msg_controllen=0, msg_flags=0}, MSG_PEEK|MSG_TRUNC) = 20 recvmsg(4, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\3\0\2\0\312h\1\0cm\0\0\0\0\0\0", 244}], msg_controllen=0, msg_flags=0}, 0) = 20 write(13, "<30>Sep 29 19:08:45 dnsmasq-dhcp[28005]: DHCPDISCOVER(tap25c0651b-89) fa:16:3e:f0:45:cf ", 88) = 88 write(13, "<30>Sep 29 19:08:45 dnsmasq-dhcp[28005]: DHCPOFFER(tap25c0651b-89) 192.168.1.100 fa:16:3e:f0:45:cf ", 99) = 99 sendmsg(3, {msg_name(16)={sa_family=AF_INET, sin_port=htons(68), sin_addr=inet_addr("192.168.1.100")}, msg_iov(1)=[{"\2\1\6\0\351\232@\21\352B\0\0\0\0\0\0\300\250\1d\300\250\1e\0\0\0\0\372\26>\360E\317\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ 0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ 0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0c\202Sc5\1\0026\4\300\250\1e3\4\0\1Q\200:\4\0\0\250\300;\4\0\1'P\1\ 4\377\377\377\0\34\4\300\250\1\377\17\16openstacklocal\f\22host-192-168-1-100\3\4\300\250\1\1\6\f\n\4\3\336\10\10\4\4\10\10\10\10\32\2\5\256\377", 340}], msg_controllen =0, msg_flags=0}, 0) = 340

It seems to be sending a DCHPOFFER to the tap interface tap25c0651b-89. And when I tcpdump that tap interface...

root@compute-122:~# ip netns exec qdhcp-d82d7270-f590-45bd-a4f4-380af49382de tcpdump -n -le -i tap25c0651b-89

19:12:29.591008 fa:16:3e:f0:45:cf > ff:ff:ff:ff:ff:ff, ethertype IPv4 (0x0800), length 322: 0.0.0.0.68 > 255.255.255.255.67: BOOTP/DHCP, Request from fa:16:3e:f0:45:cf, length 280
19:12:29.591231 fa:16:3e:a9:74:82 > fa:16:3e:f0:45:cf, ethertype IPv4 (0x0800), length 382: 192.168.1.101.67 > 192.168.1.100.68: BOOTP/DHCP, Reply, length 340
19:12:30.592498 fa:16:3e:f0:45:cf > ff:ff:ff:ff:ff:ff, ethertype IPv4 (0x0800), length 322: 0.0.0.0.68 > 255.255.255.255.67: BOOTP/DHCP, Request from fa:16:3e:f0:45:cf, length 280
19:12:30.592736 fa:16:3e:a9:74:82 > fa:16:3e:f0:45:cf, ethertype IPv4 (0x0800), length 382: 192.168.1.101.67 > 192.168.1.100.68: BOOTP/DHCP, Reply, length 340

The bridges are setup as follows:

root@compute-122:~# ovs-vsctl show
ef5cb751-fafc-4797-8a9b-8c265568d466
    Bridge br-int
        fail_mode: secure
        Port "tap25c0651b-89"
            tag: 4
            Interface "tap25c0651b-89"
                type: internal
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port br-int
            Interface br-int
                type: internal
        Port "tape737b64e-46"
            tag: 3
            Interface "tape737b64e-46"
                type: internal
    Bridge br-tun
        Port "gre-0a02047d"
            Interface "gre-0a02047d"
                type: gre
                options: {in_key=flow, local_ip="10.2.4.122", out_key=flow, remote_ip="10.2.4.125"}
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port br-tun
            Interface br-tun
                type: internal
        Port "gre-0a020481"
            Interface "gre-0a020481"
                type: gre
                options: {in_key=flow, local_ip="10.2.4.122", out_key=flow, remote_ip="10.2.4.129"}
        Port "gre-0a020478"
            Interface "gre-0a020478"
                type: gre
                options: {in_key=flow, local_ip="10.2.4.122", out_key=flow, remote_ip="10.2.4.120"}
        Port "gre-0a02047e"
            Interface "gre-0a02047e"
                type: gre
                options: {in_key=flow, local_ip="10.2.4.122", out_key=flow, remote_ip="10.2.4.126"}
        Port "gre-0a020479"
            Interface "gre-0a020479"
                type: gre
                options: {in_key=flow, local_ip="10.2.4.122", out_key=flow, remote_ip="10.2.4.121"}
        Port "gre-0a02047b"
            Interface "gre-0a02047b"
                type: gre
                options: {in_key=flow, local_ip="10.2.4.122", out_key=flow, remote_ip="10.2.4.123"}
    Bridge br-ex
        Port "eth0"
            Interface "eth0"
        Port br-ex
            Interface br-ex
                type: internal
    ovs_version: "2.0.2"

The packet reaches the br-int bridge too

root@compute-122:~# tcpdump -nle -i br-int
tcpdump: WARNING: br-int: no IPv4 address assigned
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on br-int, link-type EN10MB (Ethernet), capture size 65535 bytes
19:14:55.835951 fa:16:3e:f0:45:cf > ff:ff:ff:ff:ff:ff, ethertype 802.1Q (0x8100), length 326: vlan 4, p 0, ethertype IPv4, 0.0.0.0.68 > 255.255.255.255.67: BOOTP/DHCP, Request from fa:16:3e:f0:45:cf, length 280
19:14:57.839656 fa:16:3e:f0:45:cf > ff:ff:ff:ff:ff:ff, ethertype 802.1Q (0x8100), length 326: vlan 4, p 0, ethertype IPv4, 0.0.0.0.68 > 255.255.255.255.67: BOOTP/DHCP, Request from fa:16:3e:f0:45:cf, length 280

However, when I tcpdump br-tun, I don't seem to get any packets at all.

Some relevant outputs -

root@compute-122:~# ip netns
qdhcp-d82d7270-f590-45bd-a4f4-380af49382de
qdhcp-e0d38ee4-8036-490f-b3ed-ac274eee1769

(not sure if I should be having 2 namespaces)

root@compute-122:~# ip netns exec qdhcp-d82d7270-f590-45bd-a4f4-380af49382de route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.1.1     0.0.0.0         UG    0      0        0 tap25c0651b-89
169.254.0.0     0.0.0.0         255.255.0.0     U     0      0        0 tap25c0651b-89
192.168.1.0     0.0.0.0         255.255.255.0   U     0      0        0 tap25c0651b-89

Where exactly can the packet be going?