Revision history [back]

click to hide/show revision 1
initial version

Ubuntu OpenStack Single-Server Install network connectivity between host and main OpenStack container fails after reboot

I setup the Canonical distribution of openstack using the single server method, so that I could learn OpenStack before having to deploy and maintain it in production. I'm running into an issue with single server installation after a reboot of the main host that runs the primary LXC container that the openstack services run nested under. After a reboot, I'm no longer able to ping or ssh to any of the nested containers running within the main container. In order to connect to any of those machines, I first need to lxc-attach -n openstack-single-root, then I'm able to ssh to any other container from there.

Here is the output of the juju status:

ubuntu@openstack-single-root:~$ juju status
environment: local
machines:
  "0":
    agent-state: started
    agent-version: 1.24.5.1
    dns-name: localhost
    instance-id: localhost
    series: trusty
    state-server-member-status: has-vote
  "1":
    agent-state: started
    agent-version: 1.24.5.1
    instance-id: ubuntu-local-machine-1
    series: trusty
    containers:
      1/lxc/0:
        agent-state: started
        agent-version: 1.24.5.1
        dns-name: 10.0.6.53
        instance-id: ubuntu-local-machine-1-lxc-0
        series: trusty
        hardware: arch=amd64
      1/lxc/1:
        agent-state: started
        agent-version: 1.24.5.1
        dns-name: 10.0.6.24
        instance-id: ubuntu-local-machine-1-lxc-1
        series: trusty
        hardware: arch=amd64
      1/lxc/2:
        agent-state: started
        agent-version: 1.24.5.1
        dns-name: 10.0.6.105
        instance-id: ubuntu-local-machine-1-lxc-2
        series: trusty
        hardware: arch=amd64
      1/lxc/3:
        agent-state: started
        agent-version: 1.24.5.1
        dns-name: 10.0.6.222
        instance-id: ubuntu-local-machine-1-lxc-3
        series: trusty
        hardware: arch=amd64
      1/lxc/4:
        agent-state: started
        agent-version: 1.24.5.1
        dns-name: 10.0.6.94
        instance-id: ubuntu-local-machine-1-lxc-4
        series: trusty
        hardware: arch=amd64
      1/lxc/5:
        agent-state: started
        agent-version: 1.24.5.1
        dns-name: 10.0.6.188
        instance-id: ubuntu-local-machine-1-lxc-5
        series: trusty
        hardware: arch=amd64
      1/lxc/6:
        agent-state: started
        agent-version: 1.24.5.1
        dns-name: 10.0.6.49
        instance-id: ubuntu-local-machine-1-lxc-6
        series: trusty
        hardware: arch=amd64
      1/lxc/7:
        agent-state: started
        agent-version: 1.24.5.1
        dns-name: 10.0.6.64
        instance-id: ubuntu-local-machine-1-lxc-7
        series: trusty
        hardware: arch=amd64
    hardware: arch=amd64 cpu-cores=3 mem=6144M root-disk=20480M
  "2":
    agent-state: started
    agent-version: 1.24.5.1
    dns-name: 10.0.6.76
    instance-id: ubuntu-local-machine-2
    series: trusty
    hardware: arch=amd64 cpu-cores=1 mem=4096M root-disk=40960M
  "3":
    agent-state: started
    agent-version: 1.24.5.1
    dns-name: 10.0.6.25
    instance-id: ubuntu-local-machine-3
    series: trusty
    hardware: arch=amd64 cpu-cores=1 mem=2048M root-disk=20480M
  "4":
    agent-state: started
    agent-version: 1.24.5.1
    dns-name: 10.0.6.155
    instance-id: ubuntu-local-machine-4
    series: trusty
    hardware: arch=amd64 cpu-cores=1 mem=512M root-disk=8192M
services:
  glance:
    charm: cs:trusty/glance-20
    can-upgrade-to: cs:trusty/glance-24
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 14:17:47-04:00
    relations:
      amqp:
      - rabbitmq-server
      cluster:
      - glance
      identity-service:
      - keystone
      image-service:
      - nova-cloud-controller
      - nova-compute
      shared-db:
      - mysql
    units:
      glance/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 14:17:47-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:20:23-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: 1/lxc/7
        open-ports:
        - 9292/tcp
        public-address: 10.0.6.64
  glance-simplestreams-sync:
    charm: local:trusty/glance-simplestreams-sync-0
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 14:12:16-04:00
    relations:
      amqp:
      - rabbitmq-server
      identity-service:
      - keystone
    units:
      glance-simplestreams-sync/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 14:12:16-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:25:44-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: 1/lxc/6
        public-address: 10.0.6.49
  juju-gui:
    charm: cs:trusty/juju-gui-27
    can-upgrade-to: cs:trusty/juju-gui-38
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 15:02:45-04:00
    units:
      juju-gui/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 15:02:45-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:09:04-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: "4"
        open-ports:
        - 80/tcp
        - 443/tcp
        public-address: 10.0.6.155
  keystone:
    charm: cs:trusty/keystone-25
    can-upgrade-to: cs:trusty/keystone-28
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 14:14:59-04:00
    relations:
      cluster:
      - keystone
      identity-service:
      - glance
      - glance-simplestreams-sync
      - neutron-api
      - nova-cloud-controller
      - openstack-dashboard
      shared-db:
      - mysql
    units:
      keystone/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 14:14:59-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:20:49-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: 1/lxc/1
        public-address: 10.0.6.24
  mysql:
    charm: cs:trusty/mysql-25
    can-upgrade-to: cs:trusty/mysql-28
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 13:59:16-04:00
    relations:
      cluster:
      - mysql
      shared-db:
      - glance
      - keystone
      - neutron-api
      - neutron-gateway
      - nova-cloud-controller
      - nova-compute
    units:
      mysql/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 13:59:16-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:20:15-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: 1/lxc/0
        public-address: 10.0.6.53
  neutron-api:
    charm: cs:trusty/neutron-api-14
    can-upgrade-to: cs:trusty/neutron-api-18
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 14:15:48-04:00
    relations:
      amqp:
      - rabbitmq-server
      cluster:
      - neutron-api
      identity-service:
      - keystone
      neutron-api:
      - nova-cloud-controller
      neutron-plugin-api:
      - neutron-gateway
      - neutron-openvswitch
      shared-db:
      - mysql
    units:
      neutron-api/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 14:15:48-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:20:27-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: 1/lxc/4
        open-ports:
        - 9696/tcp
        public-address: 10.0.6.94
  neutron-gateway:
    charm: cs:trusty/neutron-gateway-3
    can-upgrade-to: cs:trusty/neutron-gateway-5
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 13:54:31-04:00
    relations:
      amqp:
      - rabbitmq-server
      cluster:
      - neutron-gateway
      juju-info:
      - ntp
      neutron-plugin-api:
      - neutron-api
      quantum-network-service:
      - nova-cloud-controller
      shared-db:
      - mysql
    units:
      neutron-gateway/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 13:54:31-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:14:25-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: "3"
        public-address: 10.0.6.25
        subordinates:
          ntp/0:
            workload-status:
              current: unknown
              since: 11 Sep 2015 14:19:30-04:00
            agent-status:
              current: idle
              since: 14 Sep 2015 09:13:30-04:00
              version: 1.24.5.1
            agent-state: started
            agent-version: 1.24.5.1
            upgrading-from: cs:trusty/ntp-8
            public-address: 10.0.6.25
  neutron-openvswitch:
    charm: cs:trusty/neutron-openvswitch-7
    can-upgrade-to: cs:trusty/neutron-openvswitch-9
    exposed: false
    service-status: {}
    relations:
      amqp:
      - rabbitmq-server
      neutron-plugin:
      - nova-compute
      neutron-plugin-api:
      - neutron-api
    subordinate-to:
    - nova-compute
  nova-cloud-controller:
    charm: cs:trusty/nova-cloud-controller-56
    can-upgrade-to: cs:trusty/nova-cloud-controller-60
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 14:17:57-04:00
    relations:
      amqp:
      - rabbitmq-server
      cloud-compute:
      - nova-compute
      cluster:
      - nova-cloud-controller
      identity-service:
      - keystone
      image-service:
      - glance
      neutron-api:
      - neutron-api
      quantum-network-service:
      - neutron-gateway
      shared-db:
      - mysql
    units:
      nova-cloud-controller/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 14:17:57-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:20:36-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: 1/lxc/3
        open-ports:
        - 3333/tcp
        - 8773/tcp
        - 8774/tcp
        - 9696/tcp
        public-address: 10.0.6.222
  nova-compute:
    charm: cs:trusty/nova-compute-24
    can-upgrade-to: cs:trusty/nova-compute-26
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 13:56:30-04:00
    relations:
      amqp:
      - rabbitmq-server
      cloud-compute:
      - nova-cloud-controller
      compute-peer:
      - nova-compute
      image-service:
      - glance
      juju-info:
      - ntp
      neutron-plugin:
      - neutron-openvswitch
      shared-db:
      - mysql
    units:
      nova-compute/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 13:56:30-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:09:14-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: "2"
        public-address: 10.0.6.76
        subordinates:
          neutron-openvswitch/0:
            workload-status:
              current: unknown
              since: 11 Sep 2015 14:21:52-04:00
            agent-status:
              current: idle
              since: 14 Sep 2015 09:09:05-04:00
              version: 1.24.5.1
            agent-state: started
            agent-version: 1.24.5.1
            upgrading-from: cs:trusty/neutron-openvswitch-7
            public-address: 10.0.6.76
          ntp/1:
            workload-status:
              current: unknown
              since: 11 Sep 2015 14:19:30-04:00
            agent-status:
              current: idle
              since: 14 Sep 2015 09:09:17-04:00
              version: 1.24.5.1
            agent-state: started
            agent-version: 1.24.5.1
            upgrading-from: cs:trusty/ntp-8
            public-address: 10.0.6.76
  ntp:
    charm: cs:trusty/ntp-8
    can-upgrade-to: cs:trusty/ntp-11
    exposed: false
    service-status: {}
    relations:
      juju-info:
      - neutron-gateway
      - nova-compute
      ntp-peers:
      - ntp
    subordinate-to:
    - neutron-gateway
    - nova-compute
  openstack-dashboard:
    charm: cs:trusty/openstack-dashboard-14
    can-upgrade-to: cs:trusty/openstack-dashboard-16
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 14:17:15-04:00
    relations:
      cluster:
      - openstack-dashboard
      identity-service:
      - keystone
    units:
      openstack-dashboard/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 14:17:15-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:20:17-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: 1/lxc/5
        open-ports:
        - 80/tcp
        - 443/tcp
        public-address: 10.0.6.188
  rabbitmq-server:
    charm: cs:trusty/rabbitmq-server-32
    can-upgrade-to: cs:trusty/rabbitmq-server-35
    exposed: false
    service-status:
      current: unknown
      since: 11 Sep 2015 14:08:51-04:00
    relations:
      amqp:
      - glance
      - glance-simplestreams-sync
      - neutron-api
      - neutron-gateway
      - neutron-openvswitch
      - nova-cloud-controller
      - nova-compute
      cluster:
      - rabbitmq-server
    units:
      rabbitmq-server/0:
        workload-status:
          current: unknown
          since: 11 Sep 2015 14:08:51-04:00
        agent-status:
          current: idle
          since: 14 Sep 2015 09:21:04-04:00
          version: 1.24.5.1
        agent-state: started
        agent-version: 1.24.5.1
        machine: 1/lxc/2
        open-ports:
        - 5672/tcp
        public-address: 10.0.6.105

Here is my networking information:

root@RDENG-OPNSTK-01:~# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:50:56:bf:3e:41
          inet addr:10.222.200.4  Bcast:10.222.200.255  Mask:255.255.255.0
          inet6 addr: fe80::250:56ff:febf:3e41/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:26471 errors:0 dropped:0 overruns:0 frame:0
          TX packets:41525 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:3153727 (3.1 MB)  TX bytes:4719357 (4.7 MB)

eth1      Link encap:Ethernet  HWaddr 00:50:56:bf:25:c5
          inet addr:10.222.203.1  Bcast:10.222.203.255  Mask:255.255.255.0
          inet6 addr: fe80::250:56ff:febf:25c5/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:3 errors:0 dropped:0 overruns:0 frame:0
          TX packets:81 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:180 (180.0 B)  TX bytes:14952 (14.9 KB)

eth2      Link encap:Ethernet  HWaddr 00:50:56:bf:5e:9f
          inet addr:172.16.7.92  Bcast:172.16.7.255  Mask:255.255.254.0
          inet6 addr: fe80::250:56ff:febf:5e9f/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:51742 errors:0 dropped:0 overruns:0 frame:0
          TX packets:127 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:4809971 (4.8 MB)  TX bytes:14247 (14.2 KB)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:48 errors:0 dropped:0 overruns:0 frame:0
          TX packets:48 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:2880 (2.8 KB)  TX bytes:2880 (2.8 KB)

lxcbr0    Link encap:Ethernet  HWaddr fe:71:aa:7e:f4:21
          inet addr:10.0.3.1  Bcast:10.0.3.255  Mask:255.255.255.0
          inet6 addr: fe80::a875:aff:fef7:d67e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:53930 errors:0 dropped:0 overruns:0 frame:0
          TX packets:44624 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:85475013 (85.4 MB)  TX bytes:3181676 (3.1 MB)

veth1NV2PJ Link encap:Ethernet  HWaddr fe:71:aa:7e:f4:21
          inet6 addr: fe80::fc71:aaff:fe7e:f421/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:53931 errors:0 dropped:0 overruns:0 frame:0
          TX packets:44629 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:86230123 (86.2 MB)  TX bytes:3182066 (3.1 MB)


root@RDENG-OPNSTK-01:~# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:50:56:bf:3e:41 brd ff:ff:ff:ff:ff:ff
    inet 10.222.200.4/24 brd 10.222.200.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::250:56ff:febf:3e41/64 scope link
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:50:56:bf:25:c5 brd ff:ff:ff:ff:ff:ff
    inet 10.222.203.1/24 brd 10.222.203.255 scope global eth1
       valid_lft forever preferred_lft forever
    inet6 fe80::250:56ff:febf:25c5/64 scope link
       valid_lft forever preferred_lft forever
4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:50:56:bf:5e:9f brd ff:ff:ff:ff:ff:ff
    inet 172.16.7.92/23 brd 172.16.7.255 scope global eth2
       valid_lft forever preferred_lft forever
    inet6 fe80::250:56ff:febf:5e9f/64 scope link
       valid_lft forever preferred_lft forever
5: lxcbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether fe:71:aa:7e:f4:21 brd ff:ff:ff:ff:ff:ff
    inet 10.0.3.1/24 brd 10.0.3.255 scope global lxcbr0
       valid_lft forever preferred_lft forever
    inet6 fe80::a875:aff:fef7:d67e/64 scope link
       valid_lft forever preferred_lft forever
7: veth1NV2PJ: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast master lxcbr0 state UP group default qlen 1000
    link/ether fe:71:aa:7e:f4:21 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc71:aaff:fe7e:f421/64 scope link
       valid_lft forever preferred_lft forever

I'm not really sure where to start diagnosing this issue.