Revision history [back]

click to hide/show revision 1
initial version

Keystone authentication failure in a HA Setup

I am running two controllers as active/active under two haproxy nodes as active/passive using a VIP. All the services in controller nodes are load balanced. I am facing a strange situation. Whenever I am trying to keystone command, for the first two attempts it gives me this error : An unexpected error prevented the server from fulfilling your request. (HTTP 500), and then when I run the commands continuously without much time gap, it works fine. Then when I try to run the same commands after sometime, again the same situation comes into picture. No significant error message in the logs is noticed. I am very new to load balancing, so unable to figure out whether it is a load balancing issue or a something else.

Keystone authentication failure in a HA Setup

I am running two controllers as active/active under two haproxy nodes as active/passive using a VIP. All the services in controller nodes are load balanced. I am facing a strange situation. Whenever I am trying to keystone command, for the first two attempts it gives me this error : An unexpected error prevented the server from fulfilling your request. (HTTP 500), and then when I run the commands continuously without much time gap, it works fine. Then when I try to run the same commands after sometime, again the same situation comes into picture. No significant error message in the logs is noticed. I am very new to load balancing, so unable to figure out whether it is a load balancing issue or a something else. This is my haproxy.cfg :

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    user haproxy
    group haproxy
    daemon

defaults
    log global
    mode    http
    option  httplog
    option  dontlognull
        contimeout 5000
        clitimeout 50000
        srvtimeout 50000
    errorfile 400 /etc/haproxy/errors/400.http
    errorfile 403 /etc/haproxy/errors/403.http
    errorfile 408 /etc/haproxy/errors/408.http
    errorfile 500 /etc/haproxy/errors/500.http
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    errorfile 504 /etc/haproxy/errors/504.http

listen galera 192.168.1.64:3306
        balance source
        mode tcp
        option tcpka
        option mysql-check user haproxy
        server Controller1 192.168.1.61:3306 check weight 1
        server Controller2 192.168.1.62:3306 check weight 1

listen keystone_admin 192.168.1.64:35357
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:35357 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:35357 check inter 2000 rise 2 fall 5

listen keystone_api 192.168.1.64:5000
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:5000 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:5000 check inter 2000 rise 2 fall 5

listen glance-api 192.168.1.64:9292
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server icehouse1 192.168.1.61:9292 check inter 2000 rise 2 fall 5
        server icehouse2 192.168.1.62:9292 check inter 2000 rise 2 fall 5

listen glance-registry 192.168.1.64:9191
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:9191 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:9191 check inter 2000 rise 2 fall 5

listen nova_ec2 192.168.1.64:8773
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:8773 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:8773 check inter 2000 rise 2 fall 5

listen nova_osapi 192.168.1.64:8774
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:8774 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:8774 check inter 2000 rise 2 fall 5

listen nova_metadata 192.168.1.64:8775
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:8775 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:8775 check inter 2000 rise 2 fall 5

listen novnc 192.168.1.64:6080
        balance source
        option tcpka
        maxconn 10000
        server Controller1 192.168.1.61:6080 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:6080 check inter 2000 rise 2 fall 5

listen dashboard 192.168.1.64:80
        balance  source
        capture  cookie vgnvisitor= len 32
        cookie  SERVERID insert indirect nocache
        mode  http
        option  forwardfor
        option  httpchk
        option  httpclose
        rspidel  ^Set-cookie:\ IP=
        server Controller1 192.168.1.61:80 cookie control01 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:80 cookie control02 check inter 2000 rise 2 fall 5

listen memcached 192.168.1.64:11211
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:11211 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:11211 check inter 2000 rise 2 fall 5

Keystone authentication failure in a HA Setup

I am running two controllers as active/active under two haproxy nodes as active/passive using a VIP. All the services in controller nodes are load balanced. I am facing a strange situation. Whenever I am trying to keystone command, for the first two attempts it gives me this error : An unexpected error prevented the server from fulfilling your request. (HTTP 500), and then when I run the commands continuously without much time gap, it works fine. Then when I try to run the same commands after sometime, again the same situation comes into picture. No significant error message in the logs is noticed. I am very new to load balancing, so unable to figure out whether it is a load balancing issue or a something else. This is my haproxy.cfg :

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    user haproxy
    group haproxy
    daemon

defaults
    log global
    mode    http
    option  httplog
    option  dontlognull
        contimeout 5000
        clitimeout 50000
        srvtimeout 50000
    errorfile 400 /etc/haproxy/errors/400.http
    errorfile 403 /etc/haproxy/errors/403.http
    errorfile 408 /etc/haproxy/errors/408.http
    errorfile 500 /etc/haproxy/errors/500.http
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    errorfile 504 /etc/haproxy/errors/504.http

listen galera 192.168.1.64:3306
        balance source
        mode tcp
        option tcpka
        option mysql-check user haproxy
        server Controller1 192.168.1.61:3306 check weight 1
        server Controller2 192.168.1.62:3306 check weight 1

listen keystone_admin 192.168.1.64:35357
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:35357 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:35357 check inter 2000 rise 2 fall 5

listen keystone_api 192.168.1.64:5000
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:5000 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:5000 check inter 2000 rise 2 fall 5

listen glance-api 192.168.1.64:9292
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server icehouse1 Controller1 192.168.1.61:9292 check inter 2000 rise 2 fall 5
        server icehouse2 Controller2 192.168.1.62:9292 check inter 2000 rise 2 fall 5

listen glance-registry 192.168.1.64:9191
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:9191 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:9191 check inter 2000 rise 2 fall 5

listen nova_ec2 192.168.1.64:8773
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:8773 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:8773 check inter 2000 rise 2 fall 5

listen nova_osapi 192.168.1.64:8774
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:8774 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:8774 check inter 2000 rise 2 fall 5

listen nova_metadata 192.168.1.64:8775
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:8775 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:8775 check inter 2000 rise 2 fall 5

listen novnc 192.168.1.64:6080
        balance source
        option tcpka
        maxconn 10000
        server Controller1 192.168.1.61:6080 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:6080 check inter 2000 rise 2 fall 5

listen dashboard 192.168.1.64:80
        balance  source
        capture  cookie vgnvisitor= len 32
        cookie  SERVERID insert indirect nocache
        mode  http
        option  forwardfor
        option  httpchk
        option  httpclose
        rspidel  ^Set-cookie:\ IP=
        server Controller1 192.168.1.61:80 cookie control01 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:80 cookie control02 check inter 2000 rise 2 fall 5

listen memcached 192.168.1.64:11211
        balance source
        option tcpka
        option httpchk
        maxconn 10000
        server Controller1 192.168.1.61:11211 check inter 2000 rise 2 fall 5
        server Controller2 192.168.1.62:11211 check inter 2000 rise 2 fall 5