Dear Folks,
We have some moderately busy OpenLDAP servers, OpenLDAP 2.4.32 running on CentOS 5.8 on HP BL495cG6 blades, with 24GB RAM, in a cluster of four, using LVS to balance the load. They peak at just over 3000 LDAP connections per second.
They stopped serving requests, or at least, began serving them so slowly that they caused an outage. After my colleague restarted slapd, they resumed serving the requests.
slapd didn't die, or dump core. Sadly, since the most minimal logging produces 25GB per day, so we have that turned off, and after the event, I now am graphing some of the statistics from the monitor database, though that provides less detail than I would wish.
My questions: =============
Has anyone had a similar experience of failure with this version of OpenLDAP specifically, or with any other version?
Has anyone any suggestions on what might have happened?
I am upgrading LDAP to 2.4.39, but would like to know if there is a known problem that I will resolve by this upgrade.
Here is a "sanitised" view of our configuration:
# slapd.conf generated by /usr/bin/conform
include /etc/openldap/schema/core.schema include /etc/openldap/schema/cosine.schema include /etc/openldap/schema/inetorgperson.schema include /etc/openldap/schema/nis.schema include /etc/openldap/schema/local.schema include /etc/openldap/schema/prefs.schema
loglevel stats allow bind_v2 pidfile /var/run/openldap/slapd.pid argsfile /var/run/openldap/slapd.args tool-threads 8 password-hash {CRYPT} password-crypt-salt-format "$1$%.8s" TLSCACertificateFile /etc/pki/tls/certs/ca-bundle.crt TLSCertificateFile /etc/openldap/ldap.syd.crt TLSCertificateKeyFile /etc/openldap/ldap.syd.key
############################################################ # GLOBAL database definition ############################################################
access to dn.base="" by peername.ip=133.3.193.2 read by peername.ip=19.21.201.10 read by * read
access to dn.base="cn=Subschema" by * read
############################################################ # ou=tree1,ou=name database definition ############################################################
database bdb suffix "ou=tree1,ou=name" rootdn cn=manager,ou=tree1,ou=name rootpw root-password directory /var/lib/ldap/ou=tree1,ou=name index domain eq,pres index entryCSN eq index entryUUID eq index mail eq,pres index objectClass eq index uid eq,pres
shm_key 331 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree1,ou=name" by peername.ip=96.76.69.162 read by * none break
access to dn.subtree="ou=tree1,ou=name" by dn.base="cn=Reader,ou=tree1,ou=name" read by peername.ip=205.191.75.42 read by peername.ip=198.132.212.127 read by peername.ip=101.169.45.146 read by peername.ip=212.38.134.82 read by peername.ip=127.0.0.1 read by peername.ip=49.15.162.36 read by peername.ip=142.226.69.215 read by peername.ip=236.107.103.158 read by peername.ip=57.167.92.229 read by peername.ip=138.57.161.36 read by peername.ip=99.33.110.154 read by peername.ip=196.119.159.149 read by peername.ip=44.116.9.128 read by peername.ip=122.234.222.27 read by self peername.ip=154.238.170.210 read by self peername.ip=31.62.15.241 read by self peername.ip=163.226.124.118 read by self peername.ip=92.190.127.61 read by self peername.ip=125.225.28.232 read by self peername.ip=122.148.131.239 read by self peername.ip=118.160.216.124 read by self peername.ip=74.226.71.41 read by self peername.ip=179.149.175.90 read by self peername.ip=235.160.10.186 read by self peername.ip=125.80.123.119 read by self peername.ip=185.5.66.203 read by self peername.ip=158.224.9.61 read by self peername.ip=46.146.95.97 read by self peername.ip=6.182.235.60 read by self peername.ip=7.137.32.196 read by self peername.ip=204.184.227.24 read by self peername.ip=56.10.14.169 read by self peername.ip=165.49.58.78 read by self peername.ip=165.211.155.235 read by self peername.ip=119.216.126.40 read by self peername.ip=94.212.46.227 read by self peername.ip=210.225.203.227 read by self peername.ip=171.235.159.67 read by self peername.ip=28.94.19.93 read by self peername.ip=206.12.128.149 read by self peername.ip=18.223.36.165 read by self peername.ip=35.164.112.79 read by self peername.ip=145.100.109.232 read by self peername.ip=10.230.19.137 read by self peername.ip=86.124.99.133 read by self peername.ip=56.223.60.177 read by self peername.ip=140.114.55.210 read by self peername.ip=216.13.135.39 read by self peername.ip=5.93.220.80 read by self peername.ip=87.215.35.140 read by self peername.ip=162.101.170.221 read by self peername.ip=100.59.12.102 read by self peername.ip=58.223.90.131 read by self peername.ip=124.165.37.123 read by self peername.ip=11.178.120.42 read by self peername.ip=99.205.107.56 read by self peername.ip=31.26.172.88 read by self peername.ip=211.133.184.38 read by self peername.ip=84.215.59.209 read by self peername.ip=13.70.104.6 read by self peername.ip=111.100.110.74 read by self peername.ip=190.199.36.1 read by self peername.ip=236.212.185.240 read by self peername.ip=180.4.69.152 read by self peername.ip=38.63.162.124 read by self peername.ip=68.34.84.216 read by self peername.ip=43.41.142.100 read by self peername.ip=118.225.164.10 read by self peername.ip=40.26.24.236 read by self peername.ip=197.161.148.167 read by self peername.ip=148.163.210.9 read by self peername.ip=161.141.31.120 read by self peername.ip=85.59.211.7 read by self peername.ip=237.59.225.239 read by self peername.ip=29.114.216.84 read by anonymous auth by * none break
syncrepl rid=001 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree1,ou=name credentials=syncrepl-password searchbase="ou=tree1,ou=name" retry="5 10 60 +"
############################################################ # ou=tree2,ou=name database definition ############################################################
database bdb suffix "ou=tree2,ou=name" rootdn cn=manager,ou=tree2,ou=name rootpw root-password directory /var/lib/ldap/ou=tree2,ou=name index avcid eq index entryCSN eq index entryUUID eq index gsid eq index objectClass eq
shm_key 320 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree2,ou=name" by peername.ip=1.224.5.235 read by * none break
access to dn.subtree="ou=tree2,ou=name" by dn.base="cn=Reader,ou=tree2,ou=name" read by peername.ip=124.182.243.237 read by peername.ip=73.200.132.164 read by peername.ip=5.106.199.123 read by peername.ip=243.60.43.234 read by peername.ip=127.0.0.1 read by anonymous auth by * none break
syncrepl rid=010 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree2,ou=name credentials=syncrepl-password searchbase="ou=tree2,ou=name" retry="5 10 60 +"
############################################################ # ou=tree3,ou=name database definition ############################################################
database bdb suffix "ou=tree3,ou=name" rootdn cn=manager,ou=tree3,ou=name rootpw root-password directory /var/lib/ldap/ou=tree3,ou=name index entryCSN eq index entryUUID eq index fnn eq index lineid eq index objectClass eq index serviceid eq index ullsid eq
shm_key 74 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree3,ou=name" by peername.ip=100.131.166.211 read by * none break
access to dn.subtree="ou=tree3,ou=name" by dn.base="cn=Reader,ou=tree3,ou=name" read by peername.ip=39.236.93.167 read by peername.ip=48.145.97.90 read by peername.ip=143.186.10.231 read by peername.ip=35.68.19.116 read by peername.ip=127.0.0.1 read by anonymous auth by * none break
syncrepl rid=008 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree3,ou=name credentials=syncrepl-password searchbase="ou=tree3,ou=name" retry="5 10 60 +"
############################################################ # ou=tree4,ou=name database definition ############################################################
database bdb suffix "ou=tree4,ou=name" rootdn cn=manager,ou=tree4,ou=name rootpw root-password directory /var/lib/ldap/ou=tree4,ou=name index cn eq index entryCSN eq index entryUUID eq index objectClass eq
shm_key 195 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree4,ou=name" by peername.ip=26.233.142.132 read by * none break
access to dn.subtree="ou=tree4,ou=name" by dn.base="cn=Reader,ou=tree4,ou=name" read by peername.ip=127.0.0.1 read by anonymous auth by * none break
access to dn.subtree="" by peername.ip=101.156.205.102 read by peername.ip=136.25.130.235 read by peername.ip=136.206.49.17 read by peername.ip=37.21.18.99 read by * none break
syncrepl rid=002 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree4,ou=name credentials=syncrepl-password searchbase="ou=tree4,ou=name" retry="5 10 60 +"
############################################################ # ou=tree5,ou=name database definition ############################################################
database bdb suffix "ou=tree5,ou=name" rootdn cn=manager,ou=tree5,ou=name rootpw root-password directory /var/lib/ldap/ou=tree5,ou=name index entryCSN eq index entryUUID eq index gsid eq index objectClass eq index uid eq
shm_key 626 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree5,ou=name" by peername.ip=225.143.210.78 read by * none break
access to dn.subtree="ou=tree5,ou=name" by dn.base="cn=Reader,ou=tree5,ou=name" read by peername.ip=106.154.145.84 read by peername.ip=197.28.75.183 read by peername.ip=179.151.185.161 read by peername.ip=4.60.182.12 read by peername.ip=127.0.0.1 read by anonymous auth by * none break
syncrepl rid=011 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree5,ou=name credentials=syncrepl-password searchbase="ou=tree5,ou=name" retry="5 10 60 +"
############################################################ # ou=tree6,ou=name database definition ############################################################
database bdb suffix "ou=tree6,ou=name" rootdn cn=manager,ou=tree6,ou=name rootpw root-password directory /var/lib/ldap/ou=tree6,ou=name index entryCSN eq index entryUUID eq index objectClass eq index uid eq index username eq
shm_key 290 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree6,ou=name" by peername.ip=195.44.90.201 read by * none break
access to dn.subtree="ou=tree6,ou=name" by dn.base="cn=Reader,ou=tree6,ou=name" read by peername.ip=131.55.25.176 read by peername.ip=52.151.227.147 read by peername.ip=144.91.87.53 read by peername.ip=51.236.131.115 read by peername.ip=127.0.0.1 read by anonymous auth by * none break
syncrepl rid=009 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree6,ou=name credentials=syncrepl-password searchbase="ou=tree6,ou=name" retry="5 10 60 +"
############################################################ # ou=tree7,ou=name database definition ############################################################
database bdb suffix "ou=tree7,ou=name" rootdn cn=manager,ou=tree7,ou=name rootpw root-password directory /var/lib/ldap/ou=tree7,ou=name index entryCSN eq index entryUUID eq index objectClass eq index uid eq index username eq
shm_key 105 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree7,ou=name" by peername.ip=82.36.151.29 read by peername.ip=69.183.15.150 read by * none break
access to dn.subtree="ou=tree7,ou=name" by peername.ip=81.44.61.5 read by peername.ip=75.34.88.223 read by peername.ip=40.106.122.233 read by dn.base="cn=Reader,ou=tree7,ou=name" read by peername.ip=127.0.0.1 read by anonymous auth by * none break
syncrepl rid=003 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree7,ou=name credentials=syncrepl-password searchbase="ou=tree7,ou=name" retry="5 10 60 +"
############################################################ # ou=tree8,ou=name database definition ############################################################
database bdb suffix "ou=tree8,ou=name" rootdn cn=manager,ou=tree8,ou=name rootpw root-password directory /var/lib/ldap/ou=tree8,ou=name index entryCSN eq index entryUUID eq index objectClass eq index uid eq index username eq
shm_key 280 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree8,ou=name" by peername.ip=221.195.40.138 read by * none break
access to dn.subtree="ou=tree8,ou=name" by dn.base="cn=Reader,ou=tree8,ou=name" read by peername.ip=211.28.110.6 read by peername.ip=4.134.128.66 read by peername.ip=194.183.122.54 read by peername.ip=89.223.206.194 read by peername.ip=28.88.175.182 read by peername.ip=209.169.46.101 read by peername.ip=230.21.178.118 read by peername.ip=41.55.202.55 read by peername.ip=127.0.0.1 read by anonymous auth by * none break
syncrepl rid=004 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree8,ou=name credentials=syncrepl-password searchbase="ou=tree8,ou=name" retry="5 10 60 +"
############################################################ # ou=tree9,ou=name database definition ############################################################
database bdb suffix "ou=tree9,ou=name" rootdn cn=manager,ou=tree9,ou=name rootpw root-password directory /var/lib/ldap/ou=tree9,ou=name index entryCSN eq index entryUUID eq index objectClass eq index uid eq index username eq
shm_key 122 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree9,ou=name" by peername.ip=153.54.1.23 read by * none break
access to dn.subtree="ou=tree9,ou=name" by dn.base="cn=Reader,ou=tree9,ou=name" read by peername.ip=164.215.79.230 read by peername.ip=52.205.194.57 read by peername.ip=69.215.8.144 read by peername.ip=240.46.54.48 read by peername.ip=127.0.0.1 read by anonymous auth by * none break
syncrepl rid=006 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree9,ou=name credentials=syncrepl-password searchbase="ou=tree9,ou=name" retry="5 10 60 +"
############################################################ # ou=tree10,ou=name database definition ############################################################
database bdb suffix "ou=tree10,ou=name" rootdn cn=manager,ou=tree10,ou=name rootpw root-password directory /var/lib/ldap/ou=tree10,ou=name index entryCSN eq index entryUUID eq index mtaMacAddress eq index mtaSubcriberKey eq index objectClass eq
shm_key 548 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree10,ou=name" by peername.ip=111.64.19.131 read by * none break
access to dn.subtree="ou=tree10,ou=name" by dn.base="cn=Reader,ou=tree10,ou=name" read by peername.ip=49.225.102.91 read by peername.ip=169.2.225.154 read by peername.ip=177.202.224.166 read by peername.ip=56.152.182.238 read by peername.ip=157.83.196.154 read by peername.ip=66.220.12.152 read by peername.ip=132.204.57.43 read by peername.ip=222.142.242.198 read by peername.ip=153.130.78.144 read by peername.ip=166.31.243.77 read by peername.ip=226.40.169.1 read by peername.ip=224.110.119.208 read by peername.ip=15.227.192.210 read by peername.ip=45.56.216.152 read by peername.ip=127.0.0.1 read by anonymous auth by * none break
syncrepl rid=000 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree10,ou=name credentials=syncrepl-password searchbase="ou=tree10,ou=name" retry="5 10 60 +"
############################################################ # ou=tree11,ou=name database definition ############################################################
database bdb suffix "ou=tree11,ou=name" rootdn cn=manager,ou=tree11,ou=name rootpw root-password directory /var/lib/ldap/ou=tree11,ou=name index entryCSN eq index entryUUID eq index objectClass eq index uid eq index username eq
shm_key 215 cachesize 100000 idlcachesize 100000 checkpoint 32768 30 idletimeout 3600 writetimeout 90000
access to dn.base="ou=tree11,ou=name" by peername.ip=51.114.241.35 read by * none break
access to dn.subtree="ou=tree11,ou=name" by dn.base="cn=Reader,ou=tree11,ou=name" read by peername.ip=17.32.79.33 read by peername.ip=140.205.127.168 read by peername.ip=190.147.122.157 read by peername.ip=170.66.104.2 read by peername.ip=45.154.226.85 read by peername.ip=116.172.183.88 read by peername.ip=96.51.58.70 read by peername.ip=85.240.27.171 read by peername.ip=127.0.0.1 read by anonymous auth by * none break
syncrepl rid=005 provider=ldap://master:389 type=refreshAndPersist bindmethod=simple binddn=cn=syncrepl,ou=tree11,ou=name credentials=syncrepl-password searchbase="ou=tree11,ou=name" retry="5 10 60 +"
############################################################ # ou=tree12,ou=name database definition ############################################################
database monitor rootdn cn=manager,ou=tree12,ou=name rootpw root-password
access to dn.subtree="ou=tree12,ou=name" by peername.ip=127.0.0.1 read by * none