Hi,
I have two node MMR ldap cluster with hdb, but for some reason one of the servers CPU get spiked every minute, I am not sure what is causing this spike every minute, we have same server configuration in both the server. Also when i do db_stat on *.bdb files this server is taking too long to respond for db_stat, i am pasting top and strace out put of that thread here. Also attach a graph shows cpu spikes and active threads spikes in both the servers (green and yellow lines),
Openldap Version: openldap-2.4.43
[root@localhost ldap]# db_stat -d id2entry.bdb
Thu Sep 21 22:16:15 2017 Local time
date
^CKilled
[root@localhost ldap]# date
Thu Sep 21 22:27:54 UTC 2017
du -sh id2entry.bdb
7.6G id2entry.bdb
DB_CONFIG
sudo cat /var/lib/ldap/DB_CONFIG |grep -v "^#"
set_cachesize 8 0 1
set_lg_regionmax 262144
set_lg_bsize 2097152
set_lk_max_locks 3000
set_lk_max_objects 1500
set_lk_max_lockers 1500
Top output of busy thread at the time of cpu spike:
ldap_top..20170922.165808.886367620: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:01.04 slapd
ldap_top..20170922.165811.390769103: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:01.04 slapd
ldap_top..20170922.165813.895158022: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:01.04 slapd
ldap_top..20170922.165816.399627768: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:01.04 slapd
ldap_top..20170922.165818.903936890: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:01.04 slapd
ldap_top..20170922.165821.408275241: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:01.04 slapd
ldap_top..20170922.165823.912707039: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:01.05 slapd
ldap_top..20170922.165826.417287284: 61437 ldap 20 0 14.8g 13g 8.2g R 101.8 47.5 18:02.51 slapd
ldap_top..20170922.165828.922186726: 61437 ldap 20 0 14.8g 13g 8.2g R 99.8 47.5 18:05.01 slapd
ldap_top..20170922.165831.427101922: 61437 ldap 20 0 14.8g 13g 8.2g R 99.8 47.5 18:07.51 slapd
ldap_top..20170922.165833.931996934: 61437 ldap 20 0 14.8g 13g 8.2g R 99.8 47.5 18:09.92 slapd
ldap_top..20170922.165836.436814788: 61437 ldap 20 0 14.8g 13g 8.2g R 99.8 47.5 18:12.42 slapd
ldap_top..20170922.165838.941597309: 61437 ldap 20 0 14.8g 13g 8.2g R 99.8 47.5 18:14.92 slapd
ldap_top..20170922.165841.446448385: 61437 ldap 20 0 14.8g 13g 8.2g R 99.8 47.5 18:17.18 slapd
ldap_top..20170922.165843.951329579: 61437 ldap 20 0 14.8g 13g 8.2g R 97.8 47.5 18:19.67 slapd
ldap_top..20170922.165846.456172321: 61437 ldap 20 0 14.8g 13g 8.2g R 99.8 47.5 18:21.85 slapd
ldap_top..20170922.165848.961017603: 61437 ldap 20 0 14.8g 13g 8.2g R 99.8 47.5 18:24.35 slapd
ldap_top..20170922.165851.465897714: 61437 ldap 20 0 14.8g 13g 8.2g R 101.8 47.5 18:26.49 slapd
ldap_top..20170922.165853.970797751: 61437 ldap 20 0 14.8g 13g 8.2g S 57.9 47.5 18:28.77 slapd
ldap_top..20170922.165856.475641225: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.77 slapd
ldap_top..20170922.165858.980268851: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.77 slapd
ldap_top..20170922.165901.485817704: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.77 slapd
ldap_top..20170922.165903.990499397: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165906.495260168: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165909.078879876: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165911.583657585: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165914.088384025: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165916.593114635: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165919.097562041: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165921.602221629: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165924.106739600: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165926.611115008: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165929.115600777: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165931.620082217: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
ldap_top..20170922.165934.124916755: 61437 ldap 20 0 14.8g 13g 8.2g S 0.0 47.5 18:28.78 slapd
strace output on that thread around that time:
grep "16:58:26" ldap_strace.61437
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea40, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea40, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea40, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea40, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x560271548120, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x560271548120, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
16:58:26 futex(0x5602707aea00, FUTEX_WAIT_PRIVATE, 2, NULL) = -1 EAGAIN (Resource temporarily unavailable)
16:58:26 futex(0x5602707aea00, FUTEX_WAKE_PRIVATE, 1) = 0
Here is the disk activity metrics and corresponding pstack ouput captured at different time when the system is not serving any traffic other than replication:
04:20:32 PM 19169 0.00 10042.00 0.00 slapd
04:20:34 PM 19169 0.00 45850.00 0.00 slapd
04:20:36 PM 19169 0.00 35706.00 0.00 slapd
04:20:38 PM 19169 0.00 37096.00 0.00 slapd
04:20:40 PM 19169 0.00 43654.00 0.00 slapd
04:20:40 PM PID kB_rd/s kB_wr/s kB_ccwr/s Command
04:20:42 PM 19169 0.00 42948.00 0.00 slapd
04:20:44 PM 19169 0.00 43272.00 0.00 slapd
04:20:46 PM 19169 0.00 37084.00 0.00 slapd
04:20:48 PM 19169 0.00 22740.00 0.00 slapd
04:20:50 PM 19169 0.00 23540.00 0.00 slapd
04:20:52 PM 19169 0.00 23540.00 0.00 slapd
04:20:54 PM 19169 0.00 7458.00 0.00 slapd
04:20:56 PM 19169 0.00 0.00 0.00 slapd
04:20:58 PM 19169 0.00 0.00 0.00 slapd
04:21:00 PM 19169 0.00 0.00 0.00 slapd
04:21:02 PM 19169 0.00 0.00 0.00 slapd
04:21:04 PM 19169 0.00 0.00 0.00 slapd
04:21:06 PM 19169 0.00 0.00 0.00 slapd
04:21:08 PM 19169 0.00 0.00 0.00 slapd
04:21:10 PM 19169 0.00 30.00 0.00 slapd
04:21:12 PM 19169 0.00 0.00 0.00 slapd
04:21:14 PM 19169 0.00 0.00 0.00 slapd
04:21:16 PM 19169 0.00 0.00 0.00 slapd
04:21:18 PM 19169 4.00 346.00 0.00 slapd
04:21:20 PM 19169 24.00 1012.00 0.00 slapd
04:21:20 PM PID kB_rd/s kB_wr/s kB_ccwr/s Command
04:21:22 PM 19169 0.00 14.00 0.00 slapd
04:21:24 PM 19169 6.00 1098.00 0.00 slapd
04:21:26 PM 19169 24.00 302.00 0.00 slapd
04:21:28 PM 19169 82.00 598.00 0.00 slapd
04:21:30 PM 19169 82.00 582.00 0.00 slapd
04:21:32 PM 19169 10.00 114.00 0.00 slapd
04:21:34 PM 19169 14.00 1854.00 0.00 slapd
04:21:36 PM 19169 6.00 608.00 0.00 slapd
04:21:38 PM 19169 6.00 502.00 0.00 slapd
04:21:40 PM 19169 10.00 992.00 0.00 slapd
04:21:42 PM 19169 28.00 5620.00 0.00 slapd
04:21:44 PM 19169 0.00 0.00 0.00 slapd
04:21:46 PM 19169 0.00 0.00 0.00 slapd
04:21:48 PM 19169 0.00 0.00 0.00 slapd
04:21:50 PM 19169 0.00 0.00 0.00 slapd
04:21:52 PM 19169 0.00 0.00 0.00 slapd
04:21:54 PM 19169 0.00 1018.00 0.00 slapd
04:21:56 PM 19169 0.00 44146.00 0.00 slapd
04:21:58 PM 19169 0.00 39188.00 0.00 slapd
04:22:00 PM 19169 0.00 43206.00 0.00 slapd
04:22:00 PM PID kB_rd/s kB_wr/s kB_ccwr/s Command
04:22:02 PM 19169 0.00 39266.00 0.00 slapd
04:22:04 PM 19169 0.00 43240.00 0.00 slapd
04:22:06 PM 19169 0.00 44358.00 0.00 slapd
04:22:08 PM 19169 0.00 43790.00 0.00 slapd
04:22:10 PM 19169 0.00 38186.00 0.00 slapd
04:22:12 PM 19169 0.00 36304.00 0.00 slapd
04:22:14 PM 19169 0.00 42903.48 0.00 slapd
04:22:16 PM 19169 0.00 42246.00 0.00 slapd
04:22:18 PM 19169 0.00 9720.00 0.00 slapd
Thu Sep 21 16:20:40 UTC 2017
Thread 14 (Thread 0x7fc4eb57c700 (LWP 19171)):
#0 0x00007fc702d3d8e3 in epoll_wait () from /lib64/libc.so.6
#1 0x00005638f66beda9 in ?? ()
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7fc4ead7b700 (LWP 19172)):
#0 0x00007fc704acce6c in __db_tas_mutex_unlock () from /lib64/
libdb-4.7.so#1 0x00007fc704b74738 in __dbc_close () from /lib64/
libdb-4.7.so#3 0x00007fc704b750da in __dbc_get () from /lib64/
libdb-4.7.so#4 0x00007fc704b7f9dd in __dbc_get_pp () from /lib64/
libdb-4.7.so#5 0x00005638f67a458c in hdb_id2entry ()
#6 0x00005638f679b6f8 in hdb_cache_find_id ()
#7 0x00005638f675041d in hdb_search ()
#8 0x00005638f67304c2 in overlay_op_walk ()
#9 0x00005638f6730624 in ?? ()
#10 0x00005638f66c69e9 in fe_op_search ()
#11 0x00005638f66c62b6 in do_search ()
#12 0x00005638f66c3ad4 in ?? ()
#13 0x00005638f66c3e3b in ?? ()
#14 0x00007fc705441afa in ?? () from /lib64/libldap_r-2.4.so.2
#15 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#16 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7fc4ea57a700 (LWP 19173)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7fc4e9d79700 (LWP 19174)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7fc4e9578700 (LWP 19175)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7fc4e8d77700 (LWP 19176)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7fc4d7fff700 (LWP 19179)):
#0 0x00007fc7032229ad in connect () from /lib64/libpthread.so.0
#1 0x00007fc705458d0c in ldap_connect_to_host () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc705442ece in ldap_int_open_connection () from /lib64/libldap_r-2.4.so.2
#3 0x00007fc705455f1d in ldap_new_connection () from /lib64/libldap_r-2.4.so.2
#4 0x00007fc7054424bf in ldap_open_defconn () from /lib64/libldap_r-2.4.so.2
#5 0x00007fc705457458 in ldap_send_initial_request () from /lib64/libldap_r-2.4.so.2
#6 0x00007fc70544c63f in ldap_sasl_bind () from /lib64/libldap_r-2.4.so.2
#7 0x00007fc70544ca19 in ldap_sasl_bind_s () from /lib64/libldap_r-2.4.so.2
#8 0x00005638f66baf52 in slap_client_connect ()
#9 0x00005638f672997d in ?? ()
#10 0x00007fc705441afa in ?? () from /lib64/libldap_r-2.4.so.2
#11 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7fc4d77fe700 (LWP 19293)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7fc4d6ffd700 (LWP 23630)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7fc4d67fc700 (LWP 23631)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7fc4d5ffb700 (LWP 23632)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7fc4d57fa700 (LWP 23633)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7fc4d4ff9700 (LWP 23634)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7fc705895780 (LWP 19169)):
#0 0x00007fc70321cf17 in pthread_join () from /lib64/libpthread.so.0
#1 0x00005638f66c0dc9 in slapd_daemon ()
#2 0x00005638f66a7d68 in main ()
Thu Sep 21 16:20:42 UTC 2017
Thread 14 (Thread 0x7fc4eb57c700 (LWP 19171)):
#0 0x00007fc702d3d8e3 in epoll_wait () from /lib64/libc.so.6
#1 0x00005638f66beda9 in ?? ()
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7fc4ead7b700 (LWP 19172)):
#0 0x00005638f66cfa88 in entry_decode ()
#1 0x00005638f67a44a2 in hdb_id2entry ()
#2 0x00005638f679b6f8 in hdb_cache_find_id ()
#3 0x00005638f675041d in hdb_search ()
#4 0x00005638f67304c2 in overlay_op_walk ()
#5 0x00005638f6730624 in ?? ()
#6 0x00005638f66c69e9 in fe_op_search ()
#7 0x00005638f66c62b6 in do_search ()
#8 0x00005638f66c3ad4 in ?? ()
#9 0x00005638f66c3e3b in ?? ()
#10 0x00007fc705441afa in ?? () from /lib64/libldap_r-2.4.so.2
#11 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7fc4ea57a700 (LWP 19173)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7fc4e9d79700 (LWP 19174)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7fc4e9578700 (LWP 19175)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7fc4e8d77700 (LWP 19176)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7fc4d7fff700 (LWP 19179)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7fc4d77fe700 (LWP 19293)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7fc4d6ffd700 (LWP 23630)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7fc4d67fc700 (LWP 23631)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7fc4d5ffb700 (LWP 23632)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7fc4d57fa700 (LWP 23633)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7fc4d4ff9700 (LWP 23634)):
#0 0x00007fc70321f905 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fc705441b4b in ?? () from /lib64/libldap_r-2.4.so.2
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7fc705895780 (LWP 19169)):
#0 0x00007fc70321cf17 in pthread_join () from /lib64/libpthread.so.0
#1 0x00005638f66c0dc9 in slapd_daemon ()
#2 0x00005638f66a7d68 in main ()
Thu Sep 21 16:20:44 UTC 2017
Thread 14 (Thread 0x7fc4eb57c700 (LWP 19171)):
#0 0x00007fc702d3d8e3 in epoll_wait () from /lib64/libc.so.6
#1 0x00005638f66beda9 in ?? ()
#2 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7fc4ead7b700 (LWP 19172)):
#0 0x00005638f6706b30 in ad_keystring ()
#1 0x00005638f6706c6e in slap_bv2ad ()
#2 0x00005638f66cf9d6 in entry_decode ()
#3 0x00005638f67a44a2 in hdb_id2entry ()
#4 0x00005638f679b6f8 in hdb_cache_find_id ()
#5 0x00005638f675041d in hdb_search ()
#6 0x00005638f67304c2 in overlay_op_walk ()
#7 0x00005638f6730624 in ?? ()
#8 0x00005638f66c69e9 in fe_op_search ()
#9 0x00005638f66c62b6 in do_search ()
#10 0x00005638f66c3ad4 in ?? ()
#11 0x00005638f66c3e3b in ?? ()
#12 0x00007fc705441afa in ?? () from /lib64/libldap_r-2.4.so.2
#13 0x00007fc70321bde5 in start_thread () from /lib64/libpthread.so.0
#14 0x00007fc702d3d30d in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7fc4ea57a700 (LWP 19173)):
After i did db_recover db_stat is responding fast but cpu spikes are still there, is there any way to find if my db is bad?
Thanks,
Ram