Full_Name: Semyon Chaichenets Version: 2.4.38 OS: Linux --- 3.2.0-58-generic #88-Ubuntu SMP Tue Dec 3 17:37:58 UTC 2013 x86_64 URL: ftp://ftp.openldap.org/incoming/ Submission from: (NULL) (142.244.152.148)
We had slapd crash four times in the past 2 weeks, on two separate nodes.
Each time, the last instruction executed was back-mdb/dn2id.c:738, and each time, it was called from back-mdb/search.c:747.
In every case, the cause appears to be an infinite loop: the stack is very deep - in once instance, we had 2492641 frames in the back-trace; while the function calls are oscillating between two addresses.
Both nodes in question had their database initialized from scratch by syncrepl.
I cannot provide core dumps for confidentiality reasons, but hopefully you will find the following information helpful:
# ldd `which slapd` linux-vdso.so.1 => (0x00007fff673ff000) libldap_r-2.4.so.2 => /usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2 (0x00007f56feb1a000) liblber-2.4.so.2 => /usr/lib/x86_64-linux-gnu/liblber-2.4.so.2 (0x00007f56fe90c000) libuuid.so.1 => /lib/x86_64-linux-gnu/libuuid.so.1 (0x00007f56fe706000) libslp.so.1 => /usr/lib/libslp.so.1 (0x00007f56fe4f5000) libsasl2.so.2 => /usr/lib/x86_64-linux-gnu/libsasl2.so.2 (0x00007f56fe2da000) libcrypt.so.1 => /lib/x86_64-linux-gnu/libcrypt.so.1 (0x00007f56fe0a0000) libslapi-2.4.so.2 => /usr/lib/x86_64-linux-gnu/libslapi-2.4.so.2 (0x00007f56fde82000) libltdl.so.7 => /usr/lib/x86_64-linux-gnu/libltdl.so.7 (0x00007f56fdc78000) libwrap.so.0 => /lib/x86_64-linux-gnu/libwrap.so.0 (0x00007f56fda6e000) libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f56fd851000) libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f56fd491000) libresolv.so.2 => /lib/x86_64-linux-gnu/libresolv.so.2 (0x00007f56fd274000) libssl.so.1.0.0 => /lib/x86_64-linux-gnu/libssl.so.1.0.0 (0x00007f56fd016000) libcrypto.so.1.0.0 => /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 (0x00007f56fcc3b000) /lib64/ld-linux-x86-64.so.2 (0x00007f56ff0f6000) libnsl.so.1 => /lib/x86_64-linux-gnu/libnsl.so.1 (0x00007f56fca20000) libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f56fc81c000) libz.so.1 => /lib/x86_64-linux-gnu/libz.so.1 (0x00007f56fc604000)
=== node 1, crash on 2014-02-07 ===
GNU gdb (Ubuntu/Linaro 7.4-2012.04-0ubuntu2.1) 7.4-2012.04 [..] Reading symbols from /usr/sbin/slapd...done.
warning: exec file is newer than core file. [New LWP 6117] [New LWP 16603] [New LWP 6106] [New LWP 6116] [New LWP 16602] [New LWP 6115] [New LWP 6114] [New LWP 6113] [New LWP 16604] [New LWP 6112]
[..]
Core was generated by `/usr/sbin/slapd -u openldap -g openldap -h ldap://127.0.0.1 ldap://-------'. Program terminated with signal 11, Segmentation fault. #0 0x00007fe30c4818b1 in mdb_idscopes (op=0x7fe0d8000c30, isc=0x7fe0f94e9670) at ../../../../../servers/slapd/back-mdb/dn2id.c:738
warning: Source file is more recent than executable. 738 isc->nrdns[isc->numrdns].bv_len = nrlen;
(gdb) info locals mdb = <optimized out> dbi = <optimized out> key = {mv_size = 8, mv_data = 0x7fe0f9369588} data = {mv_size = 38, mv_data = 0x7fe11e1e9df4} id = 315367 id2 = {mid = 315054, mval = {mv_size = 34, mv_data = 0x7fe19e83f5e6}} ptr = <optimized out> rc = 1 x = <optimized out> nrlen = 13 rlen = 13 d = 0x7fe11e1e9df4 (gdb) info registers rax 0xd 13 rbx 0x7fe0f94e9670 140604232078960 rcx 0x130c990 19974544 rdx 0x7fe11e1e9df4 140604849692148 rsi 0x7fe0fa7fe000 140604252086272 rdi 0x7fe108147010 140604479926288 rbp 0x7fe0f9369588 0x7fe0f9369588 rsp 0x7fe0f9369540 0x7fe0f9369540 r8 0x26 38 r9 0x130c97 1248407 r10 0xd 13 r11 0x7fe11e1e9df6 140604849692150 r12 0x7fe0d8000c30 140603673283632 r13 0x1 1 r14 0x7fe0ec1029c0 140604009884096 r15 0x7fe0f94f98b0 140604232145072 rip 0x7fe30c4818b1 0x7fe30c4818b1 <mdb_idscopes+177> eflags 0x10206 [ PF IF RF ] cs 0x33 51 ss 0x2b 43 ds 0x0 0 es 0x0 0 fs 0x0 0 gs 0x0 0
(gdb) info locals scopeok = 0 edata = {mv_size = 13, mv_data = 0x7fe11e1e9e04} mdb = <optimized out> id = 1437644 cursor = 140604849692164 nsubs = 13 ncand = 1036510 cscope = 0 lastid = 140604849692164 candidates = {18446744073709551615, 1, 18446744073709551615, 208401, 208402, 270076, 292060, 528090, 528091, 528092, 528093, 528094, 528095, 528096, 528097, 528098, 528099, 617378, 687228, 1448040, 1448041, 524157, 524158, 524159, 574368, 656851, 810235, 864549, 84040, 84041, 94183, 94184, 118570, 132095, 132096, 132097, 132098, 132099, 132100, 132101, 132102, 132103, 132104, 132105, 132106, 132107, 142374, 142375, 149621, 150352, 175234, 175235, 175236, 175237, 175238, 175239, 175240, 175241, 175242, 175243, 175244, 175245, 175246, 185347, 185348, 205311, 236539, 242770, 242771, 242772, 242773, 242774, 242775, 242776, 242777, 242778, 242779, 242780, 242781, 242782, 253264, 253265, 282749, 308079, 308080, 308081, 308082, 308083, 308084, 308085, 308086, 308087, 308088, 308089, 308090, 308091, 318449, 318450, 404268, 404269, 620845, 665852, 665853, 665854, 665855, 665856, 665857, 665858, 665859, 665860, 665861, 665862, 665863, 665864, 677595, 677596, 836477, 1279141, 1287440, 1287441, 1287442, 1287443, 1287444, 1287445, 1287446, 1287447, 1287448, 1287449, 1287450, 1287451, 1294713, 1294714, 1303316, 1303317, 1303318, 1303319, 1303320, 1303321, 1303322, 1303323, 1303324, 1303325, 1303326, 1303327, 1310725, 1310726, 1319676, 1319677, 1319678, 1319679, 1319680, 1319681, 1319682, 1319683, 1319684, 1319685, 1319686, 1319687, 1327161, 1327162, 1336551, 1336552, 1336553, 1336554, 1336555, 1336556, 1336557, 1336558, 1336559, 1336560, 1336561, 1336562, 1343735, 1353674, 1353675, 1353676, 1353677, 1353678, 1353679, 1353680, 1353681, 1353682, 1353683, 1353684, 1353685, 1360913, 1371324, 1371325, 1371326, 1371327, 1371328, 1371329, 1371330, 1371331, 1371332, 1371333, 1371334, 1371335, 1378691, 1388937...} iscopes = {0 <repeats 65502 times>, 140604009884096, 140604010617776, 635087, 140604479926288, 4294967298, 2, 12, 140606314528669, 9, 140606770728942, 12, 140604966947099, 9, 140606770728942, 14, 140606644291055, 24, 140606471143055, 14, 140606533666287, 14, 140606572504559, 0 <repeats 12 times>} scopes = 0x7fe108147010 e = 0xd base = 0x7fe11e1e9e04 matched = 0xd attrs = <optimized out> mask = <optimized out> stoptime = 1391778836 manageDSAit = <optimized out> isc = {mt = 0x7fe0ec1029c0, mc = 0x7fe0ec1db620, id = 1437644, scopes = 0x7fe108147010, numrdns = 1248406, nscope = 0, oscope = 2, rdns = {{bv_len = 14, bv_val = 0x7fe1115e75a1 <Address 0x7fe1115e75a1 out of bounds>}, {bv_len = 11, bv_val = 0x7fe11e1e98d2 <Address 0x7fe11e1e98d2 out of bounds>}, {bv_len = 13, bv_val = 0x7fe11e1e9e04 <Address 0x7fe11e1e9e04 out of bounds>} <repeats 2046 times>}, nrdns = {{bv_len = 13, bv_val = 0x7fe11e1e9e04 <Address 0x7fe11e1e9e04 out of bounds>} <repeats 2048 times>}} mci = 0xd mcd = 0x7fe11e1e9e04 opinfo = {moi_oe = {oe_next = {sle_next = 0xd}, oe_key = 0x7fe11e1e9e04}, moi_txn = 0xd, moi_ref = 505323012, moi_flag = -31 '\341'} moi = 0xd ltid = <optimized out>
(gdb) info registers rax 0xd 13 rbx 0x7fe0d8000c30 140603673283632 rcx 0x130c990 19974544 rdx 0x7fe11e1e9df4 140604849692148 rsi 0x7fe0fa7fe000 140604252086272 rdi 0x7fe108147010 140604479926288 rbp 0x7fe0f94faa00 0x7fe0f94faa00 rsp 0x7fe0f93695c0 0x7fe0f93695c0 r8 0x26 38 r9 0x130c97 1248407 r10 0xd 13 r11 0x7fe11e1e9df6 140604849692150 r12 0x7fe0f94e96d0 140604232079056 r13 0x7fe0f94f96d0 140604232144592 r14 0x7fe0ec1029c0 140604009884096 r15 0x7fe0f94f98b0 140604232145072 rip 0x7fe30c47ab9b 0x7fe30c47ab9b <mdb_search+7451> eflags 0x10206 [ PF IF RF ] cs 0x33 51 ss 0x2b 43 ds 0x0 0 es 0x0 0 fs 0x0 0 gs 0x0 0
(gdb) bt #0 0x00007fe30c4818b1 in mdb_idscopes (op=0x7fe0d8000c30, isc=0x7fe0f94e9670) at ../../../../../servers/slapd/back-mdb/dn2id.c:738 #1 0x00007fe30c47ab9b in mdb_search (op=0x7fe0d8000c30, rs=0x7fe0f94faa00) at ../../../../../servers/slapd/back-mdb/search.c:747 #2 0x00007fe11e1e9e04 in ?? () #3 0x000000000000000d in ?? () #4 0x00007fe11e1e9e04 in ?? () #5 0x000000000000000d in ?? () [..] #2492634 0x00007fe11e1e9df6 in ?? () #2492635 0x000000000000000d in ?? () #2492636 0x00007fe11e1e9df6 in ?? () #2492637 0x000000000000000d in ?? () #2492638 0x00007fe11e1e9df6 in ?? () #2492639 0x000000000000000d in ?? () #2492640 0x00007fe11e1e9df6 in ?? () #2492641 0x0000000000000000 in ?? ()