We had some of this crashes on LDAP slaves in the past:
In /var/log/messages
2010-12-05T11:22:57.643777+01:00 ts2mstsv001 kernel: 6>slapd[18900000000000=
0025 rip 0000003be707p 000000
Stack trace when crash occurred (search for entry 0xe932208 in back db (BD=
B) response is contained a corrupted address)
#0 0x0000003be7075b50 in strcpy () from /lib64/libc.so.6
#1 0x00002b5ffe3debeb in template_response (op=3D0xfa01aa0, rs=3D0x4813bc6=
0) at /usr/include/bits/string3.h:118
#2 0x00000000004a34eb in over_back_response (op=3D0xfa01aa0, rs=3D0x4813bc=
60) at ../servers/slapd/backover.c:237
#3 0x0000000000449865 in slap_response_play (op=3D0xfa01aa0, rs=3D0x4813bc=
60) at ../servers/slapd/result.c:402
#4 0x000000000044bfcc in slap_send_search_entry (op=3D0xfa01aa0, rs=3D0x48=
13bc60) at ../servers/slapd/result.c:887
#5 0x00000000004b695f in bdb_search (op=3D0xfa01aa0, rs=3D0x4813bc60) at s=
ervers/slapd/back-bdb/search.c:961
#6 0x00000000004a37c2 in overlay_op_walk (op=3D0xfa01aa0, rs=3D0x4813bc60,=
which=3Dop_search, oi=3D0xe5df160, on=3D0x0) at ../servers/slapd/backover.=
c:669
#7 0x00000000004a3d58 in over_op_func (op=3D0xfa01aa0, rs=3D0x4813bc60, wh=
ich=3Dop_search) at ../servers/slapd/backover.c:721
#8 0x000000000043c4e6 in fe_op_search (op=3D0xfa01aa0, rs=3D0x4813bc60) at=
../servers/slapd/search.c:376
#9 0x00000000004a37c2 in overlay_op_walk (op=3D0xfa01aa0, rs=3D0x4813bc60,=
which=3Dop_search, oi=3D0xe577ec0, on=3D0x0) at ../servers/slapd/backover.=
c:669
#10 0x00000000004a3d58 in over_op_func (op=3D0xfa01aa0, rs=3D0x4813bc60, wh=
ich=3Dop_search) at ../servers/slapd/backover.c:721
#11 0x000000000043cc95 in do_search (op=3D0xfa01aa0, rs=3D0x4813bc60) at ..=
/servers/slapd/search.c:227
#12 0x0000000000439ff4 in connection_operation (ctx=3D0x4813bdb0, arg_v=3D<=
value optimized out>) at ../servers/slapd/connection.c:1109
#13 0x000000000043a651 in connection_read_thread (ctx=3D0x4813bdb0, argv=3D=
<value optimized out>) at ../servers/slapd/connection.c:1245
#14 0x00000000005330a8 in ldap_int_thread_pool_wrapper (xpool=3D0xe546600) =
at ../libraries/libldap_r/tpool.c:685
#15 0x0000003be7c062e7 in start_thread () from /lib64/libpthread.so.0
#16 0x0000003be70ce3bd in clone () from /lib64/libc.so.6
(gdb) fr 11
#11 0x000000000043cc95 in do_search (op=3D0xfa01aa0, rs=3D0x4813bc60) at ..=
/servers/slapd/search.c:227
227 ../servers/slapd/search.c: No such file or directory.
in ../servers/slapd/search.c
(gdb) p * op
$55 =3D {o_hdr =3D 0xfa01c10, o_tag =3D 99, o_time =3D 1291544577, o_tincr =
=3D 140, o_bd =3D 0x47fb9ea0, o_req_dn =3D {bv_len =3D 41, bv_val =3D 0x102=
51e00 "ou=3Dcms,ou=3Dprofiles,ou=3Dmmo,c=3Dde,o=3Dvodafone"},
o_req_ndn =3D {bv_len =3D 41, bv_val =3D 0x10251ee0 "ou=3Dcms,ou=3Dprofil=
es,ou=3Dmmo,c=3Dde,o=3Dvodafone"}, o_request =3D {oq_add =3D {rs_modlist =
=3D 0x2, rs_e =3D 0x500000064}, oq_bind =3D {rb_method =3D 2,
rb_cred =3D {bv_len =3D 21474836580, bv_val =3D 0x0}, rb_edn =3D {bv_=
len =3D 0, bv_val =3D 0x10251fc0 "\020"}, rb_ssf =3D 270868336, rb_mech =3D=
{bv_len =3D 18,
bv_val =3D 0x10251f98 "(uid=3D491710471677)"}}, oq_compare =3D {rs_=
ava =3D 0x2}, oq_modify =3D {rs_mods =3D {rs_modlist =3D 0x2, rs_no_opattrs=
=3D 100 'd'}, rs_increment =3D 0}, oq_modrdn =3D {
rs_mods =3D {rs_modlist =3D 0x2, rs_no_opattrs =3D 100 'd'}, rs_delet=
eoldrdn =3D 0, rs_newrdn =3D {bv_len =3D 0, bv_val =3D 0x10251fc0 "\020"}, =
rs_nnewrdn =3D {bv_len =3D 270868336,
bv_val =3D 0x12 <Address 0x12 out of bounds>}, rs_newSup =3D 0x1025=
1f98, rs_nnewSup =3D 0x0}, oq_search =3D {rs_scope =3D 2, rs_deref =3D 0, r=
s_slimit =3D 100, rs_tlimit =3D 5,
rs_limit =3D 0x0, rs_attrsonly =3D 0, rs_attrs =3D 0x10251fc0, rs_fil=
ter =3D 0x10251f70, rs_filterstr =3D {bv_len =3D 18, bv_val =3D 0x10251f98 =
"(uid=3D491710471677)"}}, oq_abandon =3D {
rs_msgid =3D 2}, oq_cancel =3D {rs_msgid =3D 2}, oq_extended =3D {rs_=
reqoid =3D {bv_len =3D 2, bv_val =3D 0x500000064 <Address 0x500000064 out o=
f bounds>}, rs_flags =3D 0, rs_reqdata =3D 0x0},
oq_pwdexop =3D {rs_extended =3D {rs_reqoid =3D {bv_len =3D 2, bv_val =
=3D 0x500000064 <Address 0x500000064 out of bounds>}, rs_flags =3D 0, rs_re=
qdata =3D 0x0}, rs_old =3D {bv_len =3D 270868416,
bv_val =3D 0x10251f70 "=A3"}, rs_new =3D {bv_len =3D 18, bv_val =3D=
0x10251f98 "(uid=3D491710471677)"}, rs_mods =3D 0x0, rs_modtail =3D 0x0}},=
o_abandon =3D 0, o_cancel =3D 0, o_groups =3D 0x0,
o_do_not_cache =3D 0 '\0', o_is_auth_check =3D 0 '\0', o_dont_replicate =
=3D 0 '\0', o_acl_priv =3D ACL_NONE, o_nocaching =3D 0 '\0', o_delete_glue_=
parent =3D 0 '\0', o_no_schema_check =3D 0 '\0',
o_no_subordinate_glue =3D 0 '\0', o_ctrlflag =3D '\0' <repeats 31 times>,=
o_controls =3D 0xfa01d58, o_authz =3D {sai_method =3D 128, sai_mech =3D {b=
v_len =3D 0, bv_val =3D 0x0}, sai_dn =3D {
bv_len =3D 51, bv_val =3D 0x2aaab89f0a50 "uid=3Dadmin,ou=3Dcms,ou=3Dp=
rofiles,ou=3Dmmo,c=3Dde,o=3Dvodafone"}, sai_ndn =3D {bv_len =3D 51,
bv_val =3D 0x2aaab8a04e20 "uid=3Dadmin,ou=3Dcms,ou=3Dprofiles,ou=3Dmm=
o,c=3Dde,o=3Dvodafone"}, sai_ssf =3D 0, sai_transport_ssf =3D 0, sai_tls_ss=
f =3D 0, sai_sasl_ssf =3D 0}, o_ber =3D 0x2aaac8098630,
o_res_ber =3D 0x0, o_callback =3D 0x4813a740, o_ctrls =3D 0x0, o_csn =3D =
{bv_len =3D 0, bv_val =3D 0x0}, o_private =3D 0x0, o_extra =3D {slh_first =
=3D 0x4813a480}, o_next =3D {stqe_next =3D 0x0}}
(gdb) p * rs
$56 =3D {sr_type =3D REP_SEARCH, sr_tag =3D 0, sr_msgid =3D 0, sr_err =3D 0=
, sr_matched =3D 0x0, sr_text =3D 0x0, sr_ref =3D 0x0, sr_ctrls =3D 0x0, sr=
_un =3D {sru_search =3D {r_entry =3D 0xe932208,
r_attr_flags =3D 17, r_operational_attrs =3D 0x0, r_attrs =3D 0x10251=
fc0, r_nentries =3D 0, r_v2ref =3D 0x0}, sru_sasl =3D {r_sasldata =3D 0xe93=
2208}, sru_extended =3D {
r_rspoid =3D 0xe932208 "\2002\a", r_rspdata =3D 0x11}}, sr_flags =3D =
4}
Frame analysis (function called)
(gdb) fr 0
#0 0x0000003be7075b50 in strcpy () from /lib64/libc.so.6
(gdb) info registers
rax 0x1 1
rbx 0x1 1
rcx 0x3 3
rdx 0x47f37648 1207137864
rsi 0x25 37
rdi 0x47f37648 1207137864
rbp 0x47f265e4 0x47f265e4
rsp 0x47a25518 0x47a25518
r8 0xfefefefefefefeff -72340172838076673
r9 0x4813bdd0 1209253328
r10 0x2aaab8000020 46912719814688
r11 0x206 518
r12 0xe5e2ae0 241052384
r13 0x0 0
r14 0xfa01aa0 262150816
r15 0x6 6
rip 0x3be7075b50 0x3be7075b50 <strcpy+16>
eflags 0x10217 [ CF PF AF IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x63 99
gs 0x0 0
(gdb) disas=20
Dump of assembler code for function strcpy:
0x0000003be7075b40 <strcpy+0>: mov %rsi,%rcx
0x0000003be7075b43 <strcpy+3>: and $0x7,%ecx
0x0000003be7075b46 <strcpy+6>: mov %rdi,%rdx
0x0000003be7075b49 <strcpy+9>: je 0x3be7075b66 <strcpy+38>
0x0000003be7075b4b <strcpy+11>: neg %ecx
0x0000003be7075b4d <strcpy+13>: add $0x8,%ecx
0x0000003be7075b50 <strcpy+16>: mov (%rsi),%al
rsi is a parameter of strcpy.=20
rsi should be an address but we have 0x25 which is an invalid address. So c=
rash 2010-12-05T11:22:57.643777+01:00 ts2mstsv001 kernel: 6>slapd[189000000=
000000025 rip 0000003be707p 000000
rsi is not modified in strcpy, then, check rsi calculation in template_resp=
onse
(gdb) fr 1
#1 0x00002b5ffe3debeb in template_response (op=3D0xfa01aa0, rs=3D0x4813bc6=
0) at /usr/include/bits/string3.h:118
118 return __builtin___strcpy_chk (__dest, __src, __bos (__dest));
(gdb) info registers
rax 0x1 1
rbx 0x1 1
rcx 0x3 3
rdx 0x47f37648 1207137864
rsi 0x25 37
rdi 0x47f37648 1207137864
rbp 0x47f265e4 0x47f265e4
rsp 0x47a25520 0x47a25520
r8 0xfefefefefefefeff -72340172838076673
r9 0x4813bdd0 1209253328
r10 0x2aaab8000020 46912719814688
r11 0x206 518
r12 0xe5e2ae0 241052384
r13 0x0 0
r14 0xfa01aa0 262150816
r15 0x6 6
rip 0x2b5ffe3debeb 0x2b5ffe3debeb <template_response+3787>
eflags 0x10217 [ CF PF AF IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x63 99
gs 0x0 0
(gdb) disass 0x00002b5ffe3debeb
Dump of assembler code for function template_response:
...
0x00002b5ffe3deb86 <template_response+3686>: lea 0x5010c4(%rsp),%rbp
0x00002b5ffe3deb8e <template_response+3694>: xor %r15d,%r15d
0x00002b5ffe3deb91 <template_response+3697>: xor %r13d,%r13d
0x00002b5ffe3deb94 <template_response+3700>: mov %rdx,0x30(%rsp)
0x00002b5ffe3deb99 <template_response+3705>: mov %rcx,0x28(%rsp)
0x00002b5ffe3deb9e <template_response+3710>: jmp 0x2b5ffe3dec07 <temp=
late_response+3815>
0x00002b5ffe3deba0 <template_response+3712>: mov 0x511900(%rsp),%rax
0x00002b5ffe3deba8 <template_response+3720>: test %rax,%rax
0x00002b5ffe3debab <template_response+3723>: je 0x2b5ffe3debeb <temp=
late_response+3787>
0x00002b5ffe3debad <template_response+3725>: mov 0x8(%rax),%rsi
0x00002b5ffe3debb1 <template_response+3729>: test %rsi,%rsi
0x00002b5ffe3debb4 <template_response+3732>: je 0x2b5ffe3debeb <temp=
late_response+3787>
0x00002b5ffe3debb6 <template_response+3734>: cmpq $0x7ff,(%rax)
0x00002b5ffe3debbd <template_response+3741>: ja 0x2b5ffe3df966 <temp=
late_response+7238>
0x00002b5ffe3debc3 <template_response+3747>: movslq 0x512124(%rsp),%rdi
0x00002b5ffe3debcb <template_response+3755>: mov 0x28(%rsp),%rdx
0x00002b5ffe3debd0 <template_response+3760>: lea 0x1(%rdi),%eax
0x00002b5ffe3debd3 <template_response+3763>: shl $0xb,%rdi
0x00002b5ffe3debd7 <template_response+3767>: lea 0x808(%rdi,%rdx,1),%=
rdi
0x00002b5ffe3debdf <template_response+3775>: mov %eax,0x512124(%rsp)
0x00002b5ffe3debe6 <template_response+3782>: callq 0x2b5ffe3dc688 <strc=
py@plt>
0x00002b5ffe3debeb <template_response+3787>: lea 0x1(%r13),%eax
(gdb) p *(long **)($rsp+0x511900)
$17 =3D (long *) 0x2aaab4859d00
(gdb) x/20x 0x2aaab4859d00
0x2aaab4859d00: 0x00000000 0x00000000 0x00000025 0x00000000
rsi is broken, not a valid address.
Wolfgang Hummel
