wolfgang.hummel@hp.com wrote:

We had some of this crashes on LDAP slaves in the past:

This appears to be due to your custom template overlay. You probably need to check its correctness using a tool like valgrind or Purify.

In /var/log/messages 2010-12-05T11:22:57.643777+01:00 ts2mstsv001 kernel: 6>slapd[18900000000000= 0025 rip 0000003be707p 000000

Stack trace when crash occurred (search for entry 0xe932208 in back db (BD= B) response is contained a corrupted address) #0 0x0000003be7075b50 in strcpy () from /lib64/libc.so.6 #1 0x00002b5ffe3debeb in template_response (op=3D0xfa01aa0, rs=3D0x4813bc6= 0) at /usr/include/bits/string3.h:118 #2 0x00000000004a34eb in over_back_response (op=3D0xfa01aa0, rs=3D0x4813bc= 60) at ../servers/slapd/backover.c:237 #3 0x0000000000449865 in slap_response_play (op=3D0xfa01aa0, rs=3D0x4813bc= 60) at ../servers/slapd/result.c:402 #4 0x000000000044bfcc in slap_send_search_entry (op=3D0xfa01aa0, rs=3D0x48= 13bc60) at ../servers/slapd/result.c:887 #5 0x00000000004b695f in bdb_search (op=3D0xfa01aa0, rs=3D0x4813bc60) at s= ervers/slapd/back-bdb/search.c:961 #6 0x00000000004a37c2 in overlay_op_walk (op=3D0xfa01aa0, rs=3D0x4813bc60,= which=3Dop_search, oi=3D0xe5df160, on=3D0x0) at ../servers/slapd/backover.= c:669 #7 0x00000000004a3d58 in over_op_func (op=3D0xfa01aa0, rs=3D0x4813bc60, wh= ich=3Dop_search) at ../servers/slapd/backover.c:721 #8 0x000000000043c4e6 in fe_op_search (op=3D0xfa01aa0, rs=3D0x4813bc60) at= ../servers/slapd/search.c:376 #9 0x00000000004a37c2 in overlay_op_walk (op=3D0xfa01aa0, rs=3D0x4813bc60,= which=3Dop_search, oi=3D0xe577ec0, on=3D0x0) at ../servers/slapd/backover.= c:669 #10 0x00000000004a3d58 in over_op_func (op=3D0xfa01aa0, rs=3D0x4813bc60, wh= ich=3Dop_search) at ../servers/slapd/backover.c:721 #11 0x000000000043cc95 in do_search (op=3D0xfa01aa0, rs=3D0x4813bc60) at ..= /servers/slapd/search.c:227 #12 0x0000000000439ff4 in connection_operation (ctx=3D0x4813bdb0, arg_v=3D<= value optimized out>) at ../servers/slapd/connection.c:1109 #13 0x000000000043a651 in connection_read_thread (ctx=3D0x4813bdb0, argv=3D= <value optimized out>) at ../servers/slapd/connection.c:1245 #14 0x00000000005330a8 in ldap_int_thread_pool_wrapper (xpool=3D0xe546600) = at ../libraries/libldap_r/tpool.c:685 #15 0x0000003be7c062e7 in start_thread () from /lib64/libpthread.so.0 #16 0x0000003be70ce3bd in clone () from /lib64/libc.so.6

(gdb) fr 11 #11 0x000000000043cc95 in do_search (op=3D0xfa01aa0, rs=3D0x4813bc60) at ..= /servers/slapd/search.c:227 227 ../servers/slapd/search.c: No such file or directory. in ../servers/slapd/search.c (gdb) p * op $55 =3D {o_hdr =3D 0xfa01c10, o_tag =3D 99, o_time =3D 1291544577, o_tincr = =3D 140, o_bd =3D 0x47fb9ea0, o_req_dn =3D {bv_len =3D 41, bv_val =3D 0x102= 51e00 "ou=3Dcms,ou=3Dprofiles,ou=3Dmmo,c=3Dde,o=3Dvodafone"}, o_req_ndn =3D {bv_len =3D 41, bv_val =3D 0x10251ee0 "ou=3Dcms,ou=3Dprofil= es,ou=3Dmmo,c=3Dde,o=3Dvodafone"}, o_request =3D {oq_add =3D {rs_modlist = =3D 0x2, rs_e =3D 0x500000064}, oq_bind =3D {rb_method =3D 2, rb_cred =3D {bv_len =3D 21474836580, bv_val =3D 0x0}, rb_edn =3D {bv_= len =3D 0, bv_val =3D 0x10251fc0 "\020"}, rb_ssf =3D 270868336, rb_mech =3D= {bv_len =3D 18, bv_val =3D 0x10251f98 "(uid=3D491710471677)"}}, oq_compare =3D {rs_= ava =3D 0x2}, oq_modify =3D {rs_mods =3D {rs_modlist =3D 0x2, rs_no_opattrs= =3D 100 'd'}, rs_increment =3D 0}, oq_modrdn =3D { rs_mods =3D {rs_modlist =3D 0x2, rs_no_opattrs =3D 100 'd'}, rs_delet= eoldrdn =3D 0, rs_newrdn =3D {bv_len =3D 0, bv_val =3D 0x10251fc0 "\020"}, = rs_nnewrdn =3D {bv_len =3D 270868336, bv_val =3D 0x12<Address 0x12 out of bounds>}, rs_newSup =3D 0x1025= 1f98, rs_nnewSup =3D 0x0}, oq_search =3D {rs_scope =3D 2, rs_deref =3D 0, r= s_slimit =3D 100, rs_tlimit =3D 5, rs_limit =3D 0x0, rs_attrsonly =3D 0, rs_attrs =3D 0x10251fc0, rs_fil= ter =3D 0x10251f70, rs_filterstr =3D {bv_len =3D 18, bv_val =3D 0x10251f98 = "(uid=3D491710471677)"}}, oq_abandon =3D { rs_msgid =3D 2}, oq_cancel =3D {rs_msgid =3D 2}, oq_extended =3D {rs_= reqoid =3D {bv_len =3D 2, bv_val =3D 0x500000064<Address 0x500000064 out o= f bounds>}, rs_flags =3D 0, rs_reqdata =3D 0x0}, oq_pwdexop =3D {rs_extended =3D {rs_reqoid =3D {bv_len =3D 2, bv_val = =3D 0x500000064<Address 0x500000064 out of bounds>}, rs_flags =3D 0, rs_re= qdata =3D 0x0}, rs_old =3D {bv_len =3D 270868416, bv_val =3D 0x10251f70 "=A3"}, rs_new =3D {bv_len =3D 18, bv_val =3D= 0x10251f98 "(uid=3D491710471677)"}, rs_mods =3D 0x0, rs_modtail =3D 0x0}},= o_abandon =3D 0, o_cancel =3D 0, o_groups =3D 0x0, o_do_not_cache =3D 0 '\0', o_is_auth_check =3D 0 '\0', o_dont_replicate = =3D 0 '\0', o_acl_priv =3D ACL_NONE, o_nocaching =3D 0 '\0', o_delete_glue_= parent =3D 0 '\0', o_no_schema_check =3D 0 '\0', o_no_subordinate_glue =3D 0 '\0', o_ctrlflag =3D '\0'<repeats 31 times>,= o_controls =3D 0xfa01d58, o_authz =3D {sai_method =3D 128, sai_mech =3D {b= v_len =3D 0, bv_val =3D 0x0}, sai_dn =3D { bv_len =3D 51, bv_val =3D 0x2aaab89f0a50 "uid=3Dadmin,ou=3Dcms,ou=3Dp= rofiles,ou=3Dmmo,c=3Dde,o=3Dvodafone"}, sai_ndn =3D {bv_len =3D 51, bv_val =3D 0x2aaab8a04e20 "uid=3Dadmin,ou=3Dcms,ou=3Dprofiles,ou=3Dmm= o,c=3Dde,o=3Dvodafone"}, sai_ssf =3D 0, sai_transport_ssf =3D 0, sai_tls_ss= f =3D 0, sai_sasl_ssf =3D 0}, o_ber =3D 0x2aaac8098630, o_res_ber =3D 0x0, o_callback =3D 0x4813a740, o_ctrls =3D 0x0, o_csn =3D = {bv_len =3D 0, bv_val =3D 0x0}, o_private =3D 0x0, o_extra =3D {slh_first = =3D 0x4813a480}, o_next =3D {stqe_next =3D 0x0}} (gdb) p * rs $56 =3D {sr_type =3D REP_SEARCH, sr_tag =3D 0, sr_msgid =3D 0, sr_err =3D 0= , sr_matched =3D 0x0, sr_text =3D 0x0, sr_ref =3D 0x0, sr_ctrls =3D 0x0, sr= _un =3D {sru_search =3D {r_entry =3D 0xe932208, r_attr_flags =3D 17, r_operational_attrs =3D 0x0, r_attrs =3D 0x10251= fc0, r_nentries =3D 0, r_v2ref =3D 0x0}, sru_sasl =3D {r_sasldata =3D 0xe93= 2208}, sru_extended =3D { r_rspoid =3D 0xe932208 "\2002\a", r_rspdata =3D 0x11}}, sr_flags =3D = 4}

Frame analysis (function called) (gdb) fr 0 #0 0x0000003be7075b50 in strcpy () from /lib64/libc.so.6 (gdb) info registers rax 0x1 1 rbx 0x1 1 rcx 0x3 3 rdx 0x47f37648 1207137864 rsi 0x25 37 rdi 0x47f37648 1207137864 rbp 0x47f265e4 0x47f265e4 rsp 0x47a25518 0x47a25518 r8 0xfefefefefefefeff -72340172838076673 r9 0x4813bdd0 1209253328 r10 0x2aaab8000020 46912719814688 r11 0x206 518 r12 0xe5e2ae0 241052384 r13 0x0 0 r14 0xfa01aa0 262150816 r15 0x6 6 rip 0x3be7075b50 0x3be7075b50<strcpy+16> eflags 0x10217 [ CF PF AF IF RF ] cs 0x33 51 ss 0x2b 43 ds 0x0 0 es 0x0 0 fs 0x63 99 gs 0x0 0 (gdb) disas=20 Dump of assembler code for function strcpy: 0x0000003be7075b40<strcpy+0>: mov %rsi,%rcx 0x0000003be7075b43<strcpy+3>: and $0x7,%ecx 0x0000003be7075b46<strcpy+6>: mov %rdi,%rdx 0x0000003be7075b49<strcpy+9>: je 0x3be7075b66<strcpy+38> 0x0000003be7075b4b<strcpy+11>: neg %ecx 0x0000003be7075b4d<strcpy+13>: add $0x8,%ecx 0x0000003be7075b50<strcpy+16>: mov (%rsi),%al

rsi is a parameter of strcpy.=20 rsi should be an address but we have 0x25 which is an invalid address. So c= rash 2010-12-05T11:22:57.643777+01:00 ts2mstsv001 kernel: 6>slapd[189000000= 000000025 rip 0000003be707p 000000 rsi is not modified in strcpy, then, check rsi calculation in template_resp= onse

(gdb) fr 1 #1 0x00002b5ffe3debeb in template_response (op=3D0xfa01aa0, rs=3D0x4813bc6= 0) at /usr/include/bits/string3.h:118 118 return __builtin___strcpy_chk (__dest, __src, __bos (__dest)); (gdb) info registers rax 0x1 1 rbx 0x1 1 rcx 0x3 3 rdx 0x47f37648 1207137864 rsi 0x25 37 rdi 0x47f37648 1207137864 rbp 0x47f265e4 0x47f265e4 rsp 0x47a25520 0x47a25520 r8 0xfefefefefefefeff -72340172838076673 r9 0x4813bdd0 1209253328 r10 0x2aaab8000020 46912719814688 r11 0x206 518 r12 0xe5e2ae0 241052384 r13 0x0 0 r14 0xfa01aa0 262150816 r15 0x6 6 rip 0x2b5ffe3debeb 0x2b5ffe3debeb<template_response+3787> eflags 0x10217 [ CF PF AF IF RF ] cs 0x33 51 ss 0x2b 43 ds 0x0 0 es 0x0 0 fs 0x63 99 gs 0x0 0

(gdb) disass 0x00002b5ffe3debeb Dump of assembler code for function template_response: ... 0x00002b5ffe3deb86<template_response+3686>: lea 0x5010c4(%rsp),%rbp 0x00002b5ffe3deb8e<template_response+3694>: xor %r15d,%r15d 0x00002b5ffe3deb91<template_response+3697>: xor %r13d,%r13d 0x00002b5ffe3deb94<template_response+3700>: mov %rdx,0x30(%rsp) 0x00002b5ffe3deb99<template_response+3705>: mov %rcx,0x28(%rsp) 0x00002b5ffe3deb9e<template_response+3710>: jmp 0x2b5ffe3dec07<temp= late_response+3815> 0x00002b5ffe3deba0<template_response+3712>: mov 0x511900(%rsp),%rax 0x00002b5ffe3deba8<template_response+3720>: test %rax,%rax 0x00002b5ffe3debab<template_response+3723>: je 0x2b5ffe3debeb<temp= late_response+3787> 0x00002b5ffe3debad<template_response+3725>: mov 0x8(%rax),%rsi 0x00002b5ffe3debb1<template_response+3729>: test %rsi,%rsi 0x00002b5ffe3debb4<template_response+3732>: je 0x2b5ffe3debeb<temp= late_response+3787> 0x00002b5ffe3debb6<template_response+3734>: cmpq $0x7ff,(%rax) 0x00002b5ffe3debbd<template_response+3741>: ja 0x2b5ffe3df966<temp= late_response+7238> 0x00002b5ffe3debc3<template_response+3747>: movslq 0x512124(%rsp),%rdi 0x00002b5ffe3debcb<template_response+3755>: mov 0x28(%rsp),%rdx 0x00002b5ffe3debd0<template_response+3760>: lea 0x1(%rdi),%eax 0x00002b5ffe3debd3<template_response+3763>: shl $0xb,%rdi 0x00002b5ffe3debd7<template_response+3767>: lea 0x808(%rdi,%rdx,1),%= rdi 0x00002b5ffe3debdf<template_response+3775>: mov %eax,0x512124(%rsp) 0x00002b5ffe3debe6<template_response+3782>: callq 0x2b5ffe3dc688<strc= py@plt> 0x00002b5ffe3debeb<template_response+3787>: lea 0x1(%r13),%eax (gdb) p *(long **)($rsp+0x511900) $17 =3D (long *) 0x2aaab4859d00 (gdb) x/20x 0x2aaab4859d00 0x2aaab4859d00: 0x00000000 0x00000000 0x00000025 0x00000000

rsi is broken, not a valid address.

Wolfgang Hummel