Re: (ITS#8355) [lmdb] Issues with sorted duplicate DBs with cursor delete - openldap-bugs

24 Jan 2016


      This is a multi-part message in MIME format.
--------------010901030100040802060200
Content-Type: text/plain; charset=windows-1252; format=flowed
Content-Transfer-Encoding: 7bit
On Sunday, January 24, 2016 07:44 PM, Howard Chu wrote:
...
hhclaw.eb@gmail.com wrote:
...
Full_Name: H Law
Version: LMDB
OS: Linux
URL:
Submission from: (NULL) (42.2.241.129)
It seems that, after using a cursor delete during a cursor traversal 
on a dup
sort database, the cursor is in a strange state, when MDB_NEXT / 
MDB_NEXT_DUP
ceases to work properly while MDB_PREV /  MDB_PREV_DUP still functions.
Thanks for the report. Fixed now in git.
Thank you. It now works except for the case when the deleted record is 
just before the end of the database.
In this case the next call to cursor get with MDB_NEXT / MDB_NEXT_DUP 
will still hang the program, instead of returning not found (or other 
errors).  (It seems to be stuck at some wait state without heavy cpu usage).
I attach a version of mtest3.c modified to show the issue.
...
...
In particular, when MDB_NEXT or MDB_NEXT_DUP is called after cursor 
deletion, if
next key/value pair exists, the cursor will not advance, and got 
stuck by
returning the same record when MDB_NEXT or MDB_NEXT_DUP is called 
repeatly.  In
case there is no next record, the program was hang.
The following modified version of mtest3.c shows the issue.  I am 
testing this
on the latest commit 20dec1f69bf4860202c764ce92b1fbbe3d11a065 of lmdb 
on 20 Jan,
on x86-64 Linux.
I got a similar behaviour when a slightly earlier version of lmdb was
cross-compiled with a Java wrapper for use on Android, which is why I 
am testing
this.   The issue should therefore not be platform specific.
--------------010901030100040802060200
Content-Type: text/x-csrc;
 name="mtest3.c"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename="mtest3.c"
/* mtest3.c - memory-mapped database tester/toy */
/*
 * Copyright 2011-2015 Howard Chu, Symas Corp.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * http://www.OpenLDAP.org/license.html.
 */
/* Tests for sorted duplicate DBs with cursor delete */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "lmdb.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
    "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
int main(int argc,char * argv[])
{
    int i = 0, j = 0, rc;
    MDB_env *env;
    MDB_dbi dbi;
    MDB_val key, data;
    MDB_txn *txn;
    MDB_stat mst;
    MDB_cursor *cursor;
    int count;
    int *values;
    char sval[32];
    char kval[sizeof(int)];
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = 10;
    values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
               values[i]= i * 10;
    }
E(mdb_env_create(&env));
    E(mdb_env_set_mapsize(env, 10485760));
    E(mdb_env_set_maxdbs(env, 4));
    E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
    E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
key.mv_size = sizeof(int);
    key.mv_data = kval;
    data.mv_size = sizeof(sval);
    data.mv_data = sval;
printf("Adding %d values\n", count);
    for (i=0;i<count;i++) {
    	if (!(i & 0x07))
    		sprintf(kval, "%03x", values[i]);
    	sprintf(sval, "%03x %d foo bar", values[i], values[i]);
    	if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
    		j++;
    }
    if (j) printf("%d duplicates skipped\n", j);
    E(mdb_txn_commit(txn));
    E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
    E(mdb_cursor_open(txn, dbi, &cursor));
    while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
    	printf("key: %p %.*s, data: %p %.*s\n",
    		key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
    		data.mv_data, (int) data.mv_size, (char *) data.mv_data);
    }
    CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
    mdb_cursor_close(cursor);
    mdb_txn_abort(txn);
E(mdb_env_stat(env, &mst));
    E(mdb_txn_begin(env, NULL, 0, &txn));
    E(mdb_cursor_open(txn, dbi, &cursor));
        /*
         * There are 2 items with key = values[8] at the end of the list
         */
    sprintf(kval, "%03x", values[8]);
    key.mv_size = sizeof(int);
    key.mv_data = kval;
    printf("\nCursor set / first dup\n");
    E(mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY));
    E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST_DUP));
    printf("key: %.*s, data: %.*s\n",
    		(int) key.mv_size,  (char *) key.mv_data,
    		(int) data.mv_size, (char *) data.mv_data);
    printf("Cursor next\n");
    j=0;
        /*
         * The program will hang at the following mdb_cursor_get 
         * after cursor delete
         */
    while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP)) == 0) {
    	printf("key: %.*s, data: %.*s\n",
    		(int) key.mv_size,  (char *) key.mv_data,
    		(int) data.mv_size, (char *) data.mv_data);
                j++;
                if (j == 1) {
    	    printf("delete the above key/data\n");
                    if (RES(MDB_NOTFOUND, mdb_cursor_del(cursor, 0))) {
                    mdb_cursor_close(cursor);
    		mdb_txn_abort(txn);
                        break;
                    }
                }
                if (j > count) {
                   printf("Should not be there\n");
                   break;
                }
    }
        if (j > 1) {
        mdb_cursor_close(cursor);
        E(mdb_txn_commit(txn));
        }
E(mdb_txn_begin(env, NULL, 0, &txn));
    E(mdb_cursor_open(txn, dbi, &cursor));
    sprintf(kval, "%03x", values[0]);
    key.mv_size = sizeof(int);
    key.mv_data = kval;
    printf("\nCursor set / last dup\n");
    E(mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY));
    E(mdb_cursor_get(cursor, &key, &data, MDB_LAST_DUP));
    printf("key: %.*s, data: %.*s\n",
    		(int) key.mv_size,  (char *) key.mv_data,
    		(int) data.mv_size, (char *) data.mv_data);
printf("Cursor prev\n");
        j=0;
    while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV_DUP)) == 0) {
    	printf("key: %.*s, data: %.*s\n",
    		(int) key.mv_size,  (char *) key.mv_data,
    		(int) data.mv_size, (char *) data.mv_data);
                j++;
                if (j == 1) {
    	    printf("Delete the above key/data\n");
                    if (RES(MDB_NOTFOUND, mdb_cursor_del(cursor, 0))) {
                    mdb_cursor_close(cursor);
    		mdb_txn_abort(txn);
                        break;
                    }
                }
                if (j > count) {
                   printf("Should not be there\n");
                   break;
                }
    }
        if (j > 1) {
        mdb_cursor_close(cursor);
        E(mdb_txn_commit(txn));
        }
mdb_dbi_close(env, dbi);
    mdb_env_close(env);
        free (values);
    return 0;
}
--------------010901030100040802060200--