ruan.declercq@netronome.com wrote:

Full_Name: Ruan de Clercq Version: OS: Ubuntu 19.04 URL: ftp://ftp.openldap.org/incoming/ Submission from: (NULL) (155.93.214.171)

Hi,

I am using 0.9.23-0ubuntu1 on ubuntu 19.04.

I have an application where the exact db size isn't known beforehand. Therefore, I set the memory map size beforehand, and then increase the size when I run out of space.

This is incorrect usage of LMDB. The docs explicitly state to use a large size at the beginning and leave it alone.

However, when I use lmdb in concurrent processes, and frequently increase the size of the memory map, I eventually get a corrupt database (MDB_CORRUPTED).

Your code is incorrect. The docs explicitly state that there must not be any outstanding activity when using the set_mapsize call.

Closing this ITS.

Here's a minimal code example:

#include <stdio.h> #include <stdlib.h> #include <inttypes.h> #include <time.h> #include "lmdb.h" #include <cstring>

int resize(MDB_env *env) { // read map size and increase int val; MDB_envinfo stat; MDB_txn *txn; if ((val = mdb_env_info(env, &stat) != MDB_SUCCESS) || (val = mdb_env_set_mapsize(env, stat.me_mapsize + 4096) != MDB_SUCCESS) || (val = mdb_env_info(env, &stat) != MDB_SUCCESS)) { printf("set new mapsize %d\n", val); return val; } printf("Resized to %lu\n", stat.me_mapsize); if (((val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS)) || ((val = mdb_txn_commit(txn)))) { printf("txn %d\n", val); } return val; }

int main(int argc,char * argv[]) { timespec tp; clock_gettime(CLOCK_MONOTONIC, &tp); srand(tp.tv_sec + tp.tv_nsec);

MDB_env *env = NULL; MDB_txn *txn = NULL; MDB_dbi dbi; int val; if ((val = mdb_env_create(&env) != MDB_SUCCESS) || (val = mdb_env_set_maxreaders(env, 126) != MDB_SUCCESS)) { printf("Create error %d", val); goto out; }

if ((val = mdb_env_set_mapsize(env, 4096 * 128) != MDB_SUCCESS) || (val = mdb_env_set_maxdbs(env, 1) != MDB_SUCCESS) || (val = mdb_env_open(env, "./data/", MDB_MAPASYNC | MDB_WRITEMAP, 0664) != MDB_SUCCESS) || (val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS) || (val = mdb_dbi_open(txn, "test", MDB_CREATE, &dbi) != MDB_SUCCESS) || (val = mdb_txn_commit(txn) != MDB_SUCCESS)) { printf("create db %d\n", val); goto out; }

for (int i = 0; i < 1000; i++) { char sval[32]; sprintf(sval, "%03x %d foo bar", 32, rand()); MDB_val mdb_key, mdb_val; mdb_key.mv_size = strlen(sval);; mdb_key.mv_data = sval; mdb_val.mv_size = strlen(sval); mdb_val.mv_data = sval;

if ((val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS)) {
  printf("mdb_txn_begin %d\n", val);
  goto out;
}
if ((val = mdb_put(txn, dbi, &mdb_key, &mdb_val, 0)) != MDB_SUCCESS) {
  mdb_txn_commit(txn);
  if (val == MDB_MAP_FULL) {
    val = resize(env);
    continue;
  }
  printf("mdb_put %d\n", val);
  goto out;
}
if ((val = mdb_txn_commit(txn)) != MDB_SUCCESS) {
  if (val == MDB_MAP_FULL) {
    val = resize(env);
    continue;
  }
  printf("mdb_txn_commit %d\n", val);
  goto out;
}

} out: if (env) { mdb_dbi_close(env, dbi); mdb_env_close(env); } printf("OK\n"); }

After compiling I run the following command a couple of times: for i in {1..20}; do echo $i; (lmdb_test &); done