Full_Name: Ruan de Clercq
Version:
OS: Ubuntu 19.04
URL: ftp://ftp.openldap.org/incoming/
Submission from: (NULL) (155.93.214.171)
Hi,
I am using 0.9.23-0ubuntu1 on ubuntu 19.04.
I have an application where the exact db size isn't known beforehand. Therefore,
I set the memory map size beforehand, and then increase the size when I run out
of space.
This is incorrect usage of LMDB. The docs explicitly state to use a large size at
the beginning and leave it alone.
However, when I use lmdb in concurrent processes, and frequently
increase the size of the memory map, I eventually get a corrupt database
(MDB_CORRUPTED).
Your code is incorrect. The docs explicitly state that there must not be any outstanding
activity when using the set_mapsize call.
Closing this ITS.
#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <time.h>
#include "lmdb.h"
#include <cstring>
int resize(MDB_env *env) {
// read map size and increase
int val;
MDB_envinfo stat;
MDB_txn *txn;
if ((val = mdb_env_info(env, &stat) != MDB_SUCCESS) ||
(val = mdb_env_set_mapsize(env, stat.me_mapsize + 4096) != MDB_SUCCESS)
||
(val = mdb_env_info(env, &stat) != MDB_SUCCESS)) {
printf("set new mapsize %d\n", val);
return val;
}
printf("Resized to %lu\n", stat.me_mapsize);
if (((val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS)) ||
((val = mdb_txn_commit(txn)))) {
printf("txn %d\n", val);
}
return val;
}
int main(int argc,char * argv[])
{
timespec tp;
clock_gettime(CLOCK_MONOTONIC, &tp);
srand(tp.tv_sec + tp.tv_nsec);
MDB_env *env = NULL;
MDB_txn *txn = NULL;
MDB_dbi dbi;
int val;
if ((val = mdb_env_create(&env) != MDB_SUCCESS) ||
(val = mdb_env_set_maxreaders(env, 126) != MDB_SUCCESS)) {
printf("Create error %d", val);
goto out;
}
if ((val = mdb_env_set_mapsize(env, 4096 * 128) != MDB_SUCCESS) ||
(val = mdb_env_set_maxdbs(env, 1) != MDB_SUCCESS) ||
(val = mdb_env_open(env, "./data/", MDB_MAPASYNC | MDB_WRITEMAP, 0664) !=
MDB_SUCCESS) ||
(val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS) ||
(val = mdb_dbi_open(txn, "test", MDB_CREATE, &dbi) != MDB_SUCCESS) ||
(val = mdb_txn_commit(txn) != MDB_SUCCESS)) {
printf("create db %d\n", val);
goto out;
}
for (int i = 0; i < 1000; i++) {
char sval[32];
sprintf(sval, "%03x %d foo bar", 32, rand());
MDB_val mdb_key, mdb_val;
mdb_key.mv_size = strlen(sval);;
mdb_key.mv_data = sval;
mdb_val.mv_size = strlen(sval);
mdb_val.mv_data = sval;
if ((val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS)) {
printf("mdb_txn_begin %d\n", val);
goto out;
}
if ((val = mdb_put(txn, dbi, &mdb_key, &mdb_val, 0)) != MDB_SUCCESS) {
mdb_txn_commit(txn);
if (val == MDB_MAP_FULL) {
val = resize(env);
continue;
}
printf("mdb_put %d\n", val);
goto out;
}
if ((val = mdb_txn_commit(txn)) != MDB_SUCCESS) {
if (val == MDB_MAP_FULL) {
val = resize(env);
continue;
}
printf("mdb_txn_commit %d\n", val);
goto out;
}
}
out:
if (env) {
mdb_dbi_close(env, dbi);
mdb_env_close(env);
}
printf("OK\n");
}
After compiling I run the following command a couple of times:
for i in {1..20}; do echo $i; (lmdb_test &); done