https://bugs.openldap.org/show_bug.cgi?id=10537
Issue ID: 10537 Summary: Short writes not handled, causing error on commit Product: LMDB Version: 0.9.35 Hardware: x86_64 OS: Linux Status: UNCONFIRMED Keywords: needs_review Severity: normal Priority: --- Component: liblmdb Assignee: bugs@openldap.org Reporter: github@nicwatson.org Target Milestone: ---
The default max value length in LMDB is MAXDATASIZE == 4GiB - 1. The maximum single write/pwrite on Linux is 0x7ffff000 (2GiB - 4096) (see https://man7.org/linux/man-pages/man2/write.2.html). Writes larger than that return EIO.
When an attempt is to commit a transaction with a value with length larger than 0x7ffff000 and less than 4 GiB, the commit fails with rc=5.
The current code bails on EIO when it needs to loop and continue to write.
Here's a reproducer: ---
#include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/stat.h> #include "lmdb.h"
#define CHK(expr) do { int rc_ = (expr); if (rc_) { \ fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, #expr, \ mdb_strerror(rc_)); return 2; } } while (0)
int main(int argc, char **argv) { const char *dir = argc > 1 ? argv[1] : "./lw-repro-db"; /* 2.25 GiB: one overflow page whose single write exceeds the ~2 GiB * per-call limit (0x7ffff000). */ const size_t VALSIZE = (size_t)0x90000000UL; MDB_env *env; MDB_txn *txn; MDB_dbi dbi; MDB_val key, val, got; char *buf; int rc;
mkdir(dir, 0755);
buf = (char *)malloc(VALSIZE); if (!buf) { fprintf(stderr, "malloc(%zu) failed\n", VALSIZE); return 2; } memset(buf, 'x', VALSIZE); memcpy(buf, "HEAD", 4); memcpy(buf + VALSIZE - 4, "TAIL", 4);
CHK(mdb_env_create(&env)); CHK(mdb_env_set_mapsize(env, VALSIZE + (512UL << 20))); CHK(mdb_env_open(env, dir, 0, 0664));
CHK(mdb_txn_begin(env, NULL, 0, &txn)); CHK(mdb_dbi_open(txn, NULL, 0, &dbi)); key.mv_data = (void *)"big"; key.mv_size = 3; val.mv_data = buf; val.mv_size = VALSIZE; CHK(mdb_put(txn, dbi, &key, &val, 0));
rc = mdb_txn_commit(txn); if (rc) { fprintf(stderr, "BUG REPRODUCED: mdb_txn_commit failed: %s (rc=%d)\n", mdb_strerror(rc), rc); mdb_env_close(env); free(buf); return 1; }
/* Verify the value round-trips without truncation. */ CHK(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); CHK(mdb_get(txn, dbi, &key, &got)); if (got.mv_size != VALSIZE || memcmp(got.mv_data, "HEAD", 4) != 0 || memcmp((char *)got.mv_data + VALSIZE - 4, "TAIL", 4) != 0) { fprintf(stderr, "BUG: value corrupt/truncated: got %zu of %zu bytes\n", got.mv_size, VALSIZE); mdb_txn_abort(txn); mdb_env_close(env); free(buf); return 1; } mdb_txn_abort(txn); mdb_env_close(env); free(buf);
printf("OK: committed and read back %zu-byte value intact\n", VALSIZE); return 0; }