static int tdb_dump_chain(struct tdb_context *tdb, int i)
{
tdb_off_t rec_ptr, top;
if (i == -1) {
top = FREELIST_TOP;
} else {
top = TDB_HASH_TOP(i);
}
if (tdb_lock(tdb, i, F_WRLCK) != 0)
return -1;
if (tdb_ofs_read(tdb, top, &rec_ptr) == -1)
return tdb_unlock(tdb, i, F_WRLCK);
if (rec_ptr)
printf("hash=%d\n", i);
while (rec_ptr) {
rec_ptr = tdb_dump_record(tdb, i, rec_ptr);
}
return tdb_unlock(tdb, i, F_WRLCK);
}
/* traverse the entire database - calling fn(tdb, key, data) on each element.
return -1 on error or the record count traversed
if fn is NULL then it is not called
a non-zero return value from fn() indicates that the traversal should stop
*/
static int tdb_traverse_internal(struct tdb_context *tdb,
tdb_traverse_func fn, void *private_data,
struct tdb_traverse_lock *tl)
{
TDB_DATA key, dbuf;
struct list_struct rec;
int ret, count = 0;
/* This was in the initializaton, above, but the IRIX compiler
* did not like it. crh
*/
tl->next = tdb->travlocks.next;
/* fcntl locks don't stack: beware traverse inside traverse */
tdb->travlocks.next = tl;
/* tdb_next_lock places locks on the record returned, and its chain */
while ((ret = tdb_next_lock(tdb, tl, &rec)) > 0) {
count++;
/* now read the full record */
key.dptr = tdb_alloc_read(tdb, tl->off + sizeof(rec),
rec.key_len + rec.data_len);
if (!key.dptr) {
ret = -1;
if (tdb_unlock(tdb, tl->hash, tl->lock_rw) != 0)
goto out;
if (tdb_unlock_record(tdb, tl->off) != 0)
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_traverse: key.dptr == NULL and unlock_record failed!\n"));
goto out;
}
key.dsize = rec.key_len;
dbuf.dptr = key.dptr + rec.key_len;
dbuf.dsize = rec.data_len;
/* Drop chain lock, call out */
if (tdb_unlock(tdb, tl->hash, tl->lock_rw) != 0) {
ret = -1;
SAFE_FREE(key.dptr);
goto out;
}
if (fn && fn(tdb, key, dbuf, private_data)) {
/* They want us to terminate traversal */
ret = count;
if (tdb_unlock_record(tdb, tl->off) != 0) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_traverse: unlock_record failed!\n"));;
ret = -1;
}
SAFE_FREE(key.dptr);
goto out;
}
SAFE_FREE(key.dptr);
}
out:
tdb->travlocks.next = tl->next;
if (ret < 0)
return -1;
else
return count;
}
/* find the next entry in the database, returning its key */
TDB_DATA tdb_nextkey(struct tdb_context *tdb, TDB_DATA oldkey)
{
u32 oldhash;
TDB_DATA key = tdb_null;
struct list_struct rec;
char *k = NULL;
/* Is locked key the old key? If so, traverse will be reliable. */
if (tdb->travlocks.off) {
if (tdb_lock(tdb,tdb->travlocks.hash,tdb->travlocks.lock_rw))
return tdb_null;
if (tdb_rec_read(tdb, tdb->travlocks.off, &rec) == -1
|| !(k = tdb_alloc_read(tdb,tdb->travlocks.off+sizeof(rec),
rec.key_len))
|| memcmp(k, oldkey.dptr, oldkey.dsize) != 0) {
/* No, it wasn't: unlock it and start from scratch */
if (tdb_unlock_record(tdb, tdb->travlocks.off) != 0) {
SAFE_FREE(k);
return tdb_null;
}
if (tdb_unlock(tdb, tdb->travlocks.hash, tdb->travlocks.lock_rw) != 0) {
SAFE_FREE(k);
return tdb_null;
}
tdb->travlocks.off = 0;
}
SAFE_FREE(k);
}
if (!tdb->travlocks.off) {
/* No previous element: do normal find, and lock record */
tdb->travlocks.off = tdb_find_lock_hash(tdb, oldkey, tdb->hash_fn(&oldkey), tdb->travlocks.lock_rw, &rec);
if (!tdb->travlocks.off)
return tdb_null;
tdb->travlocks.hash = BUCKET(rec.full_hash);
if (tdb_lock_record(tdb, tdb->travlocks.off) != 0) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_nextkey: lock_record failed (%s)!\n", strerror(errno)));
return tdb_null;
}
}
oldhash = tdb->travlocks.hash;
/* Grab next record: locks chain and returned record,
unlocks old record */
if (tdb_next_lock(tdb, &tdb->travlocks, &rec) > 0) {
key.dsize = rec.key_len;
key.dptr = tdb_alloc_read(tdb, tdb->travlocks.off+sizeof(rec),
key.dsize);
/* Unlock the chain of this new record */
if (tdb_unlock(tdb, tdb->travlocks.hash, tdb->travlocks.lock_rw) != 0)
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_nextkey: WARNING tdb_unlock failed!\n"));
}
/* Unlock the chain of old record */
if (tdb_unlock(tdb, BUCKET(oldhash), tdb->travlocks.lock_rw) != 0)
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_nextkey: WARNING tdb_unlock failed!\n"));
return key;
}
/* delete an entry in the database given a key */
static int tdb_delete_hash(struct tdb_context *tdb, TDB_DATA key, u32 hash)
{
tdb_off_t rec_ptr;
struct list_struct rec;
int ret;
if (tdb->max_dead_records != 0) {
/*
* Allow for some dead records per hash chain, mainly for
* tdb's with a very high create/delete rate like locking.tdb.
*/
if (tdb_lock(tdb, BUCKET(hash), F_WRLCK) == -1)
return -1;
if (tdb_count_dead(tdb, hash) >= tdb->max_dead_records) {
/*
* Don't let the per-chain freelist grow too large,
* delete all existing dead records
*/
tdb_purge_dead(tdb, hash);
}
if (!(rec_ptr = tdb_find(tdb, key, hash, &rec))) {
tdb_unlock(tdb, BUCKET(hash), F_WRLCK);
return -1;
}
/*
* Just mark the record as dead.
*/
rec.magic = TDB_DEAD_MAGIC;
ret = tdb_rec_write(tdb, rec_ptr, &rec);
}
else {
if (!(rec_ptr = tdb_find_lock_hash(tdb, key, hash, F_WRLCK,
&rec)))
return -1;
ret = tdb_do_delete(tdb, rec_ptr, &rec);
}
if (ret == 0) {
tdb_increment_seqnum(tdb);
}
if (tdb_unlock(tdb, BUCKET(rec.full_hash), F_WRLCK) != 0)
TDB_LOG((tdb, TDB_DEBUG_WARNING, "tdb_delete: WARNING tdb_unlock failed!\n"));
return ret;
}
/* find the first entry in the database and return its key */
_PUBLIC_ TDB_DATA tdb_firstkey(struct tdb_context *tdb)
{
TDB_DATA key;
struct tdb_record rec;
tdb_off_t off;
/* release any old lock */
if (tdb_unlock_record(tdb, tdb->travlocks.off) != 0)
return tdb_null;
tdb->travlocks.off = tdb->travlocks.list = 0;
tdb->travlocks.lock_rw = F_RDLCK;
/* Grab first record: locks chain and returned record. */
off = tdb_next_lock(tdb, &tdb->travlocks, &rec);
if (off == 0 || off == TDB_NEXT_LOCK_ERR) {
tdb_trace_retrec(tdb, "tdb_firstkey", tdb_null);
return tdb_null;
}
/* now read the key */
key.dsize = rec.key_len;
key.dptr =tdb_alloc_read(tdb,tdb->travlocks.off+sizeof(rec),key.dsize);
tdb_trace_retrec(tdb, "tdb_firstkey", key);
/* Unlock the hash chain of the record we just read. */
if (tdb_unlock(tdb, tdb->travlocks.list, tdb->travlocks.lock_rw) != 0)
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_firstkey: error occurred while tdb_unlocking!\n"));
return key;
}
/* Append to an entry. Create if not exist. */
int tdb_append(struct tdb_context *tdb, TDB_DATA key, TDB_DATA new_dbuf)
{
u32 hash;
TDB_DATA dbuf;
int ret = -1;
/* find which hash bucket it is in */
hash = tdb->hash_fn(&key);
if (tdb_lock(tdb, BUCKET(hash), F_WRLCK) == -1)
return -1;
dbuf = tdb_fetch(tdb, key);
if (dbuf.dptr == NULL) {
dbuf.dptr = (char *)malloc(new_dbuf.dsize);
} else {
dbuf.dptr = (char *)realloc(dbuf.dptr,
dbuf.dsize + new_dbuf.dsize);
}
if (dbuf.dptr == NULL) {
tdb->ecode = TDB_ERR_OOM;
goto failed;
}
memcpy(dbuf.dptr + dbuf.dsize, new_dbuf.dptr, new_dbuf.dsize);
dbuf.dsize += new_dbuf.dsize;
ret = tdb_store(tdb, key, dbuf, 0);
failed:
tdb_unlock(tdb, BUCKET(hash), F_WRLCK);
SAFE_FREE(dbuf.dptr);
return ret;
}
/*
* Purge all DEAD records from a hash chain
*/
static int tdb_purge_dead(struct tdb_context *tdb, u32 hash)
{
int res = -1;
struct list_struct rec;
tdb_off_t rec_ptr;
if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
return -1;
}
/* read in the hash top */
if (tdb_ofs_read(tdb, TDB_HASH_TOP(hash), &rec_ptr) == -1)
goto fail;
while (rec_ptr) {
tdb_off_t next;
if (tdb_rec_read(tdb, rec_ptr, &rec) == -1) {
goto fail;
}
next = rec.next;
if (rec.magic == TDB_DEAD_MAGIC
&& tdb_do_delete(tdb, rec_ptr, &rec) == -1) {
goto fail;
}
rec_ptr = next;
}
res = 0;
fail:
tdb_unlock(tdb, -1, F_WRLCK);
return res;
}
_PUBLIC_ int tdb_parse_record(struct tdb_context *tdb, TDB_DATA key,
int (*parser)(TDB_DATA key, TDB_DATA data,
void *private_data),
void *private_data)
{
tdb_off_t rec_ptr;
struct tdb_record rec;
int ret;
uint32_t hash;
/* find which hash bucket it is in */
hash = tdb->hash_fn(&key);
if (!(rec_ptr = tdb_find_lock_hash(tdb,key,hash,F_RDLCK,&rec))) {
/* record not found */
tdb_trace_1rec_ret(tdb, "tdb_parse_record", key, -1);
tdb->ecode = TDB_ERR_NOEXIST;
return -1;
}
tdb_trace_1rec_ret(tdb, "tdb_parse_record", key, 0);
ret = tdb_parse_data(tdb, key, rec_ptr + sizeof(rec) + rec.key_len,
rec.data_len, parser, private_data);
tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
return ret;
}
/* check if an entry in the database exists
note that 1 is returned if the key is found and 0 is returned if not found
this doesn't match the conventions in the rest of this module, but is
compatible with gdbm
*/
static int tdb_exists_hash(struct tdb_context *tdb, TDB_DATA key, u32 hash)
{
struct list_struct rec;
if (tdb_find_lock_hash(tdb, key, hash, F_RDLCK, &rec) == 0)
return 0;
tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
return 1;
}
int tdb_printfreelist(struct tdb_context *tdb)
{
int ret;
long total_free = 0;
tdb_off_t offset, rec_ptr;
struct tdb_record rec;
if ((ret = tdb_lock(tdb, -1, F_WRLCK)) != 0)
return ret;
offset = FREELIST_TOP;
/* read in the freelist top */
if (tdb_ofs_read(tdb, offset, &rec_ptr) == -1) {
tdb_unlock(tdb, -1, F_WRLCK);
return 0;
}
printf("freelist top=[0x%08x]\n", rec_ptr );
while (rec_ptr) {
if (tdb->methods->tdb_read(tdb, rec_ptr, (char *)&rec,
sizeof(rec), DOCONV()) == -1) {
tdb_unlock(tdb, -1, F_WRLCK);
return -1;
}
if (rec.magic != TDB_FREE_MAGIC) {
printf("bad magic 0x%08x in free list\n", rec.magic);
tdb_unlock(tdb, -1, F_WRLCK);
return -1;
}
printf("entry offset=[0x%08x], rec.rec_len = [0x%08x (%d)] (end = 0x%08x)\n",
rec_ptr, rec.rec_len, rec.rec_len, rec_ptr + rec.rec_len);
total_free += rec.rec_len;
/* move to the next record */
rec_ptr = rec.next;
}
printf("total rec_len = [0x%08x (%d)]\n", (int)total_free,
(int)total_free);
return tdb_unlock(tdb, -1, F_WRLCK);
}
/* As tdb_find, but if you succeed, keep the lock */
tdb_off_t tdb_find_lock_hash(struct tdb_context *tdb, TDB_DATA key, u32 hash, int locktype,
struct list_struct *rec)
{
u32 rec_ptr;
if (tdb_lock(tdb, BUCKET(hash), locktype) == -1)
return 0;
if (!(rec_ptr = tdb_find(tdb, key, hash, rec)))
tdb_unlock(tdb, BUCKET(hash), locktype);
return rec_ptr;
}
/*
return the size of the freelist - used to decide if we should repack
*/
int tdb_freelist_size(struct tdb_context *tdb)
{
tdb_off_t ptr;
int count=0;
if (tdb_lock(tdb, -1, F_RDLCK) == -1) {
return -1;
}
ptr = FREELIST_TOP;
while (tdb_ofs_read(tdb, ptr, &ptr) == 0 && ptr != 0) {
count++;
}
tdb_unlock(tdb, -1, F_RDLCK);
return count;
}
/* If an entry doesn't exist tdb_err will be set to
* TDB_ERR_NOEXIST. If a key has no data attached
* then the TDB_DATA will have zero length but
* a non-zero pointer
*/
TDB_DATA tdb_fetch(struct tdb_context *tdb, TDB_DATA key)
{
tdb_off_t rec_ptr;
struct list_struct rec;
TDB_DATA ret;
u32 hash;
/* find which hash bucket it is in */
hash = tdb->hash_fn(&key);
if (!(rec_ptr = tdb_find_lock_hash(tdb,key,hash,F_RDLCK,&rec)))
return tdb_null;
ret.dptr = tdb_alloc_read(tdb, rec_ptr + sizeof(rec) + rec.key_len,
rec.data_len);
ret.dsize = rec.data_len;
tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
return ret;
}
/* find the first entry in the database and return its key */
TDB_DATA tdb_firstkey(struct tdb_context *tdb)
{
TDB_DATA key;
struct list_struct rec;
/* release any old lock */
if (tdb_unlock_record(tdb, tdb->travlocks.off) != 0)
return tdb_null;
tdb->travlocks.off = tdb->travlocks.hash = 0;
tdb->travlocks.lock_rw = F_RDLCK;
/* Grab first record: locks chain and returned record. */
if (tdb_next_lock(tdb, &tdb->travlocks, &rec) <= 0)
return tdb_null;
/* now read the key */
key.dsize = rec.key_len;
key.dptr =tdb_alloc_read(tdb,tdb->travlocks.off+sizeof(rec),key.dsize);
/* Unlock the hash chain of the record we just read. */
if (tdb_unlock(tdb, tdb->travlocks.hash, tdb->travlocks.lock_rw) != 0)
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_firstkey: error occurred while tdb_unlocking!\n"));
return key;
}
/*
* Purge all DEAD records from a hash chain
*/
int tdb_purge_dead(struct tdb_context *tdb, uint32_t hash)
{
int res = -1;
struct tdb_record rec;
tdb_off_t rec_ptr;
if (tdb_lock_nonblock(tdb, -1, F_WRLCK) == -1) {
/*
* Don't block the freelist if not strictly necessary
*/
return -1;
}
/* read in the hash top */
if (tdb_ofs_read(tdb, TDB_HASH_TOP(hash), &rec_ptr) == -1)
goto fail;
while (rec_ptr) {
tdb_off_t next;
if (tdb_rec_read(tdb, rec_ptr, &rec) == -1) {
goto fail;
}
next = rec.next;
if (rec.magic == TDB_DEAD_MAGIC
&& tdb_do_delete(tdb, rec_ptr, &rec) == -1) {
goto fail;
}
rec_ptr = next;
}
res = 0;
fail:
tdb_unlock(tdb, -1, F_WRLCK);
return res;
}
int tdb_parse_record(struct tdb_context *tdb, TDB_DATA key,
int (*parser)(TDB_DATA key, TDB_DATA data,
void *private_data),
void *private_data)
{
tdb_off_t rec_ptr;
struct list_struct rec;
int ret;
u32 hash;
/* find which hash bucket it is in */
hash = tdb->hash_fn(&key);
if (!(rec_ptr = tdb_find_lock_hash(tdb,key,hash,F_RDLCK,&rec))) {
return TDB_ERRCODE(TDB_ERR_NOEXIST, 0);
}
ret = tdb_parse_data(tdb, key, rec_ptr + sizeof(rec) + rec.key_len,
rec.data_len, parser, private_data);
tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
return ret;
}
/* traverse the entire database - calling fn(tdb, key, data) on each element.
return -1 on error or the record count traversed
if fn is NULL then it is not called
a non-zero return value from fn() indicates that the traversal should stop
*/
static int tdb_traverse_internal(struct tdb_context *tdb,
tdb_traverse_func fn, void *private_data,
struct tdb_traverse_lock *tl)
{
TDB_DATA key, dbuf;
struct tdb_record rec;
int ret = 0, count = 0;
tdb_off_t off;
size_t recbuf_len;
recbuf_len = 4096;
key.dptr = malloc(recbuf_len);
if (key.dptr == NULL) {
return -1;
}
/* This was in the initialization, above, but the IRIX compiler
* did not like it. crh
*/
tl->next = tdb->travlocks.next;
/* fcntl locks don't stack: beware traverse inside traverse */
tdb->travlocks.next = tl;
/* tdb_next_lock places locks on the record returned, and its chain */
while ((off = tdb_next_lock(tdb, tl, &rec)) != 0) {
tdb_len_t full_len;
int nread;
if (off == TDB_NEXT_LOCK_ERR) {
ret = -1;
goto out;
}
full_len = rec.key_len + rec.data_len;
if (full_len > recbuf_len) {
recbuf_len = full_len;
/*
* No realloc, we don't need the old data and thus can
* do without the memcpy
*/
free(key.dptr);
key.dptr = malloc(recbuf_len);
if (key.dptr == NULL) {
ret = -1;
if (tdb_unlock(tdb, tl->list, tl->lock_rw)
!= 0) {
goto out;
}
if (tdb_unlock_record(tdb, tl->off) != 0) {
TDB_LOG((tdb, TDB_DEBUG_FATAL,
"tdb_traverse: malloc "
"failed and unlock_record "
"failed!\n"));
}
goto out;
}
}
count++;
/* now read the full record */
nread = tdb->methods->tdb_read(tdb, tl->off + sizeof(rec),
key.dptr, full_len, 0);
if (nread == -1) {
ret = -1;
if (tdb_unlock(tdb, tl->list, tl->lock_rw) != 0)
goto out;
if (tdb_unlock_record(tdb, tl->off) != 0)
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_traverse: key.dptr == NULL and unlock_record failed!\n"));
goto out;
}
key.dsize = rec.key_len;
dbuf.dptr = key.dptr + rec.key_len;
dbuf.dsize = rec.data_len;
tdb_trace_1rec_retrec(tdb, "traverse", key, dbuf);
/* Drop chain lock, call out */
if (tdb_unlock(tdb, tl->list, tl->lock_rw) != 0) {
ret = -1;
goto out;
}
if (fn && fn(tdb, key, dbuf, private_data)) {
/* They want us to terminate traversal */
tdb_trace_ret(tdb, "tdb_traverse_end", count);
if (tdb_unlock_record(tdb, tl->off) != 0) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_traverse: unlock_record failed!\n"));;
ret = -1;
}
goto out;
}
}
tdb_trace(tdb, "tdb_traverse_end");
out:
SAFE_FREE(key.dptr);
tdb->travlocks.next = tl->next;
if (ret < 0)
return -1;
else
return count;
}
_PUBLIC_ int tdb_chainunlock_read(struct tdb_context *tdb, TDB_DATA key)
{
tdb_trace_1rec(tdb, "tdb_chainunlock_read", key);
return tdb_unlock(tdb, BUCKET(tdb->hash_fn(&key)), F_RDLCK);
}
/* expand the database at least size bytes by expanding the underlying
file and doing the mmap again if necessary */
int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
{
struct list_struct rec;
tdb_off_t offset;
if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n"));
return -1;
}
/* must know about any previous expansions by another process */
tdb->methods->tdb_oob(tdb, tdb->map_size + 1, 1);
/* always make room for at least 10 more records, and round
the database up to a multiple of the page size */
size = TDB_ALIGN(tdb->map_size + size*10, tdb->page_size) - tdb->map_size;
if (!(tdb->flags & TDB_INTERNAL))
tdb_munmap(tdb);
/*
* We must ensure the file is unmapped before doing this
* to ensure consistency with systems like OpenBSD where
* writes and mmaps are not consistent.
*/
/* expand the file itself */
if (!(tdb->flags & TDB_INTERNAL)) {
if (tdb->methods->tdb_expand_file(tdb, tdb->map_size, size) != 0)
goto fail;
}
tdb->map_size += size;
if (tdb->flags & TDB_INTERNAL) {
char *new_map_ptr = (char *)realloc(tdb->map_ptr,
tdb->map_size);
if (!new_map_ptr) {
tdb->map_size -= size;
goto fail;
}
tdb->map_ptr = new_map_ptr;
} else {
/*
* We must ensure the file is remapped before adding the space
* to ensure consistency with systems like OpenBSD where
* writes and mmaps are not consistent.
*/
/* We're ok if the mmap fails as we'll fallback to read/write */
tdb_mmap(tdb);
}
/* form a new freelist record */
memset(&rec,'\0',sizeof(rec));
rec.rec_len = size - sizeof(rec);
/* link it into the free list */
offset = tdb->map_size - size;
if (tdb_free(tdb, offset, &rec) == -1)
goto fail;
tdb_unlock(tdb, -1, F_WRLCK);
return 0;
fail:
tdb_unlock(tdb, -1, F_WRLCK);
return -1;
}
_PUBLIC_ int tdb_validate_freelist(struct tdb_context *tdb, int *pnum_entries)
{
struct tdb_context *mem_tdb = NULL;
struct tdb_record rec;
tdb_off_t rec_ptr, last_ptr;
int ret = -1;
*pnum_entries = 0;
mem_tdb = tdb_open("flval", tdb->hash_size,
TDB_INTERNAL, O_RDWR, 0600);
if (!mem_tdb) {
return -1;
}
if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
tdb_close(mem_tdb);
return 0;
}
last_ptr = FREELIST_TOP;
/* Store the FREELIST_TOP record. */
if (seen_insert(mem_tdb, last_ptr) == -1) {
tdb->ecode = TDB_ERR_CORRUPT;
ret = -1;
goto fail;
}
/* read in the freelist top */
if (tdb_ofs_read(tdb, FREELIST_TOP, &rec_ptr) == -1) {
goto fail;
}
while (rec_ptr) {
/* If we can't store this record (we've seen it
before) then the free list has a loop and must
be corrupt. */
if (seen_insert(mem_tdb, rec_ptr)) {
tdb->ecode = TDB_ERR_CORRUPT;
ret = -1;
goto fail;
}
if (tdb_rec_free_read(tdb, rec_ptr, &rec) == -1) {
goto fail;
}
/* move to the next record */
last_ptr = rec_ptr;
rec_ptr = rec.next;
*pnum_entries += 1;
}
ret = 0;
fail:
tdb_close(mem_tdb);
tdb_unlock(tdb, -1, F_WRLCK);
return ret;
}
int main(int argc, char *argv[])
{
struct tdb_context *tdb;
TDB_DATA key, orig_data, data;
uint32_t hashval;
tdb_off_t rec_ptr;
struct tdb_record rec;
int ret;
plan_tests(24);
tdb = tdb_open_ex("run-36-file.tdb", 1024, TDB_CLEAR_IF_FIRST,
O_CREAT|O_TRUNC|O_RDWR, 0600, &taplogctx, NULL);
ok1(tdb);
tdb->methods = &large_io_methods;
key.dsize = strlen("hi");
key.dptr = (void *)"hi";
orig_data.dsize = strlen("world");
orig_data.dptr = (void *)"world";
/* Enlarge the file (internally multiplies by 2). */
ret = tdb_expand(tdb, 1500000000);
#ifdef HAVE_INCOHERENT_MMAP
/* This can fail due to mmap failure on 32 bit systems. */
if (ret == -1) {
/* These should now fail. */
ok1(tdb_store(tdb, key, orig_data, TDB_INSERT) == -1);
data = tdb_fetch(tdb, key);
ok1(data.dptr == NULL);
ok1(tdb_traverse(tdb, test_traverse, &orig_data) == -1);
ok1(tdb_delete(tdb, key) == -1);
ok1(tdb_traverse(tdb, test_traverse, NULL) == -1);
/* Skip the rest... */
for (ret = 0; ret < 24 - 6; ret++)
ok1(1);
tdb_close(tdb);
return exit_status();
}
#endif
ok1(ret == 0);
/* Put an entry in, and check it. */
ok1(tdb_store(tdb, key, orig_data, TDB_INSERT) == 0);
data = tdb_fetch(tdb, key);
ok1(data.dsize == strlen("world"));
ok1(memcmp(data.dptr, "world", strlen("world")) == 0);
free(data.dptr);
/* That currently fills at the end, make sure that's true. */
hashval = tdb->hash_fn(&key);
rec_ptr = tdb_find_lock_hash(tdb, key, hashval, F_RDLCK, &rec);
ok1(rec_ptr);
ok1(rec_ptr > 2U*1024*1024*1024);
tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
/* Traverse must work. */
ok1(tdb_traverse(tdb, test_traverse, &orig_data) == 1);
/* Delete should work. */
ok1(tdb_delete(tdb, key) == 0);
ok1(tdb_traverse(tdb, test_traverse, NULL) == 0);
/* Transactions should work. */
ok1(tdb_transaction_start(tdb) == 0);
ok1(tdb_store(tdb, key, orig_data, TDB_INSERT) == 0);
data = tdb_fetch(tdb, key);
ok1(data.dsize == strlen("world"));
ok1(memcmp(data.dptr, "world", strlen("world")) == 0);
free(data.dptr);
ok1(tdb_transaction_commit(tdb) == 0);
ok1(tdb_traverse(tdb, test_traverse, &orig_data) == 1);
tdb_close(tdb);
return exit_status();
}
int tdb_chainunlock_read(struct tdb_context *tdb, TDB_DATA key)
{
return tdb_unlock(tdb, BUCKET(tdb->hash_fn(&key)), F_RDLCK);
}
/* unmark a chain as locked without actually locking it. Warning! use with great caution! */
int tdb_chainlock_unmark(struct tdb_context *tdb, TDB_DATA key)
{
return tdb_unlock(tdb, BUCKET(tdb->hash_fn(&key)), F_WRLCK | TDB_MARK_LOCK);
}
/* allocate some space from the free list. The offset returned points
to a unconnected tdb_record within the database with room for at
least length bytes of total data
0 is returned if the space could not be allocated
*/
tdb_off_t tdb_allocate(struct tdb_context *tdb, tdb_len_t length, struct tdb_record *rec)
{
tdb_off_t rec_ptr, last_ptr, newrec_ptr;
struct {
tdb_off_t rec_ptr, last_ptr;
tdb_len_t rec_len;
} bestfit;
float multiplier = 1.0;
if (tdb_lock(tdb, -1, F_WRLCK) == -1)
return 0;
/* over-allocate to reduce fragmentation */
length *= 1.25;
/* Extra bytes required for tailer */
length += sizeof(tdb_off_t);
length = TDB_ALIGN(length, TDB_ALIGNMENT);
again:
last_ptr = FREELIST_TOP;
/* read in the freelist top */
if (tdb_ofs_read(tdb, FREELIST_TOP, &rec_ptr) == -1)
goto fail;
bestfit.rec_ptr = 0;
bestfit.last_ptr = 0;
bestfit.rec_len = 0;
/*
this is a best fit allocation strategy. Originally we used
a first fit strategy, but it suffered from massive fragmentation
issues when faced with a slowly increasing record size.
*/
while (rec_ptr) {
if (tdb_rec_free_read(tdb, rec_ptr, rec) == -1) {
goto fail;
}
if (rec->rec_len >= length) {
if (bestfit.rec_ptr == 0 ||
rec->rec_len < bestfit.rec_len) {
bestfit.rec_len = rec->rec_len;
bestfit.rec_ptr = rec_ptr;
bestfit.last_ptr = last_ptr;
}
}
/* move to the next record */
last_ptr = rec_ptr;
rec_ptr = rec->next;
/* if we've found a record that is big enough, then
stop searching if its also not too big. The
definition of 'too big' changes as we scan
through */
if (bestfit.rec_len > 0 &&
bestfit.rec_len < length * multiplier) {
break;
}
/* this multiplier means we only extremely rarely
search more than 50 or so records. At 50 records we
accept records up to 11 times larger than what we
want */
multiplier *= 1.05;
}
if (bestfit.rec_ptr != 0) {
if (tdb_rec_free_read(tdb, bestfit.rec_ptr, rec) == -1) {
goto fail;
}
newrec_ptr = tdb_allocate_ofs(tdb, length, bestfit.rec_ptr,
rec, bestfit.last_ptr);
tdb_unlock(tdb, -1, F_WRLCK);
return newrec_ptr;
}
/* we didn't find enough space. See if we can expand the
database and if we can then try again */
if (tdb_expand(tdb, length + sizeof(*rec)) == 0)
goto again;
fail:
tdb_unlock(tdb, -1, F_WRLCK);
return 0;
}
/* Add an element into the freelist. Merge adjacent records if
necessary. */
int tdb_free(struct tdb_context *tdb, tdb_off_t offset, struct tdb_record *rec)
{
/* Allocation and tailer lock */
if (tdb_lock(tdb, -1, F_WRLCK) != 0)
return -1;
/* set an initial tailer, so if we fail we don't leave a bogus record */
if (update_tailer(tdb, offset, rec) != 0) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_free: update_tailer failed!\n"));
goto fail;
}
#if USE_RIGHT_MERGES
/* Look right first (I'm an Australian, dammit) */
if (offset + sizeof(*rec) + rec->rec_len + sizeof(*rec) <= tdb->map_size) {
tdb_off_t right = offset + sizeof(*rec) + rec->rec_len;
struct tdb_record r;
if (tdb->methods->tdb_read(tdb, right, &r, sizeof(r), DOCONV()) == -1) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_free: right read failed at %u\n", right));
goto left;
}
/* If it's free, expand to include it. */
if (r.magic == TDB_FREE_MAGIC) {
if (remove_from_freelist(tdb, right, r.next) == -1) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_free: right free failed at %u\n", right));
goto left;
}
rec->rec_len += sizeof(r) + r.rec_len;
if (update_tailer(tdb, offset, rec) == -1) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_free: update_tailer failed at %u\n", offset));
goto fail;
}
}
}
left:
#endif
/* Look left */
if (offset - sizeof(tdb_off_t) > TDB_DATA_START(tdb->header.hash_size)) {
tdb_off_t left = offset - sizeof(tdb_off_t);
struct tdb_record l;
tdb_off_t leftsize;
/* Read in tailer and jump back to header */
if (tdb_ofs_read(tdb, left, &leftsize) == -1) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_free: left offset read failed at %u\n", left));
goto update;
}
/* it could be uninitialised data */
if (leftsize == 0 || leftsize == TDB_PAD_U32) {
goto update;
}
left = offset - leftsize;
if (leftsize > offset ||
left < TDB_DATA_START(tdb->header.hash_size)) {
goto update;
}
/* Now read in the left record */
if (tdb->methods->tdb_read(tdb, left, &l, sizeof(l), DOCONV()) == -1) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_free: left read failed at %u (%u)\n", left, leftsize));
goto update;
}
/* If it's free, expand to include it. */
if (l.magic == TDB_FREE_MAGIC) {
/* we now merge the new record into the left record, rather than the other
way around. This makes the operation O(1) instead of O(n). This change
prevents traverse from being O(n^2) after a lot of deletes */
l.rec_len += sizeof(*rec) + rec->rec_len;
if (tdb_rec_write(tdb, left, &l) == -1) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_free: update_left failed at %u\n", left));
goto fail;
}
if (update_tailer(tdb, left, &l) == -1) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_free: update_tailer failed at %u\n", offset));
goto fail;
}
tdb_unlock(tdb, -1, F_WRLCK);
return 0;
}
}
update:
/* Now, prepend to free list */
rec->magic = TDB_FREE_MAGIC;
if (tdb_ofs_read(tdb, FREELIST_TOP, &rec->next) == -1 ||
tdb_rec_write(tdb, offset, rec) == -1 ||
tdb_ofs_write(tdb, FREELIST_TOP, &offset) == -1) {
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_free record write failed at offset=%d\n", offset));
goto fail;
}
/* And we're done. */
tdb_unlock(tdb, -1, F_WRLCK);
return 0;
fail:
tdb_unlock(tdb, -1, F_WRLCK);
return -1;
}
int tdb_chainunlock(struct tdb_context *tdb, TDB_DATA key)
{
tdb_trace_1rec(tdb, "tdb_chainunlock", key);
return tdb_unlock(tdb, BUCKET(tdb->hash_fn(&key)), F_WRLCK);
}
/* store an element in the database, replacing any existing element
with the same key
return 0 on success, -1 on failure
*/
int tdb_store(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, int flag)
{
struct list_struct rec;
u32 hash;
tdb_off_t rec_ptr;
char *p = NULL;
int ret = -1;
if (tdb->read_only || tdb->traverse_read) {
tdb->ecode = TDB_ERR_RDONLY;
return -1;
}
/* find which hash bucket it is in */
hash = tdb->hash_fn(&key);
if (tdb_lock(tdb, BUCKET(hash), F_WRLCK) == -1)
return -1;
/* check for it existing, on insert. */
if (flag == TDB_INSERT) {
if (tdb_exists_hash(tdb, key, hash)) {
tdb->ecode = TDB_ERR_EXISTS;
goto fail;
}
} else {
/* first try in-place update, on modify or replace. */
if (tdb_update_hash(tdb, key, hash, dbuf) == 0) {
goto done;
}
if (tdb->ecode == TDB_ERR_NOEXIST &&
flag == TDB_MODIFY) {
/* if the record doesn't exist and we are in TDB_MODIFY mode then
we should fail the store */
goto fail;
}
}
/* reset the error code potentially set by the tdb_update() */
tdb->ecode = TDB_SUCCESS;
/* delete any existing record - if it doesn't exist we don't
care. Doing this first reduces fragmentation, and avoids
coalescing with `allocated' block before it's updated. */
if (flag != TDB_INSERT)
tdb_delete_hash(tdb, key, hash);
/* Copy key+value *before* allocating free space in case malloc
fails and we are left with a dead spot in the tdb. */
if (!(p = (char *)malloc(key.dsize + dbuf.dsize))) {
tdb->ecode = TDB_ERR_OOM;
goto fail;
}
memcpy(p, key.dptr, key.dsize);
if (dbuf.dsize)
memcpy(p+key.dsize, dbuf.dptr, dbuf.dsize);
if (tdb->max_dead_records != 0) {
/*
* Allow for some dead records per hash chain, look if we can
* find one that can hold the new record. We need enough space
* for key, data and tailer. If we find one, we don't have to
* consult the central freelist.
*/
rec_ptr = tdb_find_dead(
tdb, hash, &rec,
key.dsize + dbuf.dsize + sizeof(tdb_off_t));
if (rec_ptr != 0) {
rec.key_len = key.dsize;
rec.data_len = dbuf.dsize;
rec.full_hash = hash;
rec.magic = TDB_MAGIC;
if (tdb_rec_write(tdb, rec_ptr, &rec) == -1
|| tdb->methods->tdb_write(
tdb, rec_ptr + sizeof(rec),
p, key.dsize + dbuf.dsize) == -1) {
goto fail;
}
goto done;
}
}
/*
* We have to allocate some space from the freelist, so this means we
* have to lock it. Use the chance to purge all the DEAD records from
* the hash chain under the freelist lock.
*/
if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
goto fail;
}
if ((tdb->max_dead_records != 0)
&& (tdb_purge_dead(tdb, hash) == -1)) {
tdb_unlock(tdb, -1, F_WRLCK);
goto fail;
}
/* we have to allocate some space */
rec_ptr = tdb_allocate(tdb, key.dsize + dbuf.dsize, &rec);
tdb_unlock(tdb, -1, F_WRLCK);
if (rec_ptr == 0) {
goto fail;
}
/* Read hash top into next ptr */
if (tdb_ofs_read(tdb, TDB_HASH_TOP(hash), &rec.next) == -1)
goto fail;
rec.key_len = key.dsize;
rec.data_len = dbuf.dsize;
rec.full_hash = hash;
rec.magic = TDB_MAGIC;
/* write out and point the top of the hash chain at it */
if (tdb_rec_write(tdb, rec_ptr, &rec) == -1
|| tdb->methods->tdb_write(tdb, rec_ptr+sizeof(rec), p, key.dsize+dbuf.dsize)==-1
|| tdb_ofs_write(tdb, TDB_HASH_TOP(hash), &rec_ptr) == -1) {
/* Need to tdb_unallocate() here */
goto fail;
}
done:
ret = 0;
fail:
if (ret == 0) {
tdb_increment_seqnum(tdb);
}
SAFE_FREE(p);
tdb_unlock(tdb, BUCKET(hash), F_WRLCK);
return ret;
}
/* Uses traverse lock: 0 = finish, TDB_NEXT_LOCK_ERR = error,
other = record offset */
static tdb_off_t tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tlock,
struct tdb_record *rec)
{
int want_next = (tlock->off != 0);
/* Lock each chain from the start one. */
for (; tlock->list < tdb->hash_size; tlock->list++) {
if (!tlock->off && tlock->list != 0) {
/* this is an optimisation for the common case where
the hash chain is empty, which is particularly
common for the use of tdb with ldb, where large
hashes are used. In that case we spend most of our
time in tdb_brlock(), locking empty hash chains.
To avoid this, we do an unlocked pre-check to see
if the hash chain is empty before starting to look
inside it. If it is empty then we can avoid that
hash chain. If it isn't empty then we can't believe
the value we get back, as we read it without a
lock, so instead we get the lock and re-fetch the
value below.
Notice that not doing this optimisation on the
first hash chain is critical. We must guarantee
that we have done at least one fcntl lock at the
start of a search to guarantee that memory is
coherent on SMP systems. If records are added by
others during the search then thats OK, and we
could possibly miss those with this trick, but we
could miss them anyway without this trick, so the
semantics don't change.
With a non-indexed ldb search this trick gains us a
factor of around 80 in speed on a linux 2.6.x
system (testing using ldbtest).
*/
tdb->methods->next_hash_chain(tdb, &tlock->list);
if (tlock->list == tdb->hash_size) {
continue;
}
}
if (tdb_lock(tdb, tlock->list, tlock->lock_rw) == -1)
return TDB_NEXT_LOCK_ERR;
/* No previous record? Start at top of chain. */
if (!tlock->off) {
if (tdb_ofs_read(tdb, TDB_HASH_TOP(tlock->list),
&tlock->off) == -1)
goto fail;
} else {
/* Otherwise unlock the previous record. */
if (tdb_unlock_record(tdb, tlock->off) != 0)
goto fail;
}
if (want_next) {
/* We have offset of old record: grab next */
if (tdb_rec_read(tdb, tlock->off, rec) == -1)
goto fail;
tlock->off = rec->next;
}
/* Iterate through chain */
while( tlock->off) {
tdb_off_t current;
if (tdb_rec_read(tdb, tlock->off, rec) == -1)
goto fail;
/* Detect infinite loops. From "Shlomi Yaakobovich" <*****@*****.**>. */
if (tlock->off == rec->next) {
tdb->ecode = TDB_ERR_CORRUPT;
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_next_lock: loop detected.\n"));
goto fail;
}
if (!TDB_DEAD(rec)) {
/* Woohoo: we found one! */
if (tdb_lock_record(tdb, tlock->off) != 0)
goto fail;
return tlock->off;
}
/* Try to clean dead ones from old traverses */
current = tlock->off;
tlock->off = rec->next;
if (!(tdb->read_only || tdb->traverse_read) &&
tdb_do_delete(tdb, current, rec) != 0)
goto fail;
}
tdb_unlock(tdb, tlock->list, tlock->lock_rw);
want_next = 0;
}
/* We finished iteration without finding anything */
tdb->ecode = TDB_SUCCESS;
return 0;
fail:
tlock->off = 0;
if (tdb_unlock(tdb, tlock->list, tlock->lock_rw) != 0)
TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_next_lock: On error unlock failed!\n"));
return TDB_NEXT_LOCK_ERR;
}
/* expand the database at least size bytes by expanding the underlying
file and doing the mmap again if necessary */
int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
{
struct tdb_record rec;
tdb_off_t offset, new_size, top_size, map_size;
if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n"));
return -1;
}
/* must know about any previous expansions by another process */
tdb->methods->tdb_oob(tdb, tdb->map_size + 1, 1);
/* limit size in order to avoid using up huge amounts of memory for
* in memory tdbs if an oddball huge record creeps in */
if (size > 100 * 1024) {
top_size = tdb->map_size + size * 2;
} else {
top_size = tdb->map_size + size * 100;
}
/* always make room for at least top_size more records, and at
least 25% more space. if the DB is smaller than 100MiB,
otherwise grow it by 10% only. */
if (tdb->map_size > 100 * 1024 * 1024) {
map_size = tdb->map_size * 1.10;
} else {
map_size = tdb->map_size * 1.25;
}
/* Round the database up to a multiple of the page size */
new_size = MAX(top_size, map_size);
size = TDB_ALIGN(new_size, tdb->page_size) - tdb->map_size;
if (!(tdb->flags & TDB_INTERNAL))
tdb_munmap(tdb);
/*
* We must ensure the file is unmapped before doing this
* to ensure consistency with systems like OpenBSD where
* writes and mmaps are not consistent.
*/
/* expand the file itself */
if (!(tdb->flags & TDB_INTERNAL)) {
if (tdb->methods->tdb_expand_file(tdb, tdb->map_size, size) != 0)
goto fail;
}
tdb->map_size += size;
if (tdb->flags & TDB_INTERNAL) {
char *new_map_ptr = (char *)realloc(tdb->map_ptr,
tdb->map_size);
if (!new_map_ptr) {
tdb->map_size -= size;
goto fail;
}
tdb->map_ptr = new_map_ptr;
} else {
/*
* We must ensure the file is remapped before adding the space
* to ensure consistency with systems like OpenBSD where
* writes and mmaps are not consistent.
*/
/* We're ok if the mmap fails as we'll fallback to read/write */
tdb_mmap(tdb);
}
/* form a new freelist record */
memset(&rec,'\0',sizeof(rec));
rec.rec_len = size - sizeof(rec);
/* link it into the free list */
offset = tdb->map_size - size;
if (tdb_free(tdb, offset, &rec) == -1)
goto fail;
tdb_unlock(tdb, -1, F_WRLCK);
return 0;
fail:
tdb_unlock(tdb, -1, F_WRLCK);
return -1;
}
