/**
* estimate the number of keys per page, given the keysize
*
* @remark this function is only available when
* hamsterdb is compiled with HAM_ENABLE_INTERNAL turned on.
*
* @note This is a B+-tree 'backend' method.
*/
static ham_status_t
my_fun_calc_keycount_per_page(ham_btree_t *be, ham_size_t *maxkeys,
ham_u16_t keysize)
{
ham_db_t *db=be_get_db(be);
if (keysize == 0) {
*maxkeys=btree_get_maxkeys(be);
}
else {
/*
* prevent overflow - maxkeys only has 16 bit!
*/
*maxkeys=btree_calc_maxkeys(env_get_pagesize(db_get_env(db)), keysize);
if (*maxkeys>MAX_KEYS_PER_NODE) {
ham_trace(("keysize/pagesize ratio too high"));
return HAM_INV_KEYSIZE;
}
else if (*maxkeys==0) {
ham_trace(("keysize too large for the current pagesize"));
return HAM_INV_KEYSIZE;
}
}
return (0);
}
ham_status_t
cache_check_integrity(ham_cache_t *cache)
{
#ifdef HAM_ENABLE_INTERNAL
ham_size_t elements=0;
ham_page_t *head;
/* count the cached pages */
head=cache_get_totallist(cache);
while (head) {
elements++;
head=page_get_next(head, PAGE_LIST_CACHED);
}
head=cache_get_garbagelist(cache);
while (head) {
elements++;
head=page_get_next(head, PAGE_LIST_GARBAGE);
}
/* did we count the correct numbers? */
if (cache_get_cur_elements(cache)!=elements) {
ham_trace(("cache's number of elements (%u) != actual number (%u)",
cache_get_cur_elements(cache), elements));
return (HAM_INTEGRITY_VIOLATED);
}
#endif
return (0);
}
static size_t
__writefunc(void *buffer, size_t size, size_t nmemb, void *ptr)
{
curl_buffer_t *buf=(curl_buffer_t *)ptr;
char *cbuf=(char *)buffer;
ham_size_t payload_size=0;
if (buf->offset==0) {
if (*(ham_u32_t *)&cbuf[0]!=ham_db2h32(HAM_TRANSFER_MAGIC_V1)) {
ham_trace(("invalid protocol version"));
return (0);
}
payload_size=ham_h2db32(*(ham_u32_t *)&cbuf[4]);
/* did we receive the whole data in this packet? */
if (payload_size+8==size*nmemb) {
buf->wrapper=proto_unpack((ham_size_t)(size*nmemb),
(ham_u8_t *)&cbuf[0]);
if (!buf->wrapper)
return (0);
return (size*nmemb);
}
/* otherwise we have to buffer the received data */
buf->packed_size=payload_size+8;
buf->packed_data=allocator_alloc(buf->alloc, buf->packed_size);
if (!buf->packed_data)
return (0);
memcpy(buf->packed_data, &cbuf[0], size*nmemb);
buf->offset+=(ham_size_t)(size*nmemb);
}
/* append to an existing buffer? */
else {
memcpy(buf->packed_data+buf->offset, &cbuf[0], size*nmemb);
buf->offset+=(ham_size_t)(size*nmemb);
}
/* check if we've received the whole data */
if (buf->offset==buf->packed_size) {
buf->wrapper=proto_unpack(buf->packed_size, buf->packed_data);
if (!buf->wrapper)
return (0);
allocator_free(buf->alloc, buf->packed_data);
if (!buf->wrapper)
return 0;
}
return (size*nmemb);
}
/*
BIG FAT WARNING:
This routine should NEVER be used like this:
ham_txn_t txn;
txn_begin(&txn, env, 0);
...
txn_commit/abort(&txn);
in any (C/C++) environment where the code in the '...' may trigger out of band jumps, such as longjmp()
to an outer layer or a C++ exception, as the transaction 'txn' will be bound to the 'db' structure
internally and cause a CORE DUMP once the 'db' structure is closed (and cleaned up) as then, in the
outer layer exception handler, the 'txn' stack space will have been NUKED.
This shortcutting style of coding was used throughout the unittests and it was waiting for the axe to fall...
It is also used within the hamsterdb C code itself, which is perfectly fine as this library does not
call any exception throwing code... UNLESS OF COURSE such sort of code is to be found in ANY of the
registered hooks/callbacks!
Hence any callbacks which get registered with hamsterDB should NEVER allow any C longjmp() or C++ exception
to pass /through/ the hamsterdb layer itself, or a core dump at ham_close/ham_env_close invocation
will be your share.
*/
ham_status_t
txn_begin(ham_txn_t *txn, ham_env_t *env, ham_u32_t flags)
{
ham_status_t st=0;
/* for hamsterdb 1.0.4 - only support one transaction */
if (env_get_txn(env)) {
ham_trace(("only one concurrent transaction is supported"));
return (HAM_LIMITS_REACHED);
}
memset(txn, 0, sizeof(*txn));
txn_set_env(txn, env);
txn_set_id(txn, env_get_txn_id(env)+1);
txn_set_flags(txn, flags);
env_set_txn(env, txn);
env_set_txn_id(env, txn_get_id(txn));
if (env_get_log(env) && !(flags&HAM_TXN_READ_ONLY))
st=ham_log_append_txn_begin(env_get_log(env), txn);
return st;
}
ham_status_t
blob_read(ham_db_t *db, ham_offset_t blobid,
ham_record_t *record, ham_u32_t flags)
{
ham_status_t st;
ham_page_t *page;
blob_t hdr;
ham_size_t blobsize=0;
/*
* in-memory-database: the blobid is actually a pointer to the memory
* buffer, in which the blob is stored
*/
if (env_get_rt_flags(db_get_env(db))&HAM_IN_MEMORY_DB) {
blob_t *hdr=(blob_t *)U64_TO_PTR(blobid);
ham_u8_t *data=(ham_u8_t *)(U64_TO_PTR(blobid))+sizeof(blob_t);
/* when the database is closing, the header is already deleted */
if (!hdr) {
record->size = 0;
return (0);
}
blobsize = (ham_size_t)blob_get_size(hdr);
if (flags&HAM_PARTIAL) {
if (record->partial_offset>blobsize) {
ham_trace(("partial offset is greater than the total "
"record size"));
return (HAM_INV_PARAMETER);
}
if (record->partial_offset+record->partial_size>blobsize)
blobsize=blobsize-record->partial_offset;
else
blobsize=record->partial_size;
}
if (!blobsize) {
/* empty blob? */
record->data = 0;
record->size = 0;
}
else {
ham_u8_t *d=data;
if (flags&HAM_PARTIAL)
d+=record->partial_offset;
if ((flags&HAM_DIRECT_ACCESS)
&& !(record->flags&HAM_RECORD_USER_ALLOC)) {
record->size=blobsize;
record->data=d;
}
else {
/* resize buffer, if necessary */
if (!(record->flags & HAM_RECORD_USER_ALLOC)) {
st=db_resize_record_allocdata(db, blobsize);
if (st)
return (st);
record->data = db_get_record_allocdata(db);
}
/* and copy the data */
memcpy(record->data, d, blobsize);
record->size = blobsize;
}
}
return (0);
}
ham_assert(blobid%DB_CHUNKSIZE==0, ("blobid is %llu", blobid));
/*
* first step: read the blob header
*/
st=__read_chunk(db_get_env(db), 0, &page, blobid,
(ham_u8_t *)&hdr, sizeof(hdr));
if (st)
return (st);
ham_assert(blob_get_alloc_size(&hdr)%DB_CHUNKSIZE==0, (0));
/*
* sanity check
*/
if (blob_get_self(&hdr)!=blobid)
return (HAM_BLOB_NOT_FOUND);
blobsize = (ham_size_t)blob_get_size(&hdr);
if (flags&HAM_PARTIAL) {
if (record->partial_offset>blobsize) {
ham_trace(("partial offset+size is greater than the total "
"record size"));
return (HAM_INV_PARAMETER);
}
if (record->partial_offset+record->partial_size>blobsize)
blobsize=blobsize-record->partial_offset;
else
blobsize=record->partial_size;
}
/*
* empty blob?
*/
if (!blobsize) {
record->data = 0;
record->size = 0;
return (0);
}
/*
* second step: resize the blob buffer
*/
if (!(record->flags & HAM_RECORD_USER_ALLOC)) {
st=db_resize_record_allocdata(db, blobsize);
if (st)
return (st);
record->data = db_get_record_allocdata(db);
}
/*
* third step: read the blob data
*/
st=__read_chunk(db_get_env(db), page, 0,
blobid+sizeof(blob_t)+(flags&HAM_PARTIAL
? record->partial_offset
: 0),
record->data, blobsize);
if (st)
return (st);
record->size = blobsize;
return (0);
}
static ham_status_t
_remote_fun_get_parameters(ham_db_t *db, ham_parameter_t *param)
{
static char filename[1024];
ham_status_t st;
ham_env_t *env=db_get_env(db);
proto_wrapper_t *request, *reply;
ham_size_t i, num_names=0;
ham_u32_t *names;
ham_parameter_t *p;
/* count number of parameters */
p=param;
if (p) {
for (; p->name; p++) {
num_names++;
}
}
/* allocate a memory and copy the parameter names */
names=(ham_u32_t *)allocator_alloc(env_get_allocator(env),
num_names*sizeof(ham_u32_t));
if (!names)
return (HAM_OUT_OF_MEMORY);
p=param;
if (p) {
for (i=0; p->name; p++) {
names[i]=p->name;
i++;
}
}
request=proto_init_db_get_parameters_request(db_get_remote_handle(db),
names, num_names);
st=_perform_request(env, env_get_curl(env), request, &reply);
proto_delete(request);
allocator_free(env_get_allocator(env), names);
if (st) {
if (reply)
proto_delete(reply);
return (st);
}
ham_assert(reply!=0, (""));
ham_assert(proto_has_db_get_parameters_reply(reply), (""));
st=proto_db_get_parameters_reply_get_status(reply);
if (st) {
proto_delete(reply);
return (st);
}
p=param;
while (p && p->name) {
switch (p->name) {
case HAM_PARAM_CACHESIZE:
ham_assert(proto_db_get_parameters_reply_has_cachesize(reply), (""));
p->value=proto_db_get_parameters_reply_get_cachesize(reply);
break;
case HAM_PARAM_PAGESIZE:
ham_assert(proto_db_get_parameters_reply_has_pagesize(reply), (""));
p->value=proto_db_get_parameters_reply_get_pagesize(reply);
break;
case HAM_PARAM_MAX_ENV_DATABASES:
ham_assert(proto_db_get_parameters_reply_has_max_env_databases(reply), (""));
p->value=proto_db_get_parameters_reply_get_max_env_databases(reply);
break;
case HAM_PARAM_GET_FLAGS:
ham_assert(proto_db_get_parameters_reply_has_flags(reply), (""));
p->value=proto_db_get_parameters_reply_get_flags(reply);
break;
case HAM_PARAM_GET_FILEMODE:
ham_assert(proto_db_get_parameters_reply_has_filemode(reply), (""));
p->value=proto_db_get_parameters_reply_get_filemode(reply);
break;
case HAM_PARAM_GET_FILENAME:
ham_assert(proto_db_get_parameters_reply_has_filename(reply), (""));
strncpy(filename, proto_db_get_parameters_reply_get_filename(reply),
sizeof(filename));
p->value=PTR_TO_U64(&filename[0]);
break;
case HAM_PARAM_KEYSIZE:
ham_assert(proto_db_get_parameters_reply_has_keysize(reply), (""));
p->value=proto_db_get_parameters_reply_get_keysize(reply);
break;
case HAM_PARAM_GET_DATABASE_NAME:
ham_assert(proto_db_get_parameters_reply_has_dbname(reply), (""));
p->value=proto_db_get_parameters_reply_get_dbname(reply);
break;
case HAM_PARAM_GET_KEYS_PER_PAGE:
ham_assert(proto_db_get_parameters_reply_has_keys_per_page(reply), (""));
p->value=proto_db_get_parameters_reply_get_keys_per_page(reply);
break;
case HAM_PARAM_GET_DATA_ACCESS_MODE:
ham_assert(proto_db_get_parameters_reply_has_dam(reply), (""));
p->value=proto_db_get_parameters_reply_get_dam(reply);
break;
default:
ham_trace(("unknown parameter %d", (int)p->name));
break;
}
p++;
}
proto_delete(reply);
return (st);
}
static ham_status_t
_perform_request(ham_env_t *env, CURL *handle, proto_wrapper_t *request,
proto_wrapper_t **reply)
{
CURLcode cc;
long response=0;
char header[128];
curl_buffer_t rbuf={0};
curl_buffer_t wbuf={0};
struct curl_slist *slist=0;
wbuf.alloc=env_get_allocator(env);
*reply=0;
if (!proto_pack(request, wbuf.alloc, &rbuf.packed_data, &rbuf.packed_size))
return (HAM_INTERNAL_ERROR);
sprintf(header, "Content-Length: %u", rbuf.packed_size);
slist=curl_slist_append(slist, header);
slist=curl_slist_append(slist, "Transfer-Encoding:");
slist=curl_slist_append(slist, "Expect:");
#ifdef HAM_DEBUG
SETOPT(handle, CURLOPT_VERBOSE, 1);
#endif
SETOPT(handle, CURLOPT_URL, env_get_filename(env));
SETOPT(handle, CURLOPT_READFUNCTION, __readfunc);
SETOPT(handle, CURLOPT_READDATA, &rbuf);
SETOPT(handle, CURLOPT_UPLOAD, 1);
SETOPT(handle, CURLOPT_PUT, 1);
SETOPT(handle, CURLOPT_WRITEFUNCTION, __writefunc);
SETOPT(handle, CURLOPT_WRITEDATA, &wbuf);
SETOPT(handle, CURLOPT_HTTPHEADER, slist);
cc=curl_easy_perform(handle);
if (rbuf.packed_data)
allocator_free(env_get_allocator(env), rbuf.packed_data);
curl_slist_free_all(slist);
if (cc) {
ham_trace(("network transmission failed: %s", curl_easy_strerror(cc)));
return (HAM_NETWORK_ERROR);
}
cc=curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response);
if (cc) {
ham_trace(("network transmission failed: %s", curl_easy_strerror(cc)));
return (HAM_NETWORK_ERROR);
}
if (response!=200) {
ham_trace(("server returned error %u", response));
return (HAM_NETWORK_ERROR);
}
*reply=wbuf.wrapper;
return (0);
}
ham_status_t
txn_abort(ham_txn_t *txn, ham_u32_t flags)
{
ham_status_t st;
ham_env_t *env=txn_get_env(txn);
/*
* are cursors attached to this txn? if yes, fail
*/
if (txn_get_cursor_refcount(txn)) {
ham_trace(("transaction cannot be aborted till all attached "
"cursors are closed"));
return HAM_CURSOR_STILL_OPEN;
}
if (env_get_log(env) && !(txn_get_flags(txn)&HAM_TXN_READ_ONLY)) {
st=ham_log_append_txn_abort(env_get_log(env), txn);
if (st)
return st;
}
env_set_txn(env, 0);
/*
* undo all operations from this transaction
*
* this includes allocated pages (they're moved to the freelist),
* deleted pages (they're un-deleted) and other modifications (will
* re-create the original page from the logfile)
*
* keep txn_get_pagelist(txn) intact during every round, so no
* local var for this one.
*/
while (txn_get_pagelist(txn)) {
ham_page_t *head = txn_get_pagelist(txn);
if (!(flags & DO_NOT_NUKE_PAGE_STATS)) {
/*
* nuke critical statistics, such as tracked outer bounds; imagine,
* for example, a failing erase transaction which, through erasing
* the top-most key, lowers the actual upper bound, after which
* the transaction fails at some later point in life. Now if we
* wouldn't 'rewind' our bounds-statistics, we would have a
* situation where a subsequent out-of-bounds insert (~ append)
* would possibly FAIL due to the hinter using incorrect bounds
* information then!
*
* Hence we 'reverse' our statistics here and the easiest route
* is to just nuke the critical bits; subsequent find/insert/erase
* operations will ensure that the stats will get updated again,
* anyhow. All we loose then is a few subsequent operations, which
* might have been hinted if we had played a smarter game of
* statistics 'reversal'. Soit.
*/
ham_db_t *db = page_get_owner(head);
/*
* only need to do this for index pages anyhow, and those are the
* ones which have their 'ownership' set.
*/
if (db) {
stats_page_is_nuked(db, head, HAM_FALSE);
}
}
ham_assert(page_is_in_list(txn_get_pagelist(txn), head, PAGE_LIST_TXN),
(0));
txn_get_pagelist(txn) = page_list_remove(head, PAGE_LIST_TXN, head);
/* if this page was allocated by this transaction, then we can
* move the whole page to the freelist */
if (page_get_alloc_txn_id(head)==txn_get_id(txn)) {
(void)freel_mark_free(env, 0, page_get_self(head),
env_get_pagesize(env), HAM_TRUE);
}
else {
/* remove the 'delete pending' flag */
page_set_npers_flags(head,
page_get_npers_flags(head)&~PAGE_NPERS_DELETE_PENDING);
/* if the page is dirty, and RECOVERY is enabled: recreate
* the original, unmodified page from the log */
if (env_get_log(env) && page_is_dirty(head)) {
st=ham_log_recreate(env_get_log(env), head);
if (st)
return (st);
/*page_set_undirty(head); */
}
}
/* page is no longer in use */
page_release_ref(head);
}
ham_assert(txn_get_pagelist(txn)==0, (0));
return (0);
}
ham_status_t
txn_commit(ham_txn_t *txn, ham_u32_t flags)
{
ham_status_t st;
ham_env_t *env=txn_get_env(txn);
/*
* are cursors attached to this txn? if yes, fail
*/
if (txn_get_cursor_refcount(txn)) {
ham_trace(("transaction cannot be committed till all attached "
"cursors are closed"));
return HAM_CURSOR_STILL_OPEN;
}
/*
* in case of logging: write after-images of all modified pages,
* if they were modified by this transaction;
* then write the transaction boundary
*/
if (env_get_log(env) && !(txn_get_flags(txn)&HAM_TXN_READ_ONLY))
{
ham_page_t *head=txn_get_pagelist(txn);
while (head) {
ham_page_t *next;
next=page_get_next(head, PAGE_LIST_TXN);
if (page_get_dirty_txn(head)==txn_get_id(txn)
|| page_get_dirty_txn(head)==PAGE_DUMMY_TXN_ID) {
st=ham_log_add_page_after(head);
if (st)
return st;
}
head=next;
}
st=ham_log_append_txn_commit(env_get_log(env), txn);
if (st)
return st;
}
env_set_txn(env, 0);
/*
* flush the pages
*
* shouldn't use local var for the list head, as
* txn_get_pagelist(txn) should be kept up to date and correctly
* formatted while we call db_free_page() et al.
*/
while (txn_get_pagelist(txn))
{
ham_page_t *head = txn_get_pagelist(txn);
txn_get_pagelist(txn) = page_list_remove(head, PAGE_LIST_TXN, head);
/* page is no longer in use */
page_release_ref(head);
/*
* delete the page?
*/
if (page_get_npers_flags(head)&PAGE_NPERS_DELETE_PENDING) {
/* remove page from cache, add it to garbage list */
page_set_undirty(head);
st=db_free_page(head, DB_MOVE_TO_FREELIST);
if (st)
return (st);
}
else if (flags & HAM_TXN_FORCE_WRITE) {
/* flush the page */
st=db_flush_page(env, head,
flags & HAM_TXN_FORCE_WRITE ? HAM_WRITE_THROUGH : 0);
if (st) {
page_add_ref(head);
/* failure: re-insert into transaction list! */
txn_get_pagelist(txn) = page_list_insert(txn_get_pagelist(txn),
PAGE_LIST_TXN, head);
return (st);
}
}
}
txn_set_pagelist(txn, 0);
return HAM_SUCCESS;
}
/**
* create and initialize a new backend
*
* @remark this function is called after the @a ham_db_t structure
* and the file were created
*
* the @a flags are stored in the database; only transfer
* the persistent flags!
*
* @note This is a B+-tree 'backend' method.
*/
static ham_status_t
my_fun_create(ham_btree_t *be, ham_u16_t keysize, ham_u32_t flags)
{
ham_status_t st;
ham_page_t *root;
ham_size_t maxkeys;
ham_db_t *db=be_get_db(be);
db_indexdata_t *indexdata=env_get_indexdata_ptr(db_get_env(db),
db_get_indexdata_offset(db));
if (be_is_active(be))
{
ham_trace(("backend has alread been initialized before!"));
/* HAM_INTERNAL_ERROR -- not really, when keeping custom
* backends in mind */
return HAM_ALREADY_INITIALIZED;
}
/*
* prevent overflow - maxkeys only has 16 bit!
*/
maxkeys=btree_calc_maxkeys(env_get_pagesize(db_get_env(db)), keysize);
if (maxkeys>MAX_KEYS_PER_NODE) {
ham_trace(("keysize/pagesize ratio too high"));
return HAM_INV_KEYSIZE;
}
else if (maxkeys==0) {
ham_trace(("keysize too large for the current pagesize"));
return HAM_INV_KEYSIZE;
}
/*
* allocate a new root page
*/
st=db_alloc_page(&root, db, PAGE_TYPE_B_ROOT, PAGE_IGNORE_FREELIST);
ham_assert(st ? root == NULL : 1, (0));
ham_assert(!st ? root != NULL : 1, (0));
if (!root)
return st ? st : HAM_INTERNAL_ERROR;
memset(page_get_raw_payload(root), 0,
sizeof(btree_node_t)+sizeof(ham_perm_page_union_t));
/*
* calculate the maximum number of keys for this page,
* and make sure that this number is even
*/
btree_set_maxkeys(be, (ham_u16_t)maxkeys);
be_set_dirty(be, HAM_TRUE);
be_set_keysize(be, keysize);
be_set_flags(be, flags);
btree_set_rootpage(be, page_get_self(root));
index_clear_reserved(indexdata);
index_set_max_keys(indexdata, (ham_u16_t)maxkeys);
index_set_keysize(indexdata, keysize);
index_set_self(indexdata, page_get_self(root));
index_set_flags(indexdata, flags);
index_set_recno(indexdata, 0);
index_clear_reserved(indexdata);
env_set_dirty(db_get_env(db));
be_set_active(be, HAM_TRUE);
return (0);
}
