/**
* Write value to the database file, possibly caching it and deferring write.
*
* Any registered value cleanup callback will be invoked right after the value
* is written to disk (for immediated writes) or removed from the cache (for
* deferred writes).
*
* @param dw the DBM wrapper
* @param key the key (constant-width, determined at open time)
* @param value the start of the value in memory
* @param length length of the value
*/
void
dbmw_write(dbmw_t *dw, gconstpointer key, gpointer value, size_t length)
{
struct cached *entry;
dbmw_check(dw);
g_assert(key);
g_assert(length <= dw->value_size);
g_assert(length || value == NULL);
g_assert(length == 0 || value);
dw->w_access++;
entry = map_lookup(dw->values, key);
if (entry) {
if (entry->dirty)
dw->w_hits++;
else if (entry->absent)
dw->count_needs_sync = TRUE; /* Key exists now */
fill_entry(dw, entry, value, length);
hash_list_moveto_tail(dw->keys, key);
} else if (dw->max_cached > 1) {
entry = allocate_entry(dw, key, NULL);
fill_entry(dw, entry, value, length);
dw->count_needs_sync = TRUE; /* Does not know whether key exists */
} else {
write_immediately(dw, key, value, length);
}
}
void* CLIB_DECL malloc(size_t size)
{
memory_entry *entry = allocate_entry();
UINT8 *page_base, *block_base;
size_t rounded_size;
// round the size up to a page boundary
rounded_size = ((size + PAGE_SIZE - 1) / PAGE_SIZE) * PAGE_SIZE;
// reserve that much memory, plus two guard pages
page_base = VirtualAlloc(NULL, rounded_size + 2 * PAGE_SIZE, MEM_RESERVE, PAGE_NOACCESS);
if (page_base == NULL)
return NULL;
// now allow access to everything but the first and last pages
page_base = VirtualAlloc(page_base + PAGE_SIZE, rounded_size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
if (page_base == NULL)
return NULL;
// work backwards from the page base to get to the block base
#if ALIGN_START
block_base = page_base;
#else
block_base = page_base + rounded_size - size;
#endif
// fill in the entry
entry->size = size;
entry->base = block_base;
entry->vbase = page_base - PAGE_SIZE;
return block_base;
}
/*
Allocates a entry
size = size of entry
*/
void *memory_manager_allocate(const uint32_t size) {
if (!size) { /* check it#s greater then 0 */
return NULL;
}
if (!(size & ~327684)) { /* if we have a bucket big enough */
return allocate_entry(size);
}
return malloc(size); /* else collect from the os */
}
void *malloc_file_line(size_t size, const char *file, int line)
{
UINT8 *page_base, *block_base;
size_t rounded_size;
memory_entry *entry;
if (USE_GUARD_PAGES)
{
// round the size up to a page boundary
rounded_size = ((size + PAGE_SIZE - 1) / PAGE_SIZE) * PAGE_SIZE;
// reserve that much memory, plus two guard pages
page_base = VirtualAlloc(NULL, rounded_size + 2 * PAGE_SIZE, MEM_RESERVE, PAGE_NOACCESS);
if (page_base == NULL)
return NULL;
// now allow access to everything but the first and last pages
page_base = VirtualAlloc(page_base + PAGE_SIZE, rounded_size, MEM_COMMIT, PAGE_READWRITE);
if (page_base == NULL)
return NULL;
// work backwards from the page base to get to the block base
if (ALIGN_START)
block_base = page_base;
else
block_base = page_base + rounded_size - size;
}
else
{
block_base = (UINT8 *)GlobalAlloc(GMEM_FIXED, size);
}
// fill in the entry
entry = allocate_entry();
entry->size = size;
entry->base = block_base;
entry->file = file;
entry->line = line;
entry->id = current_id++;
//if (entry->size == 72 && IsDebuggerPresent()) DebugBreak();
#if LOG_CALLS
// logging
if (entry->file)
logerror("malloc #%06d size = %d (%s:%d)\n", entry->size, entry->file, entry->line);
else
logerror("malloc #%06d size = %d\n", entry->id, entry->size);
#endif
return block_base;
}
/**
* Delete key from database.
*/
void
dbmw_delete(dbmw_t *dw, gconstpointer key)
{
struct cached *entry;
dbmw_check(dw);
g_assert(key);
dw->w_access++;
entry = map_lookup(dw->values, key);
if (entry) {
if (entry->dirty)
dw->w_hits++;
if (!entry->absent) {
dw->count_needs_sync = TRUE; /* Deferred delete */
fill_entry(dw, entry, NULL, 0);
entry->absent = TRUE;
}
hash_list_moveto_tail(dw->keys, key);
} else {
dw->ioerr = FALSE;
dbmap_remove(dw->dm, key);
if (dbmap_has_ioerr(dw->dm)) {
dw->ioerr = TRUE;
dw->error = errno;
g_warning("DBMW \"%s\" I/O error whilst deleting key: %s",
dw->name, dbmap_strerror(dw->dm));
}
/*
* If the maximum value length of the DB is 0, then it is used as a
* "search table" only, meaning there will be no read to get values,
* only existence checks.
*
* Therefore, it makes sense to cache that the key is no longer valid.
* Otherwise, possibly pushing a value out of the cache to record
* a deletion is not worth it.
*/
if (0 == dw->value_size) {
WALLOC0(entry);
entry->absent = TRUE;
(void) allocate_entry(dw, key, entry);
}
}
}
/**
* Write value to the database file, possibly caching it and deferring write.
*
* Any registered value cleanup callback will be invoked right after the value
* is written to disk (for immediated writes) or removed from the cache (for
* deferred writes).
*
* @param dw the DBM wrapper
* @param key the key (constant-width, determined at open time)
* @param value the start of the value in memory
* @param length length of the value
*/
void
dbmw_write(dbmw_t *dw, const void *key, void *value, size_t length)
{
struct cached *entry;
dbmw_check(dw);
g_assert(key);
g_assert(length <= dw->value_size);
g_assert(length || value == NULL);
g_assert(length == 0 || value);
dw->w_access++;
entry = map_lookup(dw->values, key);
if (entry) {
if (dbg_ds_debugging(dw->dbg, 2, DBG_DSF_CACHING | DBG_DSF_UPDATE)) {
dbg_ds_log(dw->dbg, dw, "%s: %s key=%s%s",
G_STRFUNC, entry->dirty ? "dirty" : "clean",
dbg_ds_keystr(dw->dbg, key, (size_t) -1),
entry->absent ? " (was absent)" : "");
}
if (entry->dirty)
dw->w_hits++;
if (entry->absent)
dw->cached++; /* Key exists now, in unflushed status */
fill_entry(dw, entry, value, length);
hash_list_moveto_tail(dw->keys, key);
} else if (dw->max_cached > 1) {
if (dbg_ds_debugging(dw->dbg, 2, DBG_DSF_CACHING | DBG_DSF_UPDATE)) {
dbg_ds_log(dw->dbg, dw, "%s: deferring key=%s",
G_STRFUNC, dbg_ds_keystr(dw->dbg, key, (size_t) -1));
}
entry = allocate_entry(dw, key, NULL);
fill_entry(dw, entry, value, length);
dw->count_needs_sync = TRUE; /* Does not know whether key exists */
} else {
if (dbg_ds_debugging(dw->dbg, 2, DBG_DSF_CACHING | DBG_DSF_UPDATE)) {
dbg_ds_log(dw->dbg, dw, "%s: writing key=%s",
G_STRFUNC, dbg_ds_keystr(dw->dbg, key, (size_t) -1));
}
write_immediately(dw, key, value, length);
}
}
/**
* Is key present in the database?
*/
gboolean
dbmw_exists(dbmw_t *dw, gconstpointer key)
{
struct cached *entry;
gboolean ret;
dbmw_check(dw);
g_assert(key);
dw->r_access++;
entry = map_lookup(dw->values, key);
if (entry) {
dw->r_hits++;
return !entry->absent;
}
dw->ioerr = FALSE;
ret = dbmap_contains(dw->dm, key);
if (dbmap_has_ioerr(dw->dm)) {
dw->ioerr = TRUE;
dw->error = errno;
g_warning("DBMW \"%s\" I/O error whilst checking key existence: %s",
dw->name, dbmap_strerror(dw->dm));
return FALSE;
}
/*
* If the maximum value length of the DB is 0, then it is used as a
* "search table" only, meaning there will be no read to get values,
* only existence checks.
*
* Therefore, it makes sense to cache existence checks. A data read
* will also correctly return a null item from the cache.
*/
if (0 == dw->value_size) {
WALLOC0(entry);
entry->absent = !ret;
(void) allocate_entry(dw, key, entry);
}
return ret;
}
/**
* Read value from database file, returning a pointer to the allocated
* deserialized data. These data can be modified freely and stored back,
* but their lifetime will not exceed that of the next call to a dbmw
* operation on the same descriptor.
*
* User code does not need to bother with freeing the allocated data, this
* is managed directly by the DBM wrapper.
*
* @param dw the DBM wrapper
* @param key the key (constant-width, determined at open time)
* @param lenptr if non-NULL, writes length of (deserialized) value
*
* @return pointer to value, or NULL if it was either not found or the
* deserialization failed.
*/
G_GNUC_HOT gpointer
dbmw_read(dbmw_t *dw, gconstpointer key, size_t *lenptr)
{
struct cached *entry;
dbmap_datum_t dval;
dbmw_check(dw);
g_assert(key);
dw->r_access++;
entry = map_lookup(dw->values, key);
if (entry) {
dw->r_hits++;
if (lenptr)
*lenptr = entry->len;
return entry->data;
}
/*
* Not cached, must read from DB.
*/
dw->ioerr = FALSE;
dval = dbmap_lookup(dw->dm, key);
if (dbmap_has_ioerr(dw->dm)) {
dw->ioerr = TRUE;
dw->error = errno;
g_warning("DBMW \"%s\" I/O error whilst reading entry: %s",
dw->name, dbmap_strerror(dw->dm));
return NULL;
} else if (NULL == dval.data)
return NULL; /* Not found in DB */
/*
* Value was found, allocate a cache entry object for it.
*/
WALLOC0(entry);
/*
* Deserialize data if needed.
*/
if (dw->unpack) {
/*
* Allocate cache entry arena to hold the deserialized version.
*/
entry->data = walloc(dw->value_size);
entry->len = dw->value_size;
bstr_reset(dw->bs, dval.data, dval.len, BSTR_F_ERROR);
if (!dbmw_deserialize(dw, dw->bs, entry->data, dw->value_size)) {
g_carp("DBMW \"%s\" deserialization error in %s(): %s",
dw->name,
stacktrace_routine_name(func_to_pointer(dw->unpack), FALSE),
bstr_error(dw->bs));
/* Not calling value free routine on deserialization failures */
wfree(entry->data, dw->value_size);
WFREE(entry);
return NULL;
}
if (lenptr)
*lenptr = dw->value_size;
} else {
g_assert(dw->value_size >= dval.len);
if (dval.len) {
entry->len = dval.len;
entry->data = wcopy(dval.data, dval.len);
} else {
entry->data = NULL;
entry->len = 0;
}
if (lenptr)
*lenptr = dval.len;
}
g_assert((entry->len != 0) == (entry->data != NULL));
/*
* Insert into cache.
*/
(void) allocate_entry(dw, key, entry);
return entry->data;
}
/**
* Read value from database file, returning a pointer to the allocated
* deserialized data. These data can be modified freely and stored back,
* but their lifetime will not exceed that of the next call to a dbmw
* operation on the same descriptor.
*
* User code does not need to bother with freeing the allocated data, this
* is managed directly by the DBM wrapper.
*
* @param dw the DBM wrapper
* @param key the key (constant-width, determined at open time)
* @param lenptr if non-NULL, writes length of (deserialized) value
*
* @return pointer to value, or NULL if it was either not found or the
* deserialization failed.
*/
G_GNUC_HOT void *
dbmw_read(dbmw_t *dw, const void *key, size_t *lenptr)
{
struct cached *entry;
dbmap_datum_t dval;
dbmw_check(dw);
g_assert(key);
dw->r_access++;
entry = map_lookup(dw->values, key);
if (entry) {
if (dbg_ds_debugging(dw->dbg, 5, DBG_DSF_CACHING | DBG_DSF_ACCESS)) {
dbg_ds_log(dw->dbg, dw, "%s: read cache hit on %s key=%s%s",
G_STRFUNC, entry->dirty ? "dirty" : "clean",
dbg_ds_keystr(dw->dbg, key, (size_t) -1),
entry->absent ? " (absent)" : "");
}
dw->r_hits++;
if (lenptr)
*lenptr = entry->len;
return entry->data;
}
/*
* Not cached, must read from DB.
*/
dw->ioerr = FALSE;
dval = dbmap_lookup(dw->dm, key);
if (dbmap_has_ioerr(dw->dm)) {
dw->ioerr = TRUE;
dw->error = errno;
s_warning_once_per(LOG_PERIOD_SECOND,
"DBMW \"%s\" I/O error whilst reading entry: %s",
dw->name, dbmap_strerror(dw->dm));
return NULL;
} else if (NULL == dval.data)
return NULL; /* Not found in DB */
/*
* Value was found, allocate a cache entry object for it.
*/
WALLOC0(entry);
/*
* Deserialize data if needed.
*/
if (dw->unpack) {
/*
* Allocate cache entry arena to hold the deserialized version.
*/
entry->data = walloc(dw->value_size);
entry->len = dw->value_size;
bstr_reset(dw->bs, dval.data, dval.len, BSTR_F_ERROR);
if (!dbmw_deserialize(dw, dw->bs, entry->data, dw->value_size)) {
s_critical("DBMW \"%s\" deserialization error in %s(): %s",
dw->name, stacktrace_function_name(dw->unpack),
bstr_error(dw->bs));
/* Not calling value free routine on deserialization failures */
wfree(entry->data, dw->value_size);
WFREE(entry);
return NULL;
}
if (lenptr)
*lenptr = dw->value_size;
} else {
g_assert(dw->value_size >= dval.len);
if (dval.len) {
entry->len = dval.len;
entry->data = wcopy(dval.data, dval.len);
} else {
entry->data = NULL;
entry->len = 0;
}
if (lenptr)
*lenptr = dval.len;
}
g_assert((entry->len != 0) == (entry->data != NULL));
/*
* Insert into cache.
*/
(void) allocate_entry(dw, key, entry);
if (dbg_ds_debugging(dw->dbg, 4, DBG_DSF_CACHING)) {
dbg_ds_log(dw->dbg, dw, "%s: cached %s key=%s%s",
G_STRFUNC, entry->dirty ? "dirty" : "clean",
dbg_ds_keystr(dw->dbg, key, (size_t) -1),
entry->absent ? " (absent)" : "");
}
return entry->data;
}
/**
* Delete key from database.
*/
void
dbmw_delete(dbmw_t *dw, const void *key)
{
struct cached *entry;
dbmw_check(dw);
g_assert(key);
dw->w_access++;
entry = map_lookup(dw->values, key);
if (entry) {
if (dbg_ds_debugging(dw->dbg, 2, DBG_DSF_CACHING | DBG_DSF_DELETE)) {
dbg_ds_log(dw->dbg, dw, "%s: %s key=%s%s",
G_STRFUNC, entry->dirty ? "dirty" : "clean",
dbg_ds_keystr(dw->dbg, key, (size_t) -1),
entry->absent ? " (was absent)" : "");
}
if (entry->dirty)
dw->w_hits++;
if (!entry->absent) {
/*
* Entry was present but is now deleted.
*
* If it was clean, then it was flushed to the database and we now
* know that there is one less entry in the database than there is
* physically present in the map.
*
* If it was dirty, then we do not know whether it exists in the
* database or not, and therefore we cannot adjust the amount
* of cached entries down.
*/
if (entry->dirty)
dw->count_needs_sync = TRUE; /* Deferred delete */
else
dw->cached--; /* One less entry in database */
fill_entry(dw, entry, NULL, 0);
entry->absent = TRUE;
}
hash_list_moveto_tail(dw->keys, key);
} else {
if (dbg_ds_debugging(dw->dbg, 2, DBG_DSF_DELETE)) {
dbg_ds_log(dw->dbg, dw, "%s: removing key=%s",
G_STRFUNC, dbg_ds_keystr(dw->dbg, key, (size_t) -1));
}
dw->ioerr = FALSE;
dbmap_remove(dw->dm, key);
if (dbmap_has_ioerr(dw->dm)) {
dw->ioerr = TRUE;
dw->error = errno;
s_warning("DBMW \"%s\" I/O error whilst deleting key: %s",
dw->name, dbmap_strerror(dw->dm));
}
/*
* If the maximum value length of the DB is 0, then it is used as a
* "search table" only, meaning there will be no read to get values,
* only existence checks.
*
* Therefore, it makes sense to cache that the key is no longer valid.
* Otherwise, possibly pushing a value out of the cache to record
* a deletion is not worth it.
*/
if (0 == dw->value_size) {
WALLOC0(entry);
entry->absent = TRUE;
(void) allocate_entry(dw, key, entry);
if (dbg_ds_debugging(dw->dbg, 2, DBG_DSF_CACHING)) {
dbg_ds_log(dw->dbg, dw, "%s: cached absent key=%s",
G_STRFUNC, dbg_ds_keystr(dw->dbg, key, (size_t) -1));
}
}
}
}
/**
* Is key present in the database?
*/
bool
dbmw_exists(dbmw_t *dw, const void *key)
{
struct cached *entry;
bool ret;
dbmw_check(dw);
g_assert(key);
dw->r_access++;
entry = map_lookup(dw->values, key);
if (entry) {
if (dbg_ds_debugging(dw->dbg, 5, DBG_DSF_CACHING | DBG_DSF_ACCESS)) {
dbg_ds_log(dw->dbg, dw, "%s: read cache hit on %s key=%s%s",
G_STRFUNC, entry->dirty ? "dirty" : "clean",
dbg_ds_keystr(dw->dbg, key, (size_t) -1),
entry->absent ? " (absent)" : "");
}
dw->r_hits++;
return !entry->absent;
}
dw->ioerr = FALSE;
ret = dbmap_contains(dw->dm, key);
if (dbmap_has_ioerr(dw->dm)) {
dw->ioerr = TRUE;
dw->error = errno;
s_warning("DBMW \"%s\" I/O error whilst checking key existence: %s",
dw->name, dbmap_strerror(dw->dm));
return FALSE;
}
/*
* If the maximum value length of the DB is 0, then it is used as a
* "search table" only, meaning there will be no read to get values,
* only existence checks.
*
* Therefore, it makes sense to cache existence checks. A data read
* will also correctly return a null item from the cache.
*
* If the value length is not 0, we only cache negative lookups (i.e.
* the value was not found) because we did not get any value so it is
* possible to record an absent cache entry.
*/
if (0 == dw->value_size || !ret) {
WALLOC0(entry);
entry->absent = !ret;
(void) allocate_entry(dw, key, entry);
if (dbg_ds_debugging(dw->dbg, 2, DBG_DSF_CACHING)) {
dbg_ds_log(dw->dbg, dw, "%s: cached %s key=%s",
G_STRFUNC, entry->absent ? "absent" : "present",
dbg_ds_keystr(dw->dbg, key, (size_t) -1));
}
}
return ret;
}