/**
* Prune the database of banned GUIDs, removing expired entries.
*/
static void
guid_prune_old(void)
{
if (GNET_PROPERTY(guid_debug))
g_debug("GUID pruning expired entries (%zu)", dbmw_count(db_guid));
dbmw_foreach_remove(db_guid, guid_prune_old_entries, NULL);
gnet_stats_set_general(GNR_BANNED_GUID_HELD, dbmw_count(db_guid));
if (GNET_PROPERTY(guid_debug)) {
g_debug("GUID pruned expired entries (%zu remaining)",
dbmw_count(db_guid));
}
}
/**
* Close DM map, keeping the SDBM file around.
*
* If the map was held in memory, it is serialized to disk.
*/
void
dbstore_close(dbmw_t *dw, const char *dir, const char *base)
{
bool ok;
char *path;
if (NULL == dw)
return;
path = make_pathname(dir, base);
if (dbstore_debug > 1)
g_debug("DBSTORE persisting DBMW \"%s\" as %s", dbmw_name(dw), path);
ok = dbmw_store(dw, path, TRUE);
HFREE_NULL(path);
if (dbstore_debug > 0) {
size_t count = dbmw_count(dw);
g_debug("DBSTORE %ssucessfully persisted DBMW \"%s\" (%u key%s)",
ok ? "" : "un", dbmw_name(dw),
(unsigned) count, 1 == count ? "" : "s");
}
dbmw_destroy(dw, TRUE);
}
/**
* Attempt to clear / shrink the DBMW database.
*/
void
dbstore_shrink(dbmw_t *dw)
{
/*
* If we retained no entries, issue a dbmw_clear() to restore underlying
* SDBM files to their smallest possible value. This is necessary because
* the database is persistent and it can grow very large on disk, but still
* holding only a few values per page. Being able to get a fresh start
* occasionally is a plus.
*/
if (0 == dbmw_count(dw)) {
if (dbstore_debug > 1) {
g_debug("DBSTORE clearing database DBMW \"%s\"", dbmw_name(dw));
}
if (!dbmw_clear(dw)) {
if (dbstore_debug) {
g_warning("DBSTORE unable to clear DBMW \"%s\"", dbmw_name(dw));
}
}
} else {
if (dbstore_debug > 1) {
g_debug("DBSTORE shrinking database DBMW \"%s\"", dbmw_name(dw));
}
if (!dbmw_shrink(dw)) {
if (dbstore_debug) {
g_warning("DBSTORE unable to shrink DBMW \"%s\"",
dbmw_name(dw));
}
}
}
}
/**
* Opens or create a disk database with an SDBM back-end.
*
* If we can't access the SDBM files on disk, we'll transparently use
* an in-core version.
*
* @param name the name of the storage created, for logs
* @param dir the directory where SDBM files will be put
* @param base the base name of SDBM files
* @param flags the sdbm_open() flags
* @param kv key/value description
* @param packing key/value serialization description
* @param cache_size Amount of items to cache (0 = no cache, 1 = default)
* @param hash_func Key hash function
* @param eq_func Key equality test function
* @param incore If TRUE, allow fallback to a RAM-only database
*
* @return the DBMW wrapping object.
*/
dbmw_t *
dbstore_open(const char *name, const char *dir, const char *base,
dbstore_kv_t kv, dbstore_packing_t packing,
size_t cache_size, hash_fn_t hash_func, eq_fn_t eq_func,
bool incore)
{
dbmw_t *dw;
dw = dbstore_create_internal(name, dir, base, O_CREAT | O_RDWR,
kv, packing, cache_size, hash_func, eq_func, FALSE);
if (dw != NULL && dbstore_debug > 0) {
size_t count = dbmw_count(dw);
g_debug("DBSTORE opened DBMW \"%s\" (%u key%s) from %s",
dbmw_name(dw), (unsigned) count, 1 == count ? "" : "s", base);
}
/*
* If they want RAM-only storage, create a new RAM DBMW and copy
* the persisted one there.
*/
if (dw != NULL && incore) {
dbmw_t *dram;
size_t count = dbmw_count(dw);
if (dbstore_debug > 0) {
g_debug("DBSTORE loading DBMW \"%s\" (%u key%s) from %s",
dbmw_name(dw), (unsigned) count, 1 == count ? "" : "s", base);
}
dram = dbstore_create_internal(name, NULL, NULL, 0,
kv, packing, cache_size, hash_func, eq_func, TRUE);
if (!dbmw_copy(dw, dram)) {
g_warning("DBSTORE could not load DBMW \"%s\" (%u key%s) from %s",
dbmw_name(dw), (unsigned) count, 1 == count ? "" : "s", base);
}
dbmw_destroy(dw, TRUE);
dw = dram;
}
return dw;
}
/**
* Prune the database, removing old entries not updated since at least
* STABLE_EXPIRE seconds and which have less than STABLE_PROBA chance of
* still being alive, given our probability density function.
*/
static void
stable_prune_old(void)
{
if (GNET_PROPERTY(dht_stable_debug)) {
g_debug("DHT STABLE pruning old stable node records (%lu)",
(unsigned long) dbmw_count(db_lifedata));
}
dbmw_foreach_remove(db_lifedata, prune_old, NULL);
gnet_stats_set_general(GNR_DHT_STABLE_NODES_HELD, dbmw_count(db_lifedata));
if (GNET_PROPERTY(dht_stable_debug)) {
g_debug("DHT STABLE pruned old stable node records (%lu remaining)",
(unsigned long) dbmw_count(db_lifedata));
}
dbstore_shrink(db_lifedata);
}
/**
* Prune the database, removing expired tokens.
*/
static void
tcache_prune_old(void)
{
size_t pruned;
if (GNET_PROPERTY(dht_tcache_debug)) {
g_debug("DHT TCACHE pruning expired tokens (%zu)",
dbmw_count(db_tokdata));
}
pruned = dbmw_foreach_remove(db_tokdata, tk_prune_old, NULL);
gnet_stats_set_general(GNR_DHT_CACHED_TOKENS_HELD, dbmw_count(db_tokdata));
if (GNET_PROPERTY(dht_tcache_debug)) {
g_debug("DHT TCACHE pruned expired tokens (%zu pruned, %zu remaining)",
pruned, dbmw_count(db_tokdata));
}
}
/**
* Remove expired items at startup time.
*/
static void
publisher_trim_pubdata(void)
{
size_t count;
if (GNET_PROPERTY(publisher_debug)) {
count = dbmw_count(db_pubdata);
g_debug("PUBLISHER scanning %u retrieved SHA1%s",
(unsigned) count, plural(count));
}
dbmw_foreach_remove(db_pubdata, publisher_remove_expired, NULL);
count = dbmw_count(db_pubdata);
if (GNET_PROPERTY(publisher_debug)) {
g_debug("PUBLISHER kept information about %u SHA1%s",
(unsigned) count, plural(count));
}
dbstore_compact(db_pubdata);
}
/**
* Delete known-to-be existing token data for specified KUID from database.
*/
static void
delete_tokdata(const kuid_t *id)
{
dbmw_delete(db_tokdata, id);
gnet_stats_dec_general(GNR_DHT_CACHED_TOKENS_HELD);
if (GNET_PROPERTY(dht_tcache_debug) > 2)
g_debug("DHT TCACHE security token from %s reclaimed",
kuid_to_hex_string(id));
if (GNET_PROPERTY(dht_tcache_debug_flags) & DBG_DSF_USR1) {
g_debug("DHT TCACHE %s: stats=%s, count=%zu, id=%s",
G_STRFUNC, uint64_to_string(
gnet_stats_get_general(GNR_DHT_CACHED_TOKENS_HELD)),
dbmw_count(db_tokdata), kuid_to_hex_string(id));
}
}
/**
* Map iterator to record security tokens in the database.
*/
static void
record_token(void *key, void *value, void *unused_u)
{
kuid_t *id = key;
lookup_token_t *ltok = value;
struct tokdata td;
(void) unused_u;
td.last_update = ltok->retrieved;
td.length = ltok->token->length;
td.token = td.length ? wcopy(ltok->token->v, td.length) : NULL;
if (GNET_PROPERTY(dht_tcache_debug) > 4) {
char buf[80];
bin_to_hex_buf(td.token, td.length, buf, sizeof buf);
g_debug("DHT TCACHE adding security token for %s: %u-byte \"%s\"",
kuid_to_hex_string(id), td.length, buf);
}
/*
* Data is put in the DBMW cache and the dynamically allocated token
* will be freed via free_tokdata() when the cached entry is released.
*/
if (!dbmw_exists(db_tokdata, id->v))
gnet_stats_inc_general(GNR_DHT_CACHED_TOKENS_HELD);
dbmw_write(db_tokdata, id->v, &td, sizeof td);
if (GNET_PROPERTY(dht_tcache_debug_flags) & DBG_DSF_USR1) {
g_debug("DHT TCACHE %s: stats=%s, count=%zu, id=%s",
G_STRFUNC, uint64_to_string(
gnet_stats_get_general(GNR_DHT_CACHED_TOKENS_HELD)),
dbmw_count(db_tokdata), kuid_to_hex_string(id));
}
}
/**
* Recreate keyinfo data from persisted information.
*/
static G_GNUC_COLD void
keys_init_keyinfo(void)
{
struct keys_create_context ctx;
if (GNET_PROPERTY(dht_keys_debug)) {
size_t count = dbmw_count(db_keydata);
g_debug("DHT KEYS scanning %u persisted key%s",
(unsigned) count, plural(count));
}
ctx.our_kuid = get_our_kuid();
ctx.dbkeys = hset_create(HASH_KEY_FIXED, sizeof(uint64));
dbmw_foreach_remove(db_keydata, reload_ki, &ctx);
if (GNET_PROPERTY(dht_keys_debug)) {
size_t count = dbmw_count(db_keydata);
g_debug("DHT KEYS kept %u key%s, now loading associated values",
(unsigned) count, plural(count));
}
/*
* FIXME:
* Unfortunately, we have to reset the keydata database for each of
* the keys we're going to recreate, because the logic adding values
* back to the key will fill each of the keydata appropriately, and it
* expects the amount of values in the keyinfo and the keydata to match.
*
* Therefore, we need to rename the old keydata database, open a new one,
* write empty keydata values in it, then discard the old keydata if
* everything goes well. In case of a crash in the middle of the
* restoration, we'll be able to recover by opening the old keydata first
* if it exists.
*
* It is far from critical though: if we reset the keydata database and
* we crash in the middle of the restore, then we'll lose all the persisted
* keys and values and will simply restart with an empty storage.
*
* Given the contorsions needed to fix that, and the little value for
* the user, just leaving a FIXME note.
* --RAM, 2012-11-17
*/
hikset_foreach(keys, keys_reset_keydata, NULL);
values_init_data(ctx.dbkeys);
hset_foreach(ctx.dbkeys, keys_free_dbkey, NULL);
hset_free_null(&ctx.dbkeys);
hikset_foreach_remove(keys, keys_discard_if_empty, NULL);
dbmw_foreach_remove(db_keydata, keys_delete_if_empty, NULL);
dbstore_compact(db_keydata);
g_soft_assert_log(hikset_count(keys) == dbmw_count(db_keydata),
"keys reloaded: %zu, key data persisted: %zu",
hikset_count(keys), dbmw_count(db_keydata));
if (GNET_PROPERTY(dht_keys_debug)) {
g_debug("DHT KEYS reloaded %zu key%s",
hikset_count(keys), plural(hikset_count(keys)));
}
gnet_stats_set_general(GNR_DHT_KEYS_HELD, hikset_count(keys));
}