Exemplo n.º 1
0
/**
 * DBMW foreach iterator to remove old entries.
 * @return  TRUE if entry must be deleted.
 */
static bool
tk_prune_old(void *key, void *value, size_t u_len, void *u_data)
{
	const kuid_t *id = key;
	const struct tokdata *td = value;
	time_delta_t d;

	(void) u_len;
	(void) u_data;

	d = delta_time(tm_time(), td->last_update);

	if (GNET_PROPERTY(dht_tcache_debug) > 2 && d > token_life) {
		g_debug("DHT TCACHE security token from %s expired",
			kuid_to_hex_string(id));
	}

	if (GNET_PROPERTY(dht_tcache_debug) > 5) {
		g_debug("DHT TCACHE %s: %s id=%s",
			G_STRFUNC, d > token_life ? "prune" : "keep ",
			kuid_to_hex_string(id));
	}

	return d > token_life;
}
Exemplo n.º 2
0
/**
 * Check whether key already holds data from the creator.
 *
 * @param id		the primary key
 * @param cid		the secondary key (creator's id)
 * @param store		whether to increment the store request count
 *
 * @return 64-bit DB key for the value if it does, 0 if key either does not
 * exist yet or does not hold data from the creator.
 */
uint64
keys_has(const kuid_t *id, const kuid_t *cid, bool store)
{
	struct keyinfo *ki;
	struct keydata *kd;
	uint64 dbkey;

	ki = hikset_lookup(keys, id);
	if (ki == NULL)
		return 0;

	if (store)
		ki->store_requests++;

	kd = get_keydata(id);
	if (kd == NULL)
		return 0;

	g_assert(ki->values == kd->values);

	dbkey = lookup_secondary(kd, cid);

	if (GNET_PROPERTY(dht_storage_debug) > 15) {
		g_debug("DHT lookup secondary for %s/%s => dbkey %s",
			kuid_to_hex_string(id), kuid_to_hex_string2(cid),
			uint64_to_string(dbkey));
	}

	return dbkey;
}
Exemplo n.º 3
0
/**
 * Retrieve cached security token for a given KUID.
 *
 * @param id		the KUID for which we'd like the security token
 * @param len_ptr	where the length of the security token is written
 * @param tok_ptr	where the address of the security token is written
 * @param time_ptr	where the last update time of token is writen
 *
 * @return TRUE if we found a token, with len_ptr and tok_ptr filled with
 * the information about the length and the token pointer.  Information is
 * returned from a static memory buffer so it must be perused immediately.
 */
bool
tcache_get(const kuid_t *id,
	uint8 *len_ptr, const void **tok_ptr, time_t *time_ptr)
{
	struct tokdata *td;

	g_assert(id != NULL);

	td = get_tokdata(id);

	if (NULL == td)
		return FALSE;

	if (delta_time(tm_time(), td->last_update) > token_life) {
		delete_tokdata(id);
		return FALSE;
	}

	if (len_ptr != NULL)	*len_ptr = td->length;
	if (tok_ptr != NULL)	*tok_ptr = td->token;
	if (time_ptr != NULL)	*time_ptr = td->last_update;

	if (GNET_PROPERTY(dht_tcache_debug) > 4) {
		char buf[80];
		bin_to_hex_buf(td->token, td->length, buf, sizeof buf);
		g_debug("DHT TCACHE security token for %s is %u-byte \"%s\" (%s)",
			kuid_to_hex_string(id), td->length, buf,
			compact_time(delta_time(tm_time(), td->last_update)));
	}

	gnet_stats_inc_general(GNR_DHT_CACHED_TOKENS_HITS);

	return TRUE;
}
Exemplo n.º 4
0
/**
 * DBMW foreach iterator to remove old entries.
 * @return  TRUE if entry must be deleted.
 */
static gboolean
prune_old(gpointer key, gpointer value, size_t u_len, gpointer u_data)
{
    const kuid_t *id = key;
    const struct lifedata *ld = value;
    time_delta_t d;
    gboolean expired;
    double p;

    (void) u_len;
    (void) u_data;

    d = delta_time(tm_time(), ld->last_seen);

    if (d <= STABLE_EXPIRE) {
        expired = FALSE;
        p = 1.0;
    } else {
        p = stable_still_alive_probability(ld->first_seen, ld->last_seen);
        expired = p < STABLE_PROBA;
    }

    if (GNET_PROPERTY(dht_stable_debug) > 4) {
        g_debug("DHT STABLE node %s life=%s last_seen=%s, p=%.2f%%%s",
                kuid_to_hex_string(id),
                compact_time(delta_time(ld->last_seen, ld->first_seen)),
                compact_time2(d), p * 100.0,
                expired ? " [EXPIRED]" : "");
    }

    return expired;
}
Exemplo n.º 5
0
/**
 * Delete known-to-be existing token data for specified KUID from database.
 */
static void
delete_tokdata(const kuid_t *id)
{
	dbmw_delete(db_tokdata, id);
	gnet_stats_dec_general(GNR_DHT_CACHED_TOKENS_HELD);

	if (GNET_PROPERTY(dht_tcache_debug) > 2)
		g_debug("DHT TCACHE security token from %s reclaimed",
			kuid_to_hex_string(id));

	if (GNET_PROPERTY(dht_tcache_debug_flags) & DBG_DSF_USR1) {
		g_debug("DHT TCACHE %s: stats=%s, count=%zu, id=%s",
			G_STRFUNC, uint64_to_string(
				gnet_stats_get_general(GNR_DHT_CACHED_TOKENS_HELD)),
			dbmw_count(db_tokdata), kuid_to_hex_string(id));
	}
}
Exemplo n.º 6
0
/**
 * Change node's version
 */
void
knode_change_version(knode_t *kn, uint8 major, uint8 minor)
{
	knode_check(kn);

	if (GNET_PROPERTY(dht_debug))
		g_warning("DHT node %s at %s changed from v%u.%u to v%u.%u",
			kuid_to_hex_string(kn->id),
			host_addr_port_to_string(kn->addr, kn->port),
			kn->major, kn->minor, major, minor);

	kn->major = major;
	kn->minor = minor;
}
Exemplo n.º 7
0
/**
 * Map iterator to record security tokens in the database.
 */
static void
record_token(void *key, void *value, void *unused_u)
{
	kuid_t *id = key;
	lookup_token_t *ltok = value;
	struct tokdata td;

	(void) unused_u;

	td.last_update = ltok->retrieved;
	td.length = ltok->token->length;
	td.token = td.length ? wcopy(ltok->token->v, td.length) : NULL;

	if (GNET_PROPERTY(dht_tcache_debug) > 4) {
		char buf[80];
		bin_to_hex_buf(td.token, td.length, buf, sizeof buf);
		g_debug("DHT TCACHE adding security token for %s: %u-byte \"%s\"",
			kuid_to_hex_string(id), td.length, buf);
	}

	/*
	 * Data is put in the DBMW cache and the dynamically allocated token
	 * will be freed via free_tokdata() when the cached entry is released.
	 */

	if (!dbmw_exists(db_tokdata, id->v))
		gnet_stats_inc_general(GNR_DHT_CACHED_TOKENS_HELD);

	dbmw_write(db_tokdata, id->v, &td, sizeof td);

	if (GNET_PROPERTY(dht_tcache_debug_flags) & DBG_DSF_USR1) {
		g_debug("DHT TCACHE %s: stats=%s, count=%zu, id=%s",
			G_STRFUNC, uint64_to_string(
				gnet_stats_get_general(GNR_DHT_CACHED_TOKENS_HELD)),
			dbmw_count(db_tokdata), kuid_to_hex_string(id));
	}
}
Exemplo n.º 8
0
/**
 * A value held under the key was updated and has a new expiration time.
 *
 * @param id		the primary key (existing already)
 * @param cid		the secondary key (creator's ID)
 * @param expire	expiration time for the value
 */
void
keys_update_value(const kuid_t *id, const kuid_t *cid, time_t expire)
{
	struct keyinfo *ki;
	struct keydata *kd;

	ki = hikset_lookup(keys, id);
	g_assert(ki != NULL);

	ki->next_expire = MIN(ki->next_expire, expire);
	kd = get_keydata(id);

	if (kd != NULL) {
		int low = 0, high = ki->values - 1;
		bool found = FALSE;

		while (low <= high) {
			int mid = low + (high - low) / 2;
			int c;

			g_assert(mid >= 0 && mid < ki->values);

			c = kuid_cmp(&kd->creators[mid], cid);

			if (0 == c) {
				kd->expire[mid] = expire;
				found = TRUE;
				break;
			} else if (c < 0) {
				low = mid + 1;
			} else {
				high = mid - 1;
			}
		}

		if (found) {
			dbmw_write(db_keydata, id, kd, sizeof *kd);
		} else if (GNET_PROPERTY(dht_keys_debug)) {
			g_warning("DHT KEYS %s(): creator %s not found under %s",
				G_STRFUNC, kuid_to_hex_string(cid), kuid_to_hex_string2(id));
		}
	}
}
Exemplo n.º 9
0
/**
 * Set iterator to remove keys with no values.
 */
static G_GNUC_COLD bool
keys_discard_if_empty(void *key, void *u_data)
{
	struct keyinfo *ki = key;

	keyinfo_check(ki);
	(void) u_data;

	if (0 == ki->values) {
		if (GNET_PROPERTY(dht_keys_debug) > 1) {
			g_debug("DHT KEYS no values retrieved for key %s, discarding",
				kuid_to_hex_string(ki->kuid));
		}

		keys_reclaim(ki, FALSE);
		return TRUE;
	}

	return FALSE;		/* Keep key */
}
Exemplo n.º 10
0
/**
 * Change node's vendor code.
 */
void
knode_change_vendor(knode_t *kn, vendor_code_t vcode)
{
	knode_check(kn);

	if (GNET_PROPERTY(dht_debug)) {
		char vc_old[VENDOR_CODE_BUFLEN];
		char vc_new[VENDOR_CODE_BUFLEN];

		vendor_code_to_string_buf(kn->vcode.u32, vc_old, sizeof vc_old);
		vendor_code_to_string_buf(vcode.u32, vc_new, sizeof vc_new);

		g_warning("DHT node %s at %s changed vendor from %s to %s",
			kuid_to_hex_string(kn->id),
			host_addr_port_to_string(kn->addr, kn->port),
			vc_old, vc_new);
	}

	kn->vcode = vcode;
}
Exemplo n.º 11
0
/**
 * Reclaim key info and data.
 *
 * @param ki			the keyinfo to reclaim
 * @param can_remove	whether to remove from the `keys' set
 */
static void
keys_reclaim(struct keyinfo *ki, bool can_remove)
{
	g_assert(ki);
	g_assert(0 == ki->values);

	if (GNET_PROPERTY(dht_storage_debug) > 2)
		g_debug("DHT STORE key %s reclaimed", kuid_to_hex_string(ki->kuid));

	dbmw_delete(db_keydata, ki->kuid);
	if (can_remove)
		hikset_remove(keys, &ki->kuid);

	gnet_stats_dec_general(GNR_DHT_KEYS_HELD);
	if (ki->flags & DHT_KEY_F_CACHED)
		gnet_stats_dec_general(GNR_DHT_CACHED_KEYS_HELD);

	kuid_atom_free_null(&ki->kuid);
	ki->magic = 0;
	WFREE(ki);
}
Exemplo n.º 12
0
/**
 * Get keydata from database.
 */
static struct keydata *
get_keydata(const kuid_t *id)
{
	struct keydata *kd;

	kd = dbmw_read(db_keydata, id, NULL);

	if (kd == NULL) {
		if (dbmw_has_ioerr(db_keydata)) {
			s_warning_once_per(LOG_PERIOD_MINUTE,
				"DBMW \"%s\" I/O error, bad things could happen...",
				dbmw_name(db_keydata));
		} else {
			s_warning_once_per(LOG_PERIOD_SECOND,
				"key %s exists but was not found in DBMW \"%s\"",
				kuid_to_hex_string(id), dbmw_name(db_keydata));
		}
		return NULL;
	}

	return kd;
}
Exemplo n.º 13
0
/**
 * Remove value from a key, discarding the association between the creator ID
 * and the 64-bit DB key.
 *
 * The keys is known to hold the value already.
 *
 * @param id		the primary key
 * @param cid		the secondary key (creator's ID)
 * @param dbkey		the 64-bit DB key (informational, for assertions)
 */
void
keys_remove_value(const kuid_t *id, const kuid_t *cid, uint64 dbkey)
{
	struct keyinfo *ki;
	struct keydata *kd;
	int idx;

	ki = hikset_lookup(keys, id);

	g_assert(ki);

	kd = get_keydata(id);
	if (NULL == kd)
		return;

	g_assert(kd->values);
	g_assert(kd->values == ki->values);
	g_assert(kd->values <= MAX_VALUES);

	idx = lookup_secondary_idx(kd, cid);

	g_assert(idx >= 0 && idx < kd->values);
	g_assert(dbkey == kd->dbkeys[idx]);

	ARRAY_REMOVE(kd->creators, idx, kd->values);
	ARRAY_REMOVE(kd->dbkeys,   idx, kd->values);
	ARRAY_REMOVE(kd->expire,   idx, kd->values);

	/*
	 * We do not synchronously delete empty keys.
	 *
	 * This lets us optimize the nominal case whereby a key loses all its
	 * values due to a STORE request causing a lifetime check.  But the
	 * STORE will precisely insert back another value.
	 *
	 * Hence lazy expiration also gives us the opportunity to further exploit
	 * caching in memory, the keyinfo being held there as a "cached" value.
	 *
	 * Reclaiming of dead keys happens during periodic key load computation.
	 */

	kd->values--;
	ki->values--;

	/*
	 * Recompute next expiration time.
	 */

	ki->next_expire = TIME_T_MAX;

	for (idx = 0; idx < ki->values; idx++) {
		ki->next_expire = MIN(ki->next_expire, kd->expire[idx]);
	}

	dbmw_write(db_keydata, id, kd, sizeof *kd);

	if (GNET_PROPERTY(dht_storage_debug) > 2) {
		g_debug("DHT STORE key %s now holds only %d/%d value%s, expire in %s",
			kuid_to_hex_string(id), ki->values, MAX_VALUES, plural(ki->values),
			compact_time(delta_time(ki->next_expire, tm_time())));
	}
}
Exemplo n.º 14
0
/**
 * See whether we can expire values stored under the key.
 *
 * @return TRUE if OK, FALSE if keyinfo was reclaimed due to inconsistency.
 */
static bool
keys_expire_values(struct keyinfo *ki, time_t now)
{
	struct keydata *kd;
	int i;
	int expired = 0;
	time_t next_expire = TIME_T_MAX;
	const char *reason = NULL;
	char buf[80];

	kd = get_keydata(ki->kuid);
	if (NULL == kd) {
		reason = "cannot retrieve associated keydata";
		goto discard_key;
	}

	if (kd->values != ki->values) {
		str_bprintf(buf, sizeof buf, "expected %u value%s, has %u in keydata",
			ki->values, plural(ki->values), kd->values);
		reason = buf;
		goto discard_key;
	}

	for (i = 0; i < kd->values; i++) {
		uint64 dbkey = kd->dbkeys[i];
		time_t expire = kd->expire[i];

		if (
			delta_time(now, expire) >= 0 &&
			values_has_expired(dbkey, now, &expire)		/* Updates `expire' */
		) {
			expired++;

			/*
			 * A mismatch indicates a severe database corruption, hence
			 * we use a mandatory warning.
			 */

			if (kd->expire[i] != expire) {
				g_warning("DHT KEYS mismatching expire time for value #%d in %s"
					" (held %u, should have been %u)",
					i, kuid_to_hex_string(ki->kuid),
					(uint) kd->expire[i], (uint) expire);
			}
		} else {
			next_expire = MIN(expire, next_expire);
		}
	}

	if (GNET_PROPERTY(dht_storage_debug) > 3)
		g_debug("DHT STORE key %s has %d expired value%s out of %d",
			kuid_to_hex_string(ki->kuid), expired, plural(expired),
			ki->values);

	if (next_expire != TIME_T_MAX)
		ki->next_expire = next_expire;	/* Next check, if values remain */

	/*
	 * Reclaim expired values, which will call keys_remove_value() for each
	 * value that has expired.
	 */

	values_reclaim_expired();
	keyinfo_check(ki);		/* Keyinfo reclaim is asynchronous */
	return TRUE;			/* OK */

discard_key:
	/*
	 * Get rid of the key since we have a missing / corrupted keydata.
	 */

	g_warning("DHT KEYS discarding corrupted key %s (%u value%s): %s",
		kuid_to_hex_string(ki->kuid), ki->values, plural(ki->values),
		reason);

	keys_reclaim(ki, TRUE);
	return FALSE;
}
Exemplo n.º 15
0
/**
 * Get key status (full and loaded boolean attributes).
 */
void
keys_get_status(const kuid_t *id, bool *full, bool *loaded)
{
	struct keyinfo *ki;
	time_t now;

	g_assert(id);
	g_assert(full);
	g_assert(loaded);

	*full = FALSE;
	*loaded = FALSE;

	ki = hikset_lookup(keys, id);
	if (ki == NULL)
		return;

	keyinfo_check(ki);

	if (GNET_PROPERTY(dht_storage_debug) > 1) {
		g_debug("DHT STORE key %s holds %d/%d value%s, "
			"load avg: get = %g [%s], store = %g [%s], expire in %s",
			kuid_to_hex_string(id), ki->values, MAX_VALUES, plural(ki->values),
			(int) (ki->get_req_load * 100) / 100.0,
			ki->get_req_load >= LOAD_GET_THRESH ? "LOADED" : "OK",
			(int) (ki->store_req_load * 100) / 100.0,
			ki->store_req_load >= LOAD_STO_THRESH ? "LOADED" : "OK",
			compact_time(delta_time(ki->next_expire, tm_time())));
	}

	if (ki->get_req_load >= LOAD_GET_THRESH) {
		*loaded = TRUE;
	} else if (ki->get_requests) {
		float limit = LOAD_GET_THRESH / LOAD_SMOOTH -
			(1.0 - LOAD_SMOOTH) / LOAD_SMOOTH * ki->get_req_load;

		/*
		 * Look whether the current amount of get requests is sufficient to
		 * bring the EMA above the threshold at the next update.
		 */

		if (1.0 * ki->get_requests > limit)
			*loaded = TRUE;
	}

	/*
	 * Check whether we reached the expiration time of one of the values held.
	 * Try to expire values before answering.
	 *
	 * NB: even if all the values are collected from the key, deletion of the
	 * `ki' structure will not happen immediately: this is done asynchronously
	 * to avoid disabling a `ki' within a call chain using it.
	 */

	now = tm_time();

	if (now >= ki->next_expire) {
		if (!keys_expire_values(ki, now))
			return;		/* Key info reclaimed */
	}

	if (ki->values >= MAX_VALUES)
		*full = TRUE;
}
Exemplo n.º 16
0
/**
 * Fill supplied value vector with the DHT values we have under the key that
 * match the specifications: among those bearing the specified secondary keys
 * (or all of them if no secondary keys are supplied), return only those with
 * the proper DHT value type.
 *
 * @param id				the primary key of the value
 * @param type				type of DHT value they want
 * @param secondary			optional secondary keys
 * @param secondary_count	amount of secondary keys supplied
 * @param valvec			value vector where results are stored
 * @param valcnt			size of value vector
 * @param loadptr			where to write the average request load for key
 * @param cached			if non-NULL, filled with whether key was cached
 *
 * @return amount of values filled into valvec.  The values are dynamically
 * created and must be freed by caller through dht_value_free().
 */
int
keys_get(const kuid_t *id, dht_value_type_t type,
	kuid_t **secondary, int secondary_count, dht_value_t **valvec, int valcnt,
	float *loadptr, bool *cached)
{
	struct keyinfo *ki;
	struct keydata *kd;
	int i;
	int vcnt = valcnt;
	dht_value_t **vvec = valvec;

	g_assert(secondary_count == 0 || secondary != NULL);
	g_assert(valvec);
	g_assert(valcnt > 0);
	g_assert(loadptr);

	ki = hikset_lookup(keys, id);

	g_assert(ki);	/* If called, we know the key exists */

	if (GNET_PROPERTY(dht_storage_debug) > 5)
		g_debug("DHT FETCH key %s (load = %g, current reqs = %u) type %s"
			" with %d secondary key%s",
			kuid_to_hex_string(id), ki->get_req_load, ki->get_requests,
			dht_value_type_to_string(type),
			secondary_count, plural(secondary_count));

	*loadptr = ki->get_req_load;
	ki->get_requests++;

	kd = get_keydata(id);
	if (kd == NULL)				/* DB failure */
		return 0;

	/*
	 * If secondary keys were requested, lookup them up and make sure
	 * they have the right DHT type (or skip them).
	 */

	for (i = 0; i < secondary_count && vcnt > 0; i++) {
		uint64 dbkey = lookup_secondary(kd, secondary[i]);
		dht_value_t *v;

		if (0 == dbkey)
			continue;

		v = values_get(dbkey, type);
		if (v == NULL)
			continue;

		g_assert(kuid_eq(dht_value_key(v), id));

		if (GNET_PROPERTY(dht_storage_debug) > 5)
			g_debug("DHT FETCH key %s via secondary key %s has matching %s",
				kuid_to_hex_string(id), kuid_to_hex_string2(secondary[i]),
				dht_value_to_string(v));

		*vvec++ = v;
		vcnt--;
	}

	/*
	 * Don't count secondary-key fetches in the local hit stats: in order to
	 * be able to get these fetches, we must have initially provided the
	 * list of these keys, and thus we have already traversed the code below
	 * for that fetch, which accounted the hit already.
	 */

	if (secondary_count) {
		int n = vvec - valvec;		/* Amount of entries filled */

		gnet_stats_count_general(GNR_DHT_CLAIMED_SECONDARY_KEYS, n);
		if (ki->flags & DHT_KEY_F_CACHED)
			gnet_stats_count_general(GNR_DHT_CLAIMED_CACHED_SECONDARY_KEYS, n);

		goto done;
	}

	/*
	 * No secondary keys specified.  Look them all up.
	 */

	for (i = 0; i < kd->values && vcnt > 0; i++) {
		uint64 dbkey = kd->dbkeys[i];
		dht_value_t *v;

		g_assert(0 != dbkey);

		v = values_get(dbkey, type);
		if (v == NULL)
			continue;

		g_assert(kuid_eq(dht_value_key(v), id));

		if (GNET_PROPERTY(dht_storage_debug) > 5)
			g_debug("DHT FETCH key %s has matching %s",
				kuid_to_hex_string(id), dht_value_to_string(v));

		*vvec++ = v;
		vcnt--;
	}

	/*
	 * Stats update: we count all the hits, plus successful hits on keys
	 * that do not fall within our k-ball, i.e. keys for which we act as
	 * a "cache".  Note that our k-ball frontier can evolve through time,
	 * so we rely on the DHT_KEY_F_CACHED flag, positionned at creation time.
	 */

	if (vvec != valvec) {
		gnet_stats_inc_general(GNR_DHT_FETCH_LOCAL_HITS);
		if (ki->flags & DHT_KEY_F_CACHED)
			gnet_stats_inc_general(GNR_DHT_FETCH_LOCAL_CACHED_HITS);
	}

done:

	if (cached)
		*cached = (ki->flags & DHT_KEY_F_CACHED) ? TRUE : FALSE;

	return vvec - valvec;		/* Amount of entries filled */
}
Exemplo n.º 17
0
/**
 * Add value to a key, recording the new association between the KUID of the
 * creator (secondary key) and the 64-bit DB key under which the value is
 * stored.
 *
 * @param id		the primary key (may not exist yet)
 * @param cid		the secondary key (creator's ID)
 * @param dbkey		the 64-bit DB key
 * @param expire	expiration time for the value
 */
void
keys_add_value(const kuid_t *id, const kuid_t *cid,
	uint64 dbkey, time_t expire)
{
	struct keyinfo *ki;
	struct keydata *kd;
	struct keydata new_kd;

	ki = hikset_lookup(keys, id);

	/*
	 * If we're storing the first value under a key, we do not have any
	 * keyinfo structure yet.
	 */

	if (NULL == ki) {
		size_t common;
		bool in_kball;

		common = kuid_common_prefix(get_our_kuid(), id);
		in_kball = bits_within_kball(common);

		if (GNET_PROPERTY(dht_storage_debug) > 5)
			g_debug("DHT STORE new %s %s (%zu common bit%s) with creator %s",
				in_kball ? "key" : "cached key",
				kuid_to_hex_string(id), common, plural(common),
				kuid_to_hex_string2(cid));

		ki = allocate_keyinfo(id, common);
		ki->next_expire = expire;
		ki->flags = in_kball ? 0 : DHT_KEY_F_CACHED;

		hikset_insert_key(keys, &ki->kuid);

		kd = &new_kd;
		kd->values = 0;						/* will be incremented below */
		kd->creators[0] = *cid;				/* struct copy */
		kd->dbkeys[0] = dbkey;
		kd->expire[0] = expire;

		gnet_stats_inc_general(GNR_DHT_KEYS_HELD);
		if (!in_kball)
			gnet_stats_inc_general(GNR_DHT_CACHED_KEYS_HELD);
	} else {
		int low = 0;
		int high = ki->values - 1;

		kd = get_keydata(id);

		if (NULL == kd)
			return;

		g_assert(kd->values == ki->values);
		g_assert(kd->values < MAX_VALUES);

		if (GNET_PROPERTY(dht_storage_debug) > 5)
			g_debug("DHT STORE existing key %s (%u common bit%s) "
				"has new creator %s",
				kuid_to_hex_string(id), ki->common_bits,
				plural(ki->common_bits), kuid_to_hex_string2(cid));

		/*
		 * Keys are collected asynchronously, so it is possible that
		 * the key structure still exists, yet holds no values.  If this
		 * happens, then we win because we spared the useless deletion of
		 * the key structure to recreate it a little bit later.
		 */

		if (0 == kd->values)
			goto empty;

		/*
		 * Insert KUID of creator in array, which must be kept sorted.
		 * We perform a binary insertion.
		 */

		while (low <= high) {
			int mid = low + (high - low) / 2;
			int c;

			g_assert(mid >= 0 && mid < ki->values);

			c = kuid_cmp(&kd->creators[mid], cid);

			if (0 == c)
				g_error("new creator KUID %s must not already be present",
					kuid_to_hex_string(cid));
			else if (c < 0)
				low = mid + 1;
			else
				high = mid - 1;
		}

		/* Make room for inserting new item at `low' */

		ARRAY_FIXED_MAKEROOM(kd->creators, low, kd->values);
		ARRAY_FIXED_MAKEROOM(kd->dbkeys,   low, kd->values);
		ARRAY_FIXED_MAKEROOM(kd->expire,   low, kd->values);

		/* FALL THROUGH */

	empty:
		/* Insert new item at `low' */

		kd->creators[low] = *cid;			/* struct copy */
		kd->dbkeys[low] = dbkey;
		kd->expire[low] = expire;

		ki->next_expire = MIN(ki->next_expire, expire);
	}

	kd->values++;
	ki->values++;

	dbmw_write(db_keydata, id, kd, sizeof *kd);

	if (GNET_PROPERTY(dht_storage_debug) > 2)
		g_debug("DHT STORE %s key %s now holds %d/%d value%s",
			&new_kd == kd ? "new" : "existing",
			kuid_to_hex_string(id), ki->values, MAX_VALUES, plural(ki->values));
}