/**
* Retry publishing after some delay.
*
* @param pe the entry to publish
* @param delay delay in seconds
* @param msg if non-NULL, logging message explaining the delay
*/
static void
publisher_retry(struct publisher_entry *pe, int delay, const char *msg)
{
struct pubdata *pd;
publisher_check(pe);
g_assert(NULL == pe->publish_ev);
g_assert(delay > 0);
pd = get_pubdata(pe->sha1);
if (pd != NULL) {
pd->next_enqueue = time_advance(tm_time(), UNSIGNED(delay));
dbmw_write(db_pubdata, pe->sha1, pd, sizeof *pd);
}
pe->publish_ev = cq_insert(publish_cq, delay * 1000, handle_entry, pe);
pe->last_delayed = tm_time();
if (GNET_PROPERTY(publisher_debug) > 3) {
shared_file_t *sf = shared_file_by_sha1(pe->sha1);
g_debug("PUBLISHER will retry SHA-1 %s %s\"%s\" in %s: %s",
sha1_to_string(pe->sha1),
(sf && sf != SHARE_REBUILDING && shared_file_is_partial(sf)) ?
"partial " : "",
(sf && sf != SHARE_REBUILDING) ? shared_file_name_nfc(sf) : "",
compact_time(delay), msg != NULL ? msg : "<no reason>");
shared_file_unref(&sf);
}
}
/**
* Delete pubdata from database.
*/
static void
delete_pubdata(const sha1_t *sha1)
{
dbmw_delete(db_pubdata, sha1);
if (GNET_PROPERTY(publisher_debug) > 2) {
shared_file_t *sf = shared_file_by_sha1(sha1);
g_debug("PUBLISHER SHA-1 %s %s\"%s\" reclaimed",
sha1_to_string(sha1),
(sf && sf != SHARE_REBUILDING && shared_file_is_partial(sf)) ?
"partial " : "",
(sf && sf != SHARE_REBUILDING) ? shared_file_name_nfc(sf) : "");
shared_file_unref(&sf);
}
}
static void
upload_stats_dump_item(gpointer p, gpointer user_data)
{
const shared_file_t *sf;
FILE *out = user_data;
struct ul_stats *s = p;
char rtime_buf[TIME_T_DEC_BUFLEN];
char dtime_buf[TIME_T_DEC_BUFLEN];
const char *pathname;
char *escaped;
g_assert(NULL != s);
sf = s->sha1 ? shared_file_by_sha1(s->sha1) : NULL;
sf = SHARE_REBUILDING != sf ? sf : NULL;
if (sf) {
pathname = shared_file_path(sf);
} else {
pathname = s->pathname;
}
escaped = url_escape_cntrl(pathname);
time_t_to_string_buf(s->rtime, rtime_buf, sizeof rtime_buf);
time_t_to_string_buf(s->dtime, dtime_buf, sizeof dtime_buf);
fprintf(out, "%s\t%s\t%u\t%u\t%lu\t%lu\t%s\t%s\t%s\n",
escaped,
uint64_to_string(s->size),
s->attempts,
s->complete,
(unsigned long) (s->bytes_sent >> 32),
(unsigned long) (s->bytes_sent & 0xffffffff),
rtime_buf,
dtime_buf,
s->sha1 ? sha1_base32(s->sha1) : "*");
if (escaped != pathname) { /* File had escaped chars */
HFREE_NULL(escaped);
}
}
/**
* Handle a SHA-1 entry, publishing its alt-loc to the DHT if still shared.
*/
static void
publisher_handle(struct publisher_entry *pe)
{
shared_file_t *sf;
bool is_partial = FALSE;
int alt_locs;
time_delta_t min_uptime;
uint32 avg_uptime;
publisher_check(pe);
g_assert(NULL == pe->publish_ev);
sf = shared_file_by_sha1(pe->sha1);
/*
* Remove SHA1 if no longer shared.
*/
if (NULL == sf) {
fileinfo_t *fi = file_info_by_sha1(pe->sha1);
/*
* If a partial file has lees than the minimum amount of data for PFSP,
* shared_file_by_sha1() will return NULL, hence we need to explicitly
* check for existence through file_info_by_sha1() and that the file
* still exists.
*/
if (fi != NULL && file_exists(fi->pathname)) {
/* Waiting for more data to be able to share, or PFSP re-enabled */
publisher_retry(pe, PUBLISH_BUSY, "partial file missing");
return;
}
if (GNET_PROPERTY(publisher_debug)) {
g_debug("PUBLISHER SHA-1 %s is no longer shared",
sha1_to_string(pe->sha1));
}
publisher_entry_free(pe, TRUE);
return;
}
/*
* Wait when rebuilding the library.
*/
if (SHARE_REBUILDING == sf) {
publisher_retry(pe, PUBLISH_BUSY, "library being rebuilt");
return;
}
is_partial = shared_file_is_partial(sf);
/*
* If the SHA1 is not available, wait.
*/
if (
!is_partial &&
(!sha1_hash_available(sf) || !sha1_hash_is_uptodate(sf))
) {
publisher_retry(pe, PUBLISH_BUSY, "SHA-1 of file unknown yet");
goto done;
}
/*
* Look whether this node has a sufficient average uptime.
*
* We're stricter to publish partial files because we want to favor
* publishing of full files in the DHT, and the benefits of publishing
* partial entries come only if we're up for a long enough time.
*
* Since publishing imposes lookup traffic in the DHT, it is not efficient
* to have transient nodes publish file sharing information because this
* will likely never be useful.
*/
min_uptime = PUBLISH_TRANSIENT;
if (is_partial)
min_uptime *= 2;
avg_uptime = get_average_servent_uptime(tm_time());
if (avg_uptime < UNSIGNED(min_uptime)) {
time_delta_t delay = min_uptime - avg_uptime;
delay = MAX(delay, PUBLISH_BUSY);
publisher_retry(pe, delay, "minimum average uptime not reached yet");
goto done;
}
/*
* If we are dealing with a file for which we know enough alternate
* locations, assume it is popular and do not publish it yet.
*
* We do not publish the SHA-1 of a partial file for which we know
* of at least two alternate locations because the purpose of us publishing
* these partial SHA-1s is to attract other PFSP-aware hosts and
* recreate a mesh.
*/
alt_locs = dmesh_count(pe->sha1);
is_partial = is_partial && !shared_file_is_finished(sf);
if (alt_locs > (is_partial ? PUBLISH_PARTIAL_MAX : PUBLISH_DMESH_MAX)) {
if (GNET_PROPERTY(publisher_debug)) {
g_debug("PUBLISHER SHA-1 %s %s\"%s\" has %d download mesh "
"entr%s, skipped", sha1_to_string(pe->sha1),
is_partial ? "partial " : "", shared_file_name_nfc(sf),
alt_locs, plural_y(alt_locs));
}
publisher_hold(pe, PUBLISH_POPULAR, "popular file");
goto done;
}
/*
* If the DHT is not enabled, postpone processing.
*/
if (!dht_enabled()) {
publisher_hold(pe, PUBLISH_BUSY, "DHT disabled");
goto done;
}
/*
* If this is a partial file for which we have less than the minimum
* for PFSP sharing, or if PFSP has been disabled, skip it.
*/
if (shared_file_is_partial(sf)) {
fileinfo_t *fi = shared_file_fileinfo(sf);
if (
!file_info_partial_shareable(fi) ||
fi->done < GNET_PROPERTY(pfsp_minimum_filesize)
) {
publisher_hold(pe, PUBLISH_BUSY, "PFSP minima not reached");
goto done;
}
}
/*
* Check whether it is time to process the entry, in case we're
* restarting quickly after a shutdown.
*/
if (0 == pe->last_publish) {
struct pubdata *pd = get_pubdata(pe->sha1);
if (pd != NULL) {
time_t now = tm_time();
time_delta_t enqueue = delta_time(pd->next_enqueue, now);
time_delta_t expire = delta_time(pd->expiration, now);
if (enqueue > 0 && (0 == pd->expiration || expire > 0)) {
int delay = MIN(enqueue, PUBLISH_POPULAR);
if (pd->expiration != 0)
delay = MIN(delay, expire);
if (GNET_PROPERTY(publisher_debug) > 1) {
g_debug("PUBLISHER SHA-1 %s delayed by %s",
sha1_to_string(pe->sha1), compact_time(enqueue));
}
publisher_retry(pe, delay, "first-time delay");
goto done;
}
}
}
/*
* Cancel possible remaining backgrounded publishing.
*/
if (pe->backgrounded) {
pdht_cancel_file(pe->sha1, FALSE);
pe->backgrounded = FALSE;
}
/*
* OK, we can publish this alternate location.
*/
if (pe->last_publish) {
if (GNET_PROPERTY(publisher_debug) > 2) {
g_debug("PUBLISHER SHA-1 %s re-enqueued %d secs "
"after last publish", sha1_to_string(pe->sha1),
(int) delta_time(tm_time(), pe->last_publish));
}
}
pe->last_enqueued = tm_time();
pdht_publish_file(sf, publisher_done, pe);
/* FALL THROUGH */
done:
shared_file_unref(&sf);
}
/**
* Publishing callback invoked when asynchronous publication is completed,
* or ended with an error.
*
* @return TRUE if we accept the publishing, FALSE otherwise to get the
* publishing layer to continue attempts to failed STORE roots and report
* on progress using the same callback.
*/
static bool
publisher_done(void *arg, pdht_error_t code, const pdht_info_t *info)
{
struct publisher_entry *pe = arg;
struct pubdata *pd;
int delay = PUBLISH_BUSY;
bool expired = FALSE;
bool accepted = TRUE;
publisher_check(pe);
pd = get_pubdata(pe->sha1);
/*
* Update stats on republishing before value expiration.
*/
if (PDHT_E_OK == code) {
if (pe->last_publish && info->roots > 0) {
if (pd != NULL) {
if (pd->expiration && delta_time(tm_time(), pd->expiration) > 0)
expired = TRUE;
} else {
time_delta_t elapsed = delta_time(tm_time(), pe->last_publish);
if (elapsed > DHT_VALUE_ALOC_EXPIRE)
expired = TRUE;
}
if (expired)
gnet_stats_inc_general(GNR_DHT_REPUBLISHED_LATE);
}
}
/*
* Compute retry delay.
*/
switch (code) {
case PDHT_E_OK:
/*
* If we were not able to publish to KDA_K nodes, decrease the
* delay before republishing. We use a non-linear decimation of
* the republish time, as a function of the number of nodes to which
* we could publish.
*/
delay = publisher_delay(info, DHT_VALUE_ALOC_EXPIRE);
accepted = publisher_is_acceptable(info);
break;
case PDHT_E_POPULAR:
/*
* Compute the suitable delay: the first time, we use PUBLISH_POPULAR,
* and then we double each time until we reach PUBLISH_POPULAR_MAX.
*
* If we already tried to publish the entry, pe->last_delayed will
* be non-zero.
*/
if (0 != pe->last_delayed) {
time_delta_t elapsed = delta_time(tm_time(), pe->last_delayed);
if (elapsed < PUBLISH_POPULAR) {
delay = PUBLISH_POPULAR;
} else if (elapsed >= PUBLISH_POPULAR_MAX / 2) {
delay = PUBLISH_POPULAR_MAX;
} else {
delay = elapsed * 2;
}
} else {
delay = PUBLISH_POPULAR;
}
break;
case PDHT_E_NOT_SHARED:
case PDHT_E_LOOKUP_EXPIRED:
case PDHT_E_LOOKUP:
case PDHT_E_UDP_CLOGGED:
case PDHT_E_PUBLISH_EXPIRED:
case PDHT_E_PUBLISH_ERROR:
case PDHT_E_SHA1:
case PDHT_E_PENDING:
case PDHT_E_CANCELLED:
case PDHT_E_GGEP:
case PDHT_E_NONE:
delay = PUBLISH_BUSY;
break;
case PDHT_E_MAX:
g_assert_not_reached();
}
/*
* For a backgrounded entry publishing, we need to adjust the computed
* delay with the time that was elapsed
*/
g_assert(!pe->backgrounded == !(pe->publish_ev != NULL));
if (pe->backgrounded) {
time_delta_t elapsed = delta_time(tm_time(), pe->last_delayed);
g_assert(pe->last_delayed > 0);
cq_cancel(&pe->publish_ev);
if (delay > elapsed) {
delay -= elapsed;
} else {
delay = 1;
}
}
/*
* Logging.
*/
if (GNET_PROPERTY(publisher_debug) > 1) {
shared_file_t *sf = shared_file_by_sha1(pe->sha1);
char retry[80];
char after[80];
const char *late = "";
after[0] = '\0';
if (pe->last_publish) {
time_delta_t elapsed = delta_time(tm_time(), pe->last_publish);
str_bprintf(after, sizeof after,
" after %s", compact_time(elapsed));
if (pd != NULL) {
if (expired)
late = "late, ";
} else {
late = "no data, ";
}
}
str_bprintf(retry, sizeof retry, "%s", compact_time(delay));
g_debug("PUBLISHER SHA-1 %s %s%s\"%s\" %spublished to %u node%s%s: %s"
" (%stook %s, total %u node%s, proba %.3f%%, retry in %s,"
" %s bg, path %u) [%s]",
sha1_to_string(pe->sha1),
pe->backgrounded ? "[bg] " : "",
(sf && sf != SHARE_REBUILDING && shared_file_is_partial(sf)) ?
"partial " : "",
(sf && sf != SHARE_REBUILDING) ? shared_file_name_nfc(sf) : "",
pe->last_publish ? "re" : "",
info->roots, plural(info->roots),
after, pdht_strerror(code), late,
compact_time(delta_time(tm_time(), pe->last_enqueued)),
info->all_roots, plural(info->all_roots),
info->presence * 100.0, retry,
info->can_bg ? "can" : "no", info->path_len,
accepted ? "OK" : "INCOMPLETE");
shared_file_unref(&sf);
}
/*
* Update last publishing time and remember expiration time.
*/
if (PDHT_E_OK == code && info->roots > 0) {
pe->last_publish = tm_time();
if (pd != NULL) {
pd->expiration =
time_advance(pe->last_publish, DHT_VALUE_ALOC_EXPIRE);
dbmw_write(db_pubdata, pe->sha1, pd, sizeof *pd);
}
}
/*
* If entry was deemed popular, we're going to delay its republishing
* by a larger amount of time and any data we published already about
* it will surely expire. Since this is our decision, we do not want
* to be told that republishing, if it occurs again, was done later than
* required. Hence call publisher_hold() to mark that we don't care.
*/
if (PDHT_E_POPULAR == code)
publisher_hold(pe, delay, "popular entry");
else
publisher_retry(pe, delay, accepted ? "accepted publish" : "published");
pe->backgrounded = !accepted;
return accepted;
}
