/**
* External interface to call for getting the hash for a shared_file.
*/
void
request_sha1(shared_file_t *sf)
{
struct sha1_cache_entry *cached;
shared_file_check(sf);
if (!shared_file_indexed(sf))
return; /* "stale" shared file, has been superseded or removed */
cached = hikset_lookup(sha1_cache, shared_file_path(sf));
if (cached && cached_entry_up_to_date(cached, sf)) {
cache_dirty = TRUE;
cached->shared = TRUE;
shared_file_set_sha1(sf, cached->sha1);
shared_file_set_tth(sf, cached->tth);
request_tigertree(sf, NULL == cached->tth);
} else {
if (GNET_PROPERTY(share_debug) > 1) {
if (cached)
g_debug("cached SHA1 entry for \"%s\" outdated: "
"had mtime %lu, now %lu",
shared_file_path(sf),
(ulong) cached->mtime,
(ulong) shared_file_modification_time(sf));
else
g_debug("queuing \"%s\" for SHA1 computation",
shared_file_path(sf));
}
queue_shared_file_for_sha1_computation(sf);
}
}
static bool
request_tigertree_callback(const struct verify *ctx, enum verify_status status,
void *user_data)
{
shared_file_t *sf = user_data;
shared_file_check(sf);
switch (status) {
case VERIFY_START:
if (!(SHARE_F_INDEXED & shared_file_flags(sf))) {
/*
* After a rescan, there might be files in the queue which are
* no longer shared.
*/
if (GNET_PROPERTY(verify_debug) > 1) {
g_debug("skipping TTH computation for %s: no longer shared",
shared_file_path(sf));
}
return FALSE;
}
if (
shared_file_tth(sf) &&
tth_cache_lookup(shared_file_tth(sf), shared_file_size(sf)) > 0
) {
if (
GNET_PROPERTY(tigertree_debug) > 1 ||
GNET_PROPERTY(verify_debug) > 1
) {
g_debug("TTH for %s is already cached (%s)",
shared_file_path(sf), tth_base32(shared_file_tth(sf)));
}
return FALSE;
}
gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, TRUE);
return TRUE;
case VERIFY_PROGRESS:
return 0 != (SHARE_F_INDEXED & shared_file_flags(sf));
case VERIFY_DONE:
{
const struct tth *tth = verify_tth_digest(ctx);
huge_update_hashes(sf, shared_file_sha1(sf), tth);
tth_cache_insert(tth, verify_tth_leaves(ctx),
verify_tth_leave_count(ctx));
}
/* FALL THROUGH */
case VERIFY_ERROR:
case VERIFY_SHUTDOWN:
shared_file_unref(&sf);
gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, FALSE);
return TRUE;
case VERIFY_INVALID:
break;
}
g_assert_not_reached();
return FALSE;
}
void
request_tigertree(shared_file_t *sf, bool high_priority)
{
int inserted;
verify_tth_init();
shared_file_check(sf);
g_return_if_fail(shared_file_is_finished(sf));
if (!shared_file_is_servable(sf))
return; /* "stale" shared file, has been superseded or removed */
/*
* This routine can be called when the VERIFY_DONE event is received by
* huge_verify_callback(). We may have already shutdown the TTH
* verification thread.
*/
if G_UNLIKELY(NULL == verify_tth.verify)
return;
sf = shared_file_ref(sf);
inserted = verify_enqueue(verify_tth.verify, high_priority,
shared_file_path(sf), 0, shared_file_size(sf),
request_tigertree_callback, sf);
if (!inserted)
shared_file_unref(&sf);
}
/**
* Add `comp' to the current completed count, and update the amount of
* bytes transferred. Note that `comp' can be zero.
* When `update_dtime' is TRUE, we update the "done time", otherwise we
* change the "last request time".
*
* If the row does not exist (race condition: deleted since upload started),
* recreate one.
*/
static void
upload_stats_file_add(
const shared_file_t *sf,
int comp, guint64 sent, gboolean update_dtime)
{
const char *pathname = shared_file_path(sf);
filesize_t size = shared_file_size(sf);
struct ul_stats *s;
const struct sha1 *sha1;
g_assert(comp >= 0);
sha1 = sha1_hash_available(sf) ? shared_file_sha1(sf) : NULL;
/* find this file in the ul_stats_clist */
s = upload_stats_find(sha1, pathname, size);
/* increment the completed counter */
if (NULL == s) {
/* uh oh, row has since been deleted, add it: 1 attempt */
upload_stats_add(pathname, size, shared_file_name_nfc(sf),
1, comp, sent, tm_time(), tm_time(), sha1);
} else {
s->bytes_sent += sent;
s->norm = 1.0 * s->bytes_sent / s->size;
s->complete += comp;
if (update_dtime)
s->dtime = tm_time();
else
s->rtime = tm_time();
gcu_upload_stats_gui_update(s);
}
dirty = TRUE; /* Request asynchronous save of stats */
}
/**
* Called when an upload starts.
*/
void
upload_stats_file_begin(const shared_file_t *sf)
{
struct ul_stats *s;
const char *pathname;
filesize_t size;
const struct sha1 *sha1;
g_return_if_fail(sf);
pathname = shared_file_path(sf);
size = shared_file_size(sf);
sha1 = sha1_hash_available(sf) ? shared_file_sha1(sf) : NULL;
/* find this file in the ul_stats_clist */
s = upload_stats_find(sha1, pathname, size);
/* increment the attempted counter */
if (NULL == s) {
upload_stats_add(pathname, size, shared_file_name_nfc(sf),
1, 0, 0, tm_time(), 0, sha1);
} else {
s->attempts++;
s->rtime = tm_time();
gcu_upload_stats_gui_update(s);
}
dirty = TRUE; /* Request asynchronous save of stats */
}
/**
* Make sure the filename associated to a SHA1 is given the name of
* the shared file and no longer bears the name of the partial file.
* This can happen when the partial file is seeded then the file is
* renamed and shared.
*/
void
upload_stats_enforce_local_filename(const shared_file_t *sf)
{
struct ul_stats *s;
const struct sha1 *sha1;
const char *name;
if (!upload_stats_by_sha1)
return; /* Nothing known by SHA1 yet */
sha1 = sha1_hash_available(sf) ? shared_file_sha1(sf) : NULL;
if (!sha1)
return; /* File's SHA1 not known yet, nothing to do here */
s = g_hash_table_lookup(upload_stats_by_sha1, sha1);
if (NULL == s)
return; /* SHA1 not in stats, nothing to do */
name = shared_file_name_nfc(sf);
if (name == s->filename) /* Both are string atoms */
return; /* Everything is fine */
/*
* We need to update the filename to match the shared file.
*/
hash_list_remove(upload_stats_list, s);
atom_str_change(&s->pathname, shared_file_path(sf));
atom_str_change(&s->filename, name);
hash_list_append(upload_stats_list, s);
gcu_upload_stats_gui_update_name(s);
}
/**
* External interface to check whether the sha1 for shared_file is known.
*/
bool
sha1_is_cached(const shared_file_t *sf)
{
const struct sha1_cache_entry *cached;
cached = hikset_lookup(sha1_cache, shared_file_path(sf));
return cached && cached_entry_up_to_date(cached, sf);
}
static bool
huge_spam_check(shared_file_t *sf, const struct sha1 *sha1)
{
if (NULL != sha1 && spam_sha1_check(sha1)) {
g_warning("file \"%s\" is listed as spam (SHA1)", shared_file_path(sf));
return TRUE;
}
if (
spam_check_filename_size(shared_file_name_nfc(sf),
shared_file_size(sf))
) {
g_warning("file \"%s\" is listed as spam (Name)", shared_file_path(sf));
return TRUE;
}
return FALSE;
}
/**
* Look whether we still need to compute the SHA1 of the given shared file
* by looking into our in-core cache to see whether the entry we have is
* up-to-date.
*
* @param sf the shared file for which we want to compute the SHA1
*
* @return TRUE if the file need SHA1 recomputation.
*/
static bool
huge_need_sha1(shared_file_t *sf)
{
struct sha1_cache_entry *cached;
shared_file_check(sf);
/*
* After a rescan, there might be files in the queue which are
* no longer shared.
*/
if (!shared_file_indexed(sf))
return FALSE;
if G_UNLIKELY(NULL == sha1_cache)
return FALSE; /* Shutdown occurred (processing TEQ event?) */
cached = hikset_lookup(sha1_cache, shared_file_path(sf));
if (cached != NULL) {
filestat_t sb;
if (-1 == stat(shared_file_path(sf), &sb)) {
g_warning("ignoring SHA1 recomputation request for \"%s\": %m",
shared_file_path(sf));
return FALSE;
}
if (
cached->size + (fileoffset_t) 0 == sb.st_size + (filesize_t) 0 &&
cached->mtime == sb.st_mtime
) {
if (GNET_PROPERTY(share_debug) > 1) {
g_warning("ignoring duplicate SHA1 work for \"%s\"",
shared_file_path(sf));
}
return FALSE;
}
}
return TRUE;
}
bool
huge_update_hashes(shared_file_t *sf,
const struct sha1 *sha1, const struct tth *tth)
{
struct sha1_cache_entry *cached;
filestat_t sb;
shared_file_check(sf);
g_return_val_if_fail(sha1, FALSE);
/*
* Make sure the file's timestamp is still accurate.
*/
if (-1 == stat(shared_file_path(sf), &sb)) {
g_warning("discarding SHA1 for file \"%s\": can't stat(): %m",
shared_file_path(sf));
shared_file_remove(sf);
return TRUE;
}
if (sb.st_mtime != shared_file_modification_time(sf)) {
g_warning("file \"%s\" was modified whilst SHA1 was computed",
shared_file_path(sf));
shared_file_set_modification_time(sf, sb.st_mtime);
request_sha1(sf); /* Retry! */
return TRUE;
}
if (huge_spam_check(sf, sha1)) {
shared_file_remove(sf);
return FALSE;
}
shared_file_set_sha1(sf, sha1);
shared_file_set_tth(sf, tth);
/* Update cache */
cached = hikset_lookup(sha1_cache, shared_file_path(sf));
if (cached) {
update_volatile_cache(cached, shared_file_size(sf),
shared_file_modification_time(sf), sha1, tth);
cache_dirty = TRUE;
/* Dump the cache at most about once per minute. */
if (!cache_dumped || delta_time(tm_time(), cache_dumped) > 60) {
dump_cache(FALSE);
}
} else {
add_volatile_cache_entry(shared_file_path(sf),
shared_file_size(sf), shared_file_modification_time(sf),
sha1, tth, TRUE);
add_persistent_cache_entry(shared_file_path(sf),
shared_file_size(sf), shared_file_modification_time(sf),
sha1, tth);
}
return TRUE;
}
/**
* Put the shared file on the stack of the things to do.
*
* We first begin with the computation of the SHA1, and when completed we
* will continue with the TTH computation.
*/
static void
queue_shared_file_for_sha1_computation(shared_file_t *sf)
{
int inserted;
shared_file_check(sf);
inserted = verify_sha1_enqueue(FALSE, shared_file_path(sf),
shared_file_size(sf), huge_verify_callback,
shared_file_ref(sf));
if (!inserted)
shared_file_unref(&sf);
}
void
request_tigertree(shared_file_t *sf, bool high_priority)
{
int inserted;
verify_tth_init();
g_return_if_fail(sf);
shared_file_check(sf);
g_return_if_fail(!shared_file_is_partial(sf));
sf = shared_file_ref(sf);
inserted = verify_enqueue(verify_tth.verify, high_priority,
shared_file_path(sf), 0, shared_file_size(sf),
request_tigertree_callback, sf);
if (!inserted)
shared_file_unref(&sf);
}
static void
upload_stats_dump_item(gpointer p, gpointer user_data)
{
const shared_file_t *sf;
FILE *out = user_data;
struct ul_stats *s = p;
char rtime_buf[TIME_T_DEC_BUFLEN];
char dtime_buf[TIME_T_DEC_BUFLEN];
const char *pathname;
char *escaped;
g_assert(NULL != s);
sf = s->sha1 ? shared_file_by_sha1(s->sha1) : NULL;
sf = SHARE_REBUILDING != sf ? sf : NULL;
if (sf) {
pathname = shared_file_path(sf);
} else {
pathname = s->pathname;
}
escaped = url_escape_cntrl(pathname);
time_t_to_string_buf(s->rtime, rtime_buf, sizeof rtime_buf);
time_t_to_string_buf(s->dtime, dtime_buf, sizeof dtime_buf);
fprintf(out, "%s\t%s\t%u\t%u\t%lu\t%lu\t%s\t%s\t%s\n",
escaped,
uint64_to_string(s->size),
s->attempts,
s->complete,
(unsigned long) (s->bytes_sent >> 32),
(unsigned long) (s->bytes_sent & 0xffffffff),
rtime_buf,
dtime_buf,
s->sha1 ? sha1_base32(s->sha1) : "*");
if (escaped != pathname) { /* File had escaped chars */
HFREE_NULL(escaped);
}
}
static bool
request_tigertree_callback(const struct verify *ctx, enum verify_status status,
void *user_data)
{
shared_file_t *sf = user_data;
shared_file_check(sf);
switch (status) {
case VERIFY_START:
if (!shared_file_is_servable(sf)) {
/*
* After a rescan, there might be files in the queue which are
* no longer shared.
*/
if (GNET_PROPERTY(verify_debug) > 1) {
g_debug("skipping TTH computation for %s: not a servable file",
shared_file_path(sf));
}
return FALSE;
}
if (shared_file_tth_is_available(sf)) {
if (
GNET_PROPERTY(tigertree_debug) > 1 ||
GNET_PROPERTY(verify_debug) > 1
) {
g_debug("TTH for %s is already cached (%s)",
shared_file_path(sf), tth_base32(shared_file_tth(sf)));
}
return FALSE;
}
gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, TRUE);
return TRUE;
case VERIFY_PROGRESS:
/*
* Processing can continue whilst the library file is indexed or the
* completed file is still beeing seeded.
*/
return shared_file_is_servable(sf);
case VERIFY_DONE:
{
const struct tth *tth = verify_tth_digest(ctx);
size_t n_leaves = verify_tth_leave_count(ctx);
if (GNET_PROPERTY(verify_debug)) {
g_debug("%s(): computed TTH %s (%zu lea%s) for %s",
G_STRFUNC, tth_base32(tth),
n_leaves, plural_f(n_leaves),
shared_file_path(sf));
}
/*
* Write the TTH to the cache first, before updating the hashes.
* That way, the logic behind huge_update_hashes() can rely on
* the fact that the TTH is persisted already.
*
* This is important for seeded files for which we re-compute
* the TTH once they are completed (to make sure we can serve
* THEX requests at the proper good depth). In order to update
* the GUI information, we'll need to probe the cache to determine
* how large the TTH is exactly, since all we pass back to the
* routines is the TTH root hash.
* --RAM, 2017-10-20
*/
tth_cache_insert(tth, verify_tth_leaves(ctx), n_leaves);
huge_update_hashes(sf, shared_file_sha1(sf), tth);
}
goto done;
case VERIFY_ERROR:
if (GNET_PROPERTY(verify_debug)) {
g_debug("%s(): unable to compute TTH for %s",
G_STRFUNC, shared_file_path(sf));
}
/* FALL THROUGH */
case VERIFY_SHUTDOWN:
goto done;
case VERIFY_INVALID:
break;
}
g_assert_not_reached();
return FALSE;
done:
shared_file_unref(&sf);
gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, FALSE);
return TRUE;
}
