bool huge_update_hashes(shared_file_t *sf, const struct sha1 *sha1, const struct tth *tth) { struct sha1_cache_entry *cached; filestat_t sb; shared_file_check(sf); g_return_val_if_fail(sha1, FALSE); /* * Make sure the file's timestamp is still accurate. */ if (-1 == stat(shared_file_path(sf), &sb)) { g_warning("discarding SHA1 for file \"%s\": can't stat(): %m", shared_file_path(sf)); shared_file_remove(sf); return TRUE; } if (sb.st_mtime != shared_file_modification_time(sf)) { g_warning("file \"%s\" was modified whilst SHA1 was computed", shared_file_path(sf)); shared_file_set_modification_time(sf, sb.st_mtime); request_sha1(sf); /* Retry! */ return TRUE; } if (huge_spam_check(sf, sha1)) { shared_file_remove(sf); return FALSE; } shared_file_set_sha1(sf, sha1); shared_file_set_tth(sf, tth); /* Update cache */ cached = hikset_lookup(sha1_cache, shared_file_path(sf)); if (cached) { update_volatile_cache(cached, shared_file_size(sf), shared_file_modification_time(sf), sha1, tth); cache_dirty = TRUE; /* Dump the cache at most about once per minute. */ if (!cache_dumped || delta_time(tm_time(), cache_dumped) > 60) { dump_cache(FALSE); } } else { add_volatile_cache_entry(shared_file_path(sf), shared_file_size(sf), shared_file_modification_time(sf), sha1, tth, TRUE); add_persistent_cache_entry(shared_file_path(sf), shared_file_size(sf), shared_file_modification_time(sf), sha1, tth); } return TRUE; }
/** * External interface to call for getting the hash for a shared_file. */ void request_sha1(shared_file_t *sf) { struct sha1_cache_entry *cached; shared_file_check(sf); if (!shared_file_indexed(sf)) return; /* "stale" shared file, has been superseded or removed */ cached = hikset_lookup(sha1_cache, shared_file_path(sf)); if (cached && cached_entry_up_to_date(cached, sf)) { cache_dirty = TRUE; cached->shared = TRUE; shared_file_set_sha1(sf, cached->sha1); shared_file_set_tth(sf, cached->tth); request_tigertree(sf, NULL == cached->tth); } else { if (GNET_PROPERTY(share_debug) > 1) { if (cached) g_debug("cached SHA1 entry for \"%s\" outdated: " "had mtime %lu, now %lu", shared_file_path(sf), (ulong) cached->mtime, (ulong) shared_file_modification_time(sf)); else g_debug("queuing \"%s\" for SHA1 computation", shared_file_path(sf)); } queue_shared_file_for_sha1_computation(sf); } }
static bool huge_verify_callback(const struct verify *ctx, enum verify_status status, void *user_data) { shared_file_t *sf = user_data; shared_file_check(sf); switch (status) { case VERIFY_START: if (!huge_need_sha1(sf)) return FALSE; gnet_prop_set_boolean_val(PROP_SHA1_REBUILDING, TRUE); return TRUE; case VERIFY_PROGRESS: return 0 != (SHARE_F_INDEXED & shared_file_flags(sf)); case VERIFY_DONE: huge_update_hashes(sf, verify_sha1_digest(ctx), NULL); request_tigertree(sf, TRUE); /* FALL THROUGH */ case VERIFY_ERROR: case VERIFY_SHUTDOWN: gnet_prop_set_boolean_val(PROP_SHA1_REBUILDING, FALSE); shared_file_unref(&sf); return TRUE; case VERIFY_INVALID: break; } g_assert_not_reached(); return FALSE; }
void request_tigertree(shared_file_t *sf, bool high_priority) { int inserted; verify_tth_init(); shared_file_check(sf); g_return_if_fail(shared_file_is_finished(sf)); if (!shared_file_is_servable(sf)) return; /* "stale" shared file, has been superseded or removed */ /* * This routine can be called when the VERIFY_DONE event is received by * huge_verify_callback(). We may have already shutdown the TTH * verification thread. */ if G_UNLIKELY(NULL == verify_tth.verify) return; sf = shared_file_ref(sf); inserted = verify_enqueue(verify_tth.verify, high_priority, shared_file_path(sf), 0, shared_file_size(sf), request_tigertree_callback, sf); if (!inserted) shared_file_unref(&sf); }
static bool request_tigertree_callback(const struct verify *ctx, enum verify_status status, void *user_data) { shared_file_t *sf = user_data; shared_file_check(sf); switch (status) { case VERIFY_START: if (!(SHARE_F_INDEXED & shared_file_flags(sf))) { /* * After a rescan, there might be files in the queue which are * no longer shared. */ if (GNET_PROPERTY(verify_debug) > 1) { g_debug("skipping TTH computation for %s: no longer shared", shared_file_path(sf)); } return FALSE; } if ( shared_file_tth(sf) && tth_cache_lookup(shared_file_tth(sf), shared_file_size(sf)) > 0 ) { if ( GNET_PROPERTY(tigertree_debug) > 1 || GNET_PROPERTY(verify_debug) > 1 ) { g_debug("TTH for %s is already cached (%s)", shared_file_path(sf), tth_base32(shared_file_tth(sf))); } return FALSE; } gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, TRUE); return TRUE; case VERIFY_PROGRESS: return 0 != (SHARE_F_INDEXED & shared_file_flags(sf)); case VERIFY_DONE: { const struct tth *tth = verify_tth_digest(ctx); huge_update_hashes(sf, shared_file_sha1(sf), tth); tth_cache_insert(tth, verify_tth_leaves(ctx), verify_tth_leave_count(ctx)); } /* FALL THROUGH */ case VERIFY_ERROR: case VERIFY_SHUTDOWN: shared_file_unref(&sf); gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, FALSE); return TRUE; case VERIFY_INVALID: break; } g_assert_not_reached(); return FALSE; }
/** * Put the shared file on the stack of the things to do. * * We first begin with the computation of the SHA1, and when completed we * will continue with the TTH computation. */ static void queue_shared_file_for_sha1_computation(shared_file_t *sf) { int inserted; shared_file_check(sf); inserted = verify_sha1_enqueue(FALSE, shared_file_path(sf), shared_file_size(sf), huge_verify_callback, shared_file_ref(sf)); if (!inserted) shared_file_unref(&sf); }
void request_tigertree(shared_file_t *sf, bool high_priority) { int inserted; verify_tth_init(); g_return_if_fail(sf); shared_file_check(sf); g_return_if_fail(!shared_file_is_partial(sf)); sf = shared_file_ref(sf); inserted = verify_enqueue(verify_tth.verify, high_priority, shared_file_path(sf), 0, shared_file_size(sf), request_tigertree_callback, sf); if (!inserted) shared_file_unref(&sf); }
/** * Look whether we still need to compute the SHA1 of the given shared file * by looking into our in-core cache to see whether the entry we have is * up-to-date. * * @param sf the shared file for which we want to compute the SHA1 * * @return TRUE if the file need SHA1 recomputation. */ static bool huge_need_sha1(shared_file_t *sf) { struct sha1_cache_entry *cached; shared_file_check(sf); /* * After a rescan, there might be files in the queue which are * no longer shared. */ if (!shared_file_indexed(sf)) return FALSE; if G_UNLIKELY(NULL == sha1_cache) return FALSE; /* Shutdown occurred (processing TEQ event?) */ cached = hikset_lookup(sha1_cache, shared_file_path(sf)); if (cached != NULL) { filestat_t sb; if (-1 == stat(shared_file_path(sf), &sb)) { g_warning("ignoring SHA1 recomputation request for \"%s\": %m", shared_file_path(sf)); return FALSE; } if ( cached->size + (fileoffset_t) 0 == sb.st_size + (filesize_t) 0 && cached->mtime == sb.st_mtime ) { if (GNET_PROPERTY(share_debug) > 1) { g_warning("ignoring duplicate SHA1 work for \"%s\"", shared_file_path(sf)); } return FALSE; } } return TRUE; }
/** * Add file to the current query hit. * * @return TRUE if we kept the file, FALSE if we did not include it in the hit. */ static bool g2_build_qh2_add(struct g2_qh2_builder *ctx, const shared_file_t *sf) { const sha1_t *sha1; g2_tree_t *h, *c; shared_file_check(sf); /* * Make sure the file is still in the library. */ if (0 == shared_file_index(sf)) return FALSE; /* * On G2, the H/URN child is required, meaning we need the SHA1 at least. */ if (!sha1_hash_available(sf)) return FALSE; /* * Do not send duplicates, as determined by the SHA1 of the resource. * * A user may share several files with different names but the same SHA1, * and if all of them are hits, we only want to send one instance. * * When generating hits for host-browsing, we do not care about duplicates * and ctx->hs is NULL then. */ sha1 = shared_file_sha1(sf); /* This is an atom */ if (ctx->hs != NULL) { if (hset_contains(ctx->hs, sha1)) return FALSE; hset_insert(ctx->hs, sha1); } /* * Create the "H" child and attach it to the current tree. */ if (NULL == ctx->t) g2_build_qh2_start(ctx); h = g2_tree_alloc_empty("H"); g2_tree_add_child(ctx->t, h); /* * URN -- Universal Resource Name * * If there is a known TTH, then we can generate a bitprint, otherwise * we just convey the SHA1. */ { const tth_t * const tth = shared_file_tth(sf); char payload[SHA1_RAW_SIZE + TTH_RAW_SIZE + sizeof G2_URN_BITPRINT]; char *p = payload; if (NULL == tth) { p = mempcpy(p, G2_URN_SHA1, sizeof G2_URN_SHA1); p += clamp_memcpy(p, sizeof payload - ptr_diff(p, payload), sha1, SHA1_RAW_SIZE); } else { p = mempcpy(p, G2_URN_BITPRINT, sizeof G2_URN_BITPRINT); p += clamp_memcpy(p, sizeof payload - ptr_diff(p, payload), sha1, SHA1_RAW_SIZE); p += clamp_memcpy(p, sizeof payload - ptr_diff(p, payload), tth, TTH_RAW_SIZE); } g_assert(ptr_diff(p, payload) <= sizeof payload); c = g2_tree_alloc_copy("URN", payload, ptr_diff(p, payload)); g2_tree_add_child(h, c); } /* * URL -- empty to indicate that we share the file via uri-res. */ if (ctx->flags & QHIT_F_G2_URL) { uint known; uint16 csc; c = g2_tree_alloc_empty("URL"); g2_tree_add_child(h, c); /* * CSC -- if we know alternate sources, indicate how many in "CSC". * * This child is only emitted when they requested "URL". */ known = dmesh_count(sha1); csc = MIN(known, MAX_INT_VAL(uint16)); if (csc != 0) { char payload[2]; poke_le16(payload, csc); c = g2_tree_alloc_copy("CSC", payload, sizeof payload); g2_tree_add_child(h, c); } /* * PART -- if we only have a partial file, indicate how much we have. * * This child is only emitted when they requested "URL". */ if (shared_file_is_partial(sf) && !shared_file_is_finished(sf)) { filesize_t available = shared_file_available(sf); char payload[8]; /* If we have to encode file size as 64-bit */ uint32 av32; time_t mtime = shared_file_modification_time(sf); c = g2_tree_alloc_empty("PART"); g2_tree_add_child(h, c); av32 = available; if (av32 == available) { /* Fits within a 32-bit quantity */ poke_le32(payload, av32); g2_tree_set_payload(c, payload, sizeof av32, TRUE); } else { /* Encode as a 64-bit quantity then */ poke_le64(payload, available); g2_tree_set_payload(c, payload, sizeof payload, TRUE); } /* * GTKG extension: encode the last modification time of the * partial file in an "MT" child. This lets the other party * determine whether the host is still able to actively complete * the file. */ poke_le32(payload, (uint32) mtime); g2_tree_add_child(c, g2_tree_alloc_copy("MT", payload, sizeof(uint32))); } /* * CT -- creation time of the resource (GTKG extension). */ { time_t create_time = shared_file_creation_time(sf); if ((time_t) -1 != create_time) { char payload[8]; int n; create_time = MAX(0, create_time); n = vlint_encode(create_time, payload); g2_tree_add_child(h, g2_tree_alloc_copy("CT", payload, n)); /* No trailing 0s */ } } } /* * DN -- distinguished name. * * Note that the presence of DN also governs the presence of SZ if the * file length does not fit a 32-bit unsigned quantity. */ if (ctx->flags & QHIT_F_G2_DN) { char payload[8]; /* If we have to encode file size as 64-bit */ uint32 fs32; filesize_t fs = shared_file_size(sf); const char *name; const char *rp; c = g2_tree_alloc_empty("DN"); fs32 = fs; if (fs32 == fs) { /* Fits within a 32-bit quantity */ poke_le32(payload, fs32); g2_tree_set_payload(c, payload, sizeof fs32, TRUE); } else { /* Does not fit a 32-bit quantity, emit a SZ child */ poke_le64(payload, fs); g2_tree_add_child(h, g2_tree_alloc_copy("SZ", payload, sizeof payload)); } name = shared_file_name_nfc(sf); g2_tree_append_payload(c, name, shared_file_name_nfc_len(sf)); g2_tree_add_child(h, c); /* * GTKG extension: if there is a file path, expose it as a "P" child * under the DN node. */ rp = shared_file_relative_path(sf); if (rp != NULL) { g2_tree_add_child(c, g2_tree_alloc_copy("P", rp, strlen(rp))); } } /* * GTKG extension: if they requested alt-locs in the /Q2/I with "A", then * send them some known alt-locs in an "ALT" child. * * Note that these alt-locs can be for Gnutella hosts: since both Gnutella * and G2 share a common HTTP-based file transfer mechanism with compatible * extra headers, there is no need to handle them separately. */ if (ctx->flags & QHIT_F_G2_ALT) { gnet_host_t hvec[G2_BUILD_QH2_MAX_ALT]; int hcnt = 0; hcnt = dmesh_fill_alternate(sha1, hvec, N_ITEMS(hvec)); if (hcnt > 0) { int i; c = g2_tree_alloc_empty("ALT"); for (i = 0; i < hcnt; i++) { host_addr_t addr; uint16 port; addr = gnet_host_get_addr(&hvec[i]); port = gnet_host_get_port(&hvec[i]); if (host_addr_is_ipv4(addr)) { char payload[6]; host_ip_port_poke(payload, addr, port, NULL); g2_tree_append_payload(c, payload, sizeof payload); } } /* * If the payload is still empty, then drop the "ALT" child. * Otherwise, attach it to the "H" node. */ if (NULL == g2_tree_node_payload(c, NULL)) { g2_tree_free_null(&c); } else { g2_tree_add_child(h, c); } } } /* * Update the size of the query hit we're generating. */ ctx->current_size += g2_frame_serialize(h, NULL, 0); return TRUE; }
static bool request_tigertree_callback(const struct verify *ctx, enum verify_status status, void *user_data) { shared_file_t *sf = user_data; shared_file_check(sf); switch (status) { case VERIFY_START: if (!shared_file_is_servable(sf)) { /* * After a rescan, there might be files in the queue which are * no longer shared. */ if (GNET_PROPERTY(verify_debug) > 1) { g_debug("skipping TTH computation for %s: not a servable file", shared_file_path(sf)); } return FALSE; } if (shared_file_tth_is_available(sf)) { if ( GNET_PROPERTY(tigertree_debug) > 1 || GNET_PROPERTY(verify_debug) > 1 ) { g_debug("TTH for %s is already cached (%s)", shared_file_path(sf), tth_base32(shared_file_tth(sf))); } return FALSE; } gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, TRUE); return TRUE; case VERIFY_PROGRESS: /* * Processing can continue whilst the library file is indexed or the * completed file is still beeing seeded. */ return shared_file_is_servable(sf); case VERIFY_DONE: { const struct tth *tth = verify_tth_digest(ctx); size_t n_leaves = verify_tth_leave_count(ctx); if (GNET_PROPERTY(verify_debug)) { g_debug("%s(): computed TTH %s (%zu lea%s) for %s", G_STRFUNC, tth_base32(tth), n_leaves, plural_f(n_leaves), shared_file_path(sf)); } /* * Write the TTH to the cache first, before updating the hashes. * That way, the logic behind huge_update_hashes() can rely on * the fact that the TTH is persisted already. * * This is important for seeded files for which we re-compute * the TTH once they are completed (to make sure we can serve * THEX requests at the proper good depth). In order to update * the GUI information, we'll need to probe the cache to determine * how large the TTH is exactly, since all we pass back to the * routines is the TTH root hash. * --RAM, 2017-10-20 */ tth_cache_insert(tth, verify_tth_leaves(ctx), n_leaves); huge_update_hashes(sf, shared_file_sha1(sf), tth); } goto done; case VERIFY_ERROR: if (GNET_PROPERTY(verify_debug)) { g_debug("%s(): unable to compute TTH for %s", G_STRFUNC, shared_file_path(sf)); } /* FALL THROUGH */ case VERIFY_SHUTDOWN: goto done; case VERIFY_INVALID: break; } g_assert_not_reached(); return FALSE; done: shared_file_unref(&sf); gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, FALSE); return TRUE; }