/** * Add `comp' to the current completed count, and update the amount of * bytes transferred. Note that `comp' can be zero. * When `update_dtime' is TRUE, we update the "done time", otherwise we * change the "last request time". * * If the row does not exist (race condition: deleted since upload started), * recreate one. */ static void upload_stats_file_add( const shared_file_t *sf, int comp, guint64 sent, gboolean update_dtime) { const char *pathname = shared_file_path(sf); filesize_t size = shared_file_size(sf); struct ul_stats *s; const struct sha1 *sha1; g_assert(comp >= 0); sha1 = sha1_hash_available(sf) ? shared_file_sha1(sf) : NULL; /* find this file in the ul_stats_clist */ s = upload_stats_find(sha1, pathname, size); /* increment the completed counter */ if (NULL == s) { /* uh oh, row has since been deleted, add it: 1 attempt */ upload_stats_add(pathname, size, shared_file_name_nfc(sf), 1, comp, sent, tm_time(), tm_time(), sha1); } else { s->bytes_sent += sent; s->norm = 1.0 * s->bytes_sent / s->size; s->complete += comp; if (update_dtime) s->dtime = tm_time(); else s->rtime = tm_time(); gcu_upload_stats_gui_update(s); } dirty = TRUE; /* Request asynchronous save of stats */ }
/** * Called when an upload starts. */ void upload_stats_file_begin(const shared_file_t *sf) { struct ul_stats *s; const char *pathname; filesize_t size; const struct sha1 *sha1; g_return_if_fail(sf); pathname = shared_file_path(sf); size = shared_file_size(sf); sha1 = sha1_hash_available(sf) ? shared_file_sha1(sf) : NULL; /* find this file in the ul_stats_clist */ s = upload_stats_find(sha1, pathname, size); /* increment the attempted counter */ if (NULL == s) { upload_stats_add(pathname, size, shared_file_name_nfc(sf), 1, 0, 0, tm_time(), 0, sha1); } else { s->attempts++; s->rtime = tm_time(); gcu_upload_stats_gui_update(s); } dirty = TRUE; /* Request asynchronous save of stats */ }
/** * Make sure the filename associated to a SHA1 is given the name of * the shared file and no longer bears the name of the partial file. * This can happen when the partial file is seeded then the file is * renamed and shared. */ void upload_stats_enforce_local_filename(const shared_file_t *sf) { struct ul_stats *s; const struct sha1 *sha1; const char *name; if (!upload_stats_by_sha1) return; /* Nothing known by SHA1 yet */ sha1 = sha1_hash_available(sf) ? shared_file_sha1(sf) : NULL; if (!sha1) return; /* File's SHA1 not known yet, nothing to do here */ s = g_hash_table_lookup(upload_stats_by_sha1, sha1); if (NULL == s) return; /* SHA1 not in stats, nothing to do */ name = shared_file_name_nfc(sf); if (name == s->filename) /* Both are string atoms */ return; /* Everything is fine */ /* * We need to update the filename to match the shared file. */ hash_list_remove(upload_stats_list, s); atom_str_change(&s->pathname, shared_file_path(sf)); atom_str_change(&s->filename, name); hash_list_append(upload_stats_list, s); gcu_upload_stats_gui_update_name(s); }
/** * Writes the browse host data of the context ``ctx'' to the buffer * ``dest''. This must be called multiple times to retrieve the complete * data until zero is returned i.e., the end of file is reached. * * This routine deals with HTML data generation. * * @param ctx an initialized browse host context. * @param dest the destination buffer. * @param size the amount of bytes ``dest'' can hold. * * @return -1 on failure, zero at the end-of-file condition or if size * was zero. On success, the amount of bytes copied to ``dest'' * is returned. */ static ssize_t browse_host_read_html(struct special_upload *ctx, void *const dest, size_t size) { static const char header[] = "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\r\n" "<html>\r\n" "<head>\r\n" "<title>Browse Host</title>\r\n" "</head>\r\n" "<body>\r\n"; static const char trailer[] = "</ul>\r\n</body>\r\n</html>\r\n"; struct browse_host_upload *bh = cast_to_browse_host_upload(ctx); char *p = dest; g_assert(NULL != bh); g_assert(NULL != dest); g_assert(size <= INT_MAX); g_assert(UNSIGNED(bh->state) < NUM_BH_STATES); g_assert(bh->b_size <= INT_MAX); g_assert(bh->b_offset <= bh->b_size); do { switch (bh->state) { case BH_STATE_HEADER: if (!bh->b_data) { bh->b_data = header; bh->b_size = CONST_STRLEN(header); } p += browse_host_read_data(bh, p, &size); if (bh->b_size == bh->b_offset) browse_host_next_state(bh, BH_STATE_LIBRARY_INFO); break; case BH_STATE_LIBRARY_INFO: if (!bh->b_data) { bh->w_buf_size = w_concat_strings(&bh->w_buf, "<h1>", product_get_name(), "</h1>\r\n" "<h3>", version_get_string(), " sharing ", uint64_to_string(shared_files_scanned()), " file", shared_files_scanned() == 1 ? "" : "s", " ", short_kb_size(shared_kbytes_scanned(), GNET_PROPERTY(display_metric_units)), " total</h3>\r\n" "<ul>\r\n", (void *) 0); bh->b_data = bh->w_buf; bh->b_size = bh->w_buf_size - 1; /* minus trailing NUL */ bh->b_offset = 0; } p += browse_host_read_data(bh, p, &size); if (bh->b_size == bh->b_offset) browse_host_next_state(bh, BH_STATE_FILES); break; case BH_STATE_TRAILER: if (!bh->b_data) { bh->b_data = trailer; bh->b_size = CONST_STRLEN(trailer); } p += browse_host_read_data(bh, p, &size); if (bh->b_size == bh->b_offset) browse_host_next_state(bh, BH_STATE_EOF); break; case BH_STATE_FILES: if (bh->b_data && bh->b_size == bh->b_offset) { g_assert(bh->w_buf == bh->b_data); wfree(bh->w_buf, bh->w_buf_size); bh->w_buf = NULL; bh->w_buf_size = 0; bh->b_data = NULL; } if (!bh->b_data) { const shared_file_t *sf; bh->file_index++; sf = shared_file_sorted(bh->file_index); if (!sf) { if (bh->file_index > shared_files_scanned()) browse_host_next_state(bh, BH_STATE_TRAILER); /* Skip holes in the file_index table */ } else if (SHARE_REBUILDING == sf) { browse_host_next_state(bh, BH_STATE_REBUILDING); } else { const char * const name_nfc = shared_file_name_nfc(sf); const filesize_t file_size = shared_file_size(sf); size_t html_size; char *html_name; { const char *dir; char *name; dir = shared_file_relative_path(sf); if (dir) { name = h_strconcat(dir, "/", name_nfc, (void *) 0); } else { name = deconstify_char(name_nfc); } html_size = 1 + html_escape(name, NULL, 0); html_name = walloc(html_size); html_escape(name, html_name, html_size); if (name != name_nfc) { HFREE_NULL(name); } } if (sha1_hash_available(sf)) { const struct sha1 *sha1 = shared_file_sha1(sf); bh->w_buf_size = w_concat_strings(&bh->w_buf, "<li><a href=\"/uri-res/N2R?urn:sha1:", sha1_base32(sha1), "\">", html_name, "</a> [", short_html_size(file_size, GNET_PROPERTY(display_metric_units)), "]</li>\r\n", (void *) 0); } else { char *escaped; escaped = url_escape(name_nfc); bh->w_buf_size = w_concat_strings(&bh->w_buf, "<li><a href=\"/get/", uint32_to_string(shared_file_index(sf)), "/", escaped, "\">", html_name, "</a>" " [", short_html_size(file_size, GNET_PROPERTY(display_metric_units)), "]</li>\r\n", (void *) 0); if (escaped != name_nfc) { HFREE_NULL(escaped); } } wfree(html_name, html_size); bh->b_data = bh->w_buf; bh->b_size = bh->w_buf_size - 1; /* minus trailing NUL */ bh->b_offset = 0; } } if (bh->b_data) p += browse_host_read_data(bh, p, &size); break; case BH_STATE_REBUILDING: if (!bh->b_data) { static const char msg[] = "<li>" "<b>" "The library is currently being rebuild. Please, " "try again in a moment." "</b>" "</li>"; bh->b_data = msg; bh->b_size = CONST_STRLEN(msg); } p += browse_host_read_data(bh, p, &size); if (bh->b_size == bh->b_offset) browse_host_next_state(bh, BH_STATE_TRAILER); break; case BH_STATE_EOF: return p - cast_to_char_ptr(dest); case NUM_BH_STATES: g_assert_not_reached(); } } while (size > 0); return p - cast_to_char_ptr(dest); }
/** * Add file to the current query hit. * * @return TRUE if we kept the file, FALSE if we did not include it in the hit. */ static bool g2_build_qh2_add(struct g2_qh2_builder *ctx, const shared_file_t *sf) { const sha1_t *sha1; g2_tree_t *h, *c; shared_file_check(sf); /* * Make sure the file is still in the library. */ if (0 == shared_file_index(sf)) return FALSE; /* * On G2, the H/URN child is required, meaning we need the SHA1 at least. */ if (!sha1_hash_available(sf)) return FALSE; /* * Do not send duplicates, as determined by the SHA1 of the resource. * * A user may share several files with different names but the same SHA1, * and if all of them are hits, we only want to send one instance. * * When generating hits for host-browsing, we do not care about duplicates * and ctx->hs is NULL then. */ sha1 = shared_file_sha1(sf); /* This is an atom */ if (ctx->hs != NULL) { if (hset_contains(ctx->hs, sha1)) return FALSE; hset_insert(ctx->hs, sha1); } /* * Create the "H" child and attach it to the current tree. */ if (NULL == ctx->t) g2_build_qh2_start(ctx); h = g2_tree_alloc_empty("H"); g2_tree_add_child(ctx->t, h); /* * URN -- Universal Resource Name * * If there is a known TTH, then we can generate a bitprint, otherwise * we just convey the SHA1. */ { const tth_t * const tth = shared_file_tth(sf); char payload[SHA1_RAW_SIZE + TTH_RAW_SIZE + sizeof G2_URN_BITPRINT]; char *p = payload; if (NULL == tth) { p = mempcpy(p, G2_URN_SHA1, sizeof G2_URN_SHA1); p += clamp_memcpy(p, sizeof payload - ptr_diff(p, payload), sha1, SHA1_RAW_SIZE); } else { p = mempcpy(p, G2_URN_BITPRINT, sizeof G2_URN_BITPRINT); p += clamp_memcpy(p, sizeof payload - ptr_diff(p, payload), sha1, SHA1_RAW_SIZE); p += clamp_memcpy(p, sizeof payload - ptr_diff(p, payload), tth, TTH_RAW_SIZE); } g_assert(ptr_diff(p, payload) <= sizeof payload); c = g2_tree_alloc_copy("URN", payload, ptr_diff(p, payload)); g2_tree_add_child(h, c); } /* * URL -- empty to indicate that we share the file via uri-res. */ if (ctx->flags & QHIT_F_G2_URL) { uint known; uint16 csc; c = g2_tree_alloc_empty("URL"); g2_tree_add_child(h, c); /* * CSC -- if we know alternate sources, indicate how many in "CSC". * * This child is only emitted when they requested "URL". */ known = dmesh_count(sha1); csc = MIN(known, MAX_INT_VAL(uint16)); if (csc != 0) { char payload[2]; poke_le16(payload, csc); c = g2_tree_alloc_copy("CSC", payload, sizeof payload); g2_tree_add_child(h, c); } /* * PART -- if we only have a partial file, indicate how much we have. * * This child is only emitted when they requested "URL". */ if (shared_file_is_partial(sf) && !shared_file_is_finished(sf)) { filesize_t available = shared_file_available(sf); char payload[8]; /* If we have to encode file size as 64-bit */ uint32 av32; time_t mtime = shared_file_modification_time(sf); c = g2_tree_alloc_empty("PART"); g2_tree_add_child(h, c); av32 = available; if (av32 == available) { /* Fits within a 32-bit quantity */ poke_le32(payload, av32); g2_tree_set_payload(c, payload, sizeof av32, TRUE); } else { /* Encode as a 64-bit quantity then */ poke_le64(payload, available); g2_tree_set_payload(c, payload, sizeof payload, TRUE); } /* * GTKG extension: encode the last modification time of the * partial file in an "MT" child. This lets the other party * determine whether the host is still able to actively complete * the file. */ poke_le32(payload, (uint32) mtime); g2_tree_add_child(c, g2_tree_alloc_copy("MT", payload, sizeof(uint32))); } /* * CT -- creation time of the resource (GTKG extension). */ { time_t create_time = shared_file_creation_time(sf); if ((time_t) -1 != create_time) { char payload[8]; int n; create_time = MAX(0, create_time); n = vlint_encode(create_time, payload); g2_tree_add_child(h, g2_tree_alloc_copy("CT", payload, n)); /* No trailing 0s */ } } } /* * DN -- distinguished name. * * Note that the presence of DN also governs the presence of SZ if the * file length does not fit a 32-bit unsigned quantity. */ if (ctx->flags & QHIT_F_G2_DN) { char payload[8]; /* If we have to encode file size as 64-bit */ uint32 fs32; filesize_t fs = shared_file_size(sf); const char *name; const char *rp; c = g2_tree_alloc_empty("DN"); fs32 = fs; if (fs32 == fs) { /* Fits within a 32-bit quantity */ poke_le32(payload, fs32); g2_tree_set_payload(c, payload, sizeof fs32, TRUE); } else { /* Does not fit a 32-bit quantity, emit a SZ child */ poke_le64(payload, fs); g2_tree_add_child(h, g2_tree_alloc_copy("SZ", payload, sizeof payload)); } name = shared_file_name_nfc(sf); g2_tree_append_payload(c, name, shared_file_name_nfc_len(sf)); g2_tree_add_child(h, c); /* * GTKG extension: if there is a file path, expose it as a "P" child * under the DN node. */ rp = shared_file_relative_path(sf); if (rp != NULL) { g2_tree_add_child(c, g2_tree_alloc_copy("P", rp, strlen(rp))); } } /* * GTKG extension: if they requested alt-locs in the /Q2/I with "A", then * send them some known alt-locs in an "ALT" child. * * Note that these alt-locs can be for Gnutella hosts: since both Gnutella * and G2 share a common HTTP-based file transfer mechanism with compatible * extra headers, there is no need to handle them separately. */ if (ctx->flags & QHIT_F_G2_ALT) { gnet_host_t hvec[G2_BUILD_QH2_MAX_ALT]; int hcnt = 0; hcnt = dmesh_fill_alternate(sha1, hvec, N_ITEMS(hvec)); if (hcnt > 0) { int i; c = g2_tree_alloc_empty("ALT"); for (i = 0; i < hcnt; i++) { host_addr_t addr; uint16 port; addr = gnet_host_get_addr(&hvec[i]); port = gnet_host_get_port(&hvec[i]); if (host_addr_is_ipv4(addr)) { char payload[6]; host_ip_port_poke(payload, addr, port, NULL); g2_tree_append_payload(c, payload, sizeof payload); } } /* * If the payload is still empty, then drop the "ALT" child. * Otherwise, attach it to the "H" node. */ if (NULL == g2_tree_node_payload(c, NULL)) { g2_tree_free_null(&c); } else { g2_tree_add_child(h, c); } } } /* * Update the size of the query hit we're generating. */ ctx->current_size += g2_frame_serialize(h, NULL, 0); return TRUE; }
/** * Handle a SHA-1 entry, publishing its alt-loc to the DHT if still shared. */ static void publisher_handle(struct publisher_entry *pe) { shared_file_t *sf; bool is_partial = FALSE; int alt_locs; time_delta_t min_uptime; uint32 avg_uptime; publisher_check(pe); g_assert(NULL == pe->publish_ev); sf = shared_file_by_sha1(pe->sha1); /* * Remove SHA1 if no longer shared. */ if (NULL == sf) { fileinfo_t *fi = file_info_by_sha1(pe->sha1); /* * If a partial file has lees than the minimum amount of data for PFSP, * shared_file_by_sha1() will return NULL, hence we need to explicitly * check for existence through file_info_by_sha1() and that the file * still exists. */ if (fi != NULL && file_exists(fi->pathname)) { /* Waiting for more data to be able to share, or PFSP re-enabled */ publisher_retry(pe, PUBLISH_BUSY, "partial file missing"); return; } if (GNET_PROPERTY(publisher_debug)) { g_debug("PUBLISHER SHA-1 %s is no longer shared", sha1_to_string(pe->sha1)); } publisher_entry_free(pe, TRUE); return; } /* * Wait when rebuilding the library. */ if (SHARE_REBUILDING == sf) { publisher_retry(pe, PUBLISH_BUSY, "library being rebuilt"); return; } is_partial = shared_file_is_partial(sf); /* * If the SHA1 is not available, wait. */ if ( !is_partial && (!sha1_hash_available(sf) || !sha1_hash_is_uptodate(sf)) ) { publisher_retry(pe, PUBLISH_BUSY, "SHA-1 of file unknown yet"); goto done; } /* * Look whether this node has a sufficient average uptime. * * We're stricter to publish partial files because we want to favor * publishing of full files in the DHT, and the benefits of publishing * partial entries come only if we're up for a long enough time. * * Since publishing imposes lookup traffic in the DHT, it is not efficient * to have transient nodes publish file sharing information because this * will likely never be useful. */ min_uptime = PUBLISH_TRANSIENT; if (is_partial) min_uptime *= 2; avg_uptime = get_average_servent_uptime(tm_time()); if (avg_uptime < UNSIGNED(min_uptime)) { time_delta_t delay = min_uptime - avg_uptime; delay = MAX(delay, PUBLISH_BUSY); publisher_retry(pe, delay, "minimum average uptime not reached yet"); goto done; } /* * If we are dealing with a file for which we know enough alternate * locations, assume it is popular and do not publish it yet. * * We do not publish the SHA-1 of a partial file for which we know * of at least two alternate locations because the purpose of us publishing * these partial SHA-1s is to attract other PFSP-aware hosts and * recreate a mesh. */ alt_locs = dmesh_count(pe->sha1); is_partial = is_partial && !shared_file_is_finished(sf); if (alt_locs > (is_partial ? PUBLISH_PARTIAL_MAX : PUBLISH_DMESH_MAX)) { if (GNET_PROPERTY(publisher_debug)) { g_debug("PUBLISHER SHA-1 %s %s\"%s\" has %d download mesh " "entr%s, skipped", sha1_to_string(pe->sha1), is_partial ? "partial " : "", shared_file_name_nfc(sf), alt_locs, plural_y(alt_locs)); } publisher_hold(pe, PUBLISH_POPULAR, "popular file"); goto done; } /* * If the DHT is not enabled, postpone processing. */ if (!dht_enabled()) { publisher_hold(pe, PUBLISH_BUSY, "DHT disabled"); goto done; } /* * If this is a partial file for which we have less than the minimum * for PFSP sharing, or if PFSP has been disabled, skip it. */ if (shared_file_is_partial(sf)) { fileinfo_t *fi = shared_file_fileinfo(sf); if ( !file_info_partial_shareable(fi) || fi->done < GNET_PROPERTY(pfsp_minimum_filesize) ) { publisher_hold(pe, PUBLISH_BUSY, "PFSP minima not reached"); goto done; } } /* * Check whether it is time to process the entry, in case we're * restarting quickly after a shutdown. */ if (0 == pe->last_publish) { struct pubdata *pd = get_pubdata(pe->sha1); if (pd != NULL) { time_t now = tm_time(); time_delta_t enqueue = delta_time(pd->next_enqueue, now); time_delta_t expire = delta_time(pd->expiration, now); if (enqueue > 0 && (0 == pd->expiration || expire > 0)) { int delay = MIN(enqueue, PUBLISH_POPULAR); if (pd->expiration != 0) delay = MIN(delay, expire); if (GNET_PROPERTY(publisher_debug) > 1) { g_debug("PUBLISHER SHA-1 %s delayed by %s", sha1_to_string(pe->sha1), compact_time(enqueue)); } publisher_retry(pe, delay, "first-time delay"); goto done; } } } /* * Cancel possible remaining backgrounded publishing. */ if (pe->backgrounded) { pdht_cancel_file(pe->sha1, FALSE); pe->backgrounded = FALSE; } /* * OK, we can publish this alternate location. */ if (pe->last_publish) { if (GNET_PROPERTY(publisher_debug) > 2) { g_debug("PUBLISHER SHA-1 %s re-enqueued %d secs " "after last publish", sha1_to_string(pe->sha1), (int) delta_time(tm_time(), pe->last_publish)); } } pe->last_enqueued = tm_time(); pdht_publish_file(sf, publisher_done, pe); /* FALL THROUGH */ done: shared_file_unref(&sf); }