size_t
thex_upload_get_content_length(const shared_file_t *sf)
{
const struct tth *tth;
size_t n_leaves, n_nodes;
size_t size = 0;
g_return_val_if_fail(sf, 0);
tth = shared_file_tth(sf);
g_return_val_if_fail(tth, 0);
size += thex_upload_prepare_xml(NULL, tth, shared_file_size(sf));
n_nodes = 1;
n_leaves = tt_good_node_count(shared_file_size(sf));
while (n_leaves > 1) {
n_nodes += n_leaves;
n_leaves = (n_leaves + 1) / 2;
}
size += thex_upload_prepare_tree(NULL, tth, NULL, n_nodes);
return size;
}
bool
huge_update_hashes(shared_file_t *sf,
const struct sha1 *sha1, const struct tth *tth)
{
struct sha1_cache_entry *cached;
filestat_t sb;
shared_file_check(sf);
g_return_val_if_fail(sha1, FALSE);
/*
* Make sure the file's timestamp is still accurate.
*/
if (-1 == stat(shared_file_path(sf), &sb)) {
g_warning("discarding SHA1 for file \"%s\": can't stat(): %m",
shared_file_path(sf));
shared_file_remove(sf);
return TRUE;
}
if (sb.st_mtime != shared_file_modification_time(sf)) {
g_warning("file \"%s\" was modified whilst SHA1 was computed",
shared_file_path(sf));
shared_file_set_modification_time(sf, sb.st_mtime);
request_sha1(sf); /* Retry! */
return TRUE;
}
if (huge_spam_check(sf, sha1)) {
shared_file_remove(sf);
return FALSE;
}
shared_file_set_sha1(sf, sha1);
shared_file_set_tth(sf, tth);
/* Update cache */
cached = hikset_lookup(sha1_cache, shared_file_path(sf));
if (cached) {
update_volatile_cache(cached, shared_file_size(sf),
shared_file_modification_time(sf), sha1, tth);
cache_dirty = TRUE;
/* Dump the cache at most about once per minute. */
if (!cache_dumped || delta_time(tm_time(), cache_dumped) > 60) {
dump_cache(FALSE);
}
} else {
add_volatile_cache_entry(shared_file_path(sf),
shared_file_size(sf), shared_file_modification_time(sf),
sha1, tth, TRUE);
add_persistent_cache_entry(shared_file_path(sf),
shared_file_size(sf), shared_file_modification_time(sf),
sha1, tth);
}
return TRUE;
}
/**
* Check to see if an (in-memory) entry cache is up to date.
*
* @return true (in the C sense) if it is, or false otherwise.
*/
static bool
cached_entry_up_to_date(const struct sha1_cache_entry *cache_entry,
const shared_file_t *sf)
{
return cache_entry->size == shared_file_size(sf)
&& cache_entry->mtime == shared_file_modification_time(sf);
}
void
request_tigertree(shared_file_t *sf, bool high_priority)
{
int inserted;
verify_tth_init();
shared_file_check(sf);
g_return_if_fail(shared_file_is_finished(sf));
if (!shared_file_is_servable(sf))
return; /* "stale" shared file, has been superseded or removed */
/*
* This routine can be called when the VERIFY_DONE event is received by
* huge_verify_callback(). We may have already shutdown the TTH
* verification thread.
*/
if G_UNLIKELY(NULL == verify_tth.verify)
return;
sf = shared_file_ref(sf);
inserted = verify_enqueue(verify_tth.verify, high_priority,
shared_file_path(sf), 0, shared_file_size(sf),
request_tigertree_callback, sf);
if (!inserted)
shared_file_unref(&sf);
}
/**
* Add `comp' to the current completed count, and update the amount of
* bytes transferred. Note that `comp' can be zero.
* When `update_dtime' is TRUE, we update the "done time", otherwise we
* change the "last request time".
*
* If the row does not exist (race condition: deleted since upload started),
* recreate one.
*/
static void
upload_stats_file_add(
const shared_file_t *sf,
int comp, guint64 sent, gboolean update_dtime)
{
const char *pathname = shared_file_path(sf);
filesize_t size = shared_file_size(sf);
struct ul_stats *s;
const struct sha1 *sha1;
g_assert(comp >= 0);
sha1 = sha1_hash_available(sf) ? shared_file_sha1(sf) : NULL;
/* find this file in the ul_stats_clist */
s = upload_stats_find(sha1, pathname, size);
/* increment the completed counter */
if (NULL == s) {
/* uh oh, row has since been deleted, add it: 1 attempt */
upload_stats_add(pathname, size, shared_file_name_nfc(sf),
1, comp, sent, tm_time(), tm_time(), sha1);
} else {
s->bytes_sent += sent;
s->norm = 1.0 * s->bytes_sent / s->size;
s->complete += comp;
if (update_dtime)
s->dtime = tm_time();
else
s->rtime = tm_time();
gcu_upload_stats_gui_update(s);
}
dirty = TRUE; /* Request asynchronous save of stats */
}
/**
* Called when an upload starts.
*/
void
upload_stats_file_begin(const shared_file_t *sf)
{
struct ul_stats *s;
const char *pathname;
filesize_t size;
const struct sha1 *sha1;
g_return_if_fail(sf);
pathname = shared_file_path(sf);
size = shared_file_size(sf);
sha1 = sha1_hash_available(sf) ? shared_file_sha1(sf) : NULL;
/* find this file in the ul_stats_clist */
s = upload_stats_find(sha1, pathname, size);
/* increment the attempted counter */
if (NULL == s) {
upload_stats_add(pathname, size, shared_file_name_nfc(sf),
1, 0, 0, tm_time(), 0, sha1);
} else {
s->attempts++;
s->rtime = tm_time();
gcu_upload_stats_gui_update(s);
}
dirty = TRUE; /* Request asynchronous save of stats */
}
/**
* Creates a new THEX upload context. The context must be freed with
* thex_upload_close().
*
* @param owner the owner of the TX stack (the upload)
* @param host the host to which we're talking to
* @param writable no document
* @param link_cb callbacks for the link layer
* @param wio no document
* @param flags opening flags
*
* @return An initialized THEX upload context.
*/
struct special_upload *
thex_upload_open(
void *owner,
const struct gnutella_host *host,
const shared_file_t *sf,
special_upload_writable_t writable,
const struct tx_link_cb *link_cb,
struct wrap_io *wio,
int flags)
{
struct thex_upload *ctx;
WALLOC(ctx);
ctx->special_upload.read = thex_upload_read;
ctx->special_upload.write = thex_upload_write;
ctx->special_upload.flush = thex_upload_flush;
ctx->special_upload.close = thex_upload_close;
ctx->tth = atom_tth_get(shared_file_tth(sf));
ctx->filesize = shared_file_size(sf);
ctx->data = NULL;
ctx->size = 0;
ctx->offset = 0;
ctx->state = THEX_STATE_INITIAL;
/*
* Instantiate the TX stack.
*/
{
struct tx_link_args args;
args.cb = link_cb;
args.wio = wio;
args.bws = bsched_out_select_by_addr(gnet_host_get_addr(host));
ctx->tx = tx_make(owner, host, tx_link_get_ops(), &args);
}
if (flags & THEX_UPLOAD_F_CHUNKED) {
ctx->tx = tx_make_above(ctx->tx, tx_chunk_get_ops(), 0);
}
/*
* Put stack in "eager" mode: we want to be notified whenever
* we can write something.
*/
tx_srv_register(ctx->tx, writable, owner);
tx_eager_mode(ctx->tx, TRUE);
/*
* Update statistics.
*/
gnet_prop_incr_guint32(PROP_THEX_FILES_REQUESTED);
return &ctx->special_upload;
}
static bool
request_tigertree_callback(const struct verify *ctx, enum verify_status status,
void *user_data)
{
shared_file_t *sf = user_data;
shared_file_check(sf);
switch (status) {
case VERIFY_START:
if (!(SHARE_F_INDEXED & shared_file_flags(sf))) {
/*
* After a rescan, there might be files in the queue which are
* no longer shared.
*/
if (GNET_PROPERTY(verify_debug) > 1) {
g_debug("skipping TTH computation for %s: no longer shared",
shared_file_path(sf));
}
return FALSE;
}
if (
shared_file_tth(sf) &&
tth_cache_lookup(shared_file_tth(sf), shared_file_size(sf)) > 0
) {
if (
GNET_PROPERTY(tigertree_debug) > 1 ||
GNET_PROPERTY(verify_debug) > 1
) {
g_debug("TTH for %s is already cached (%s)",
shared_file_path(sf), tth_base32(shared_file_tth(sf)));
}
return FALSE;
}
gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, TRUE);
return TRUE;
case VERIFY_PROGRESS:
return 0 != (SHARE_F_INDEXED & shared_file_flags(sf));
case VERIFY_DONE:
{
const struct tth *tth = verify_tth_digest(ctx);
huge_update_hashes(sf, shared_file_sha1(sf), tth);
tth_cache_insert(tth, verify_tth_leaves(ctx),
verify_tth_leave_count(ctx));
}
/* FALL THROUGH */
case VERIFY_ERROR:
case VERIFY_SHUTDOWN:
shared_file_unref(&sf);
gnet_prop_set_boolean_val(PROP_TTH_REBUILDING, FALSE);
return TRUE;
case VERIFY_INVALID:
break;
}
g_assert_not_reached();
return FALSE;
}
/**
* Put the shared file on the stack of the things to do.
*
* We first begin with the computation of the SHA1, and when completed we
* will continue with the TTH computation.
*/
static void
queue_shared_file_for_sha1_computation(shared_file_t *sf)
{
int inserted;
shared_file_check(sf);
inserted = verify_sha1_enqueue(FALSE, shared_file_path(sf),
shared_file_size(sf), huge_verify_callback,
shared_file_ref(sf));
if (!inserted)
shared_file_unref(&sf);
}
static bool
huge_spam_check(shared_file_t *sf, const struct sha1 *sha1)
{
if (NULL != sha1 && spam_sha1_check(sha1)) {
g_warning("file \"%s\" is listed as spam (SHA1)", shared_file_path(sf));
return TRUE;
}
if (
spam_check_filename_size(shared_file_name_nfc(sf),
shared_file_size(sf))
) {
g_warning("file \"%s\" is listed as spam (Name)", shared_file_path(sf));
return TRUE;
}
return FALSE;
}
void
request_tigertree(shared_file_t *sf, bool high_priority)
{
int inserted;
verify_tth_init();
g_return_if_fail(sf);
shared_file_check(sf);
g_return_if_fail(!shared_file_is_partial(sf));
sf = shared_file_ref(sf);
inserted = verify_enqueue(verify_tth.verify, high_priority,
shared_file_path(sf), 0, shared_file_size(sf),
request_tigertree_callback, sf);
if (!inserted)
shared_file_unref(&sf);
}
/**
* Writes the browse host data of the context ``ctx'' to the buffer
* ``dest''. This must be called multiple times to retrieve the complete
* data until zero is returned i.e., the end of file is reached.
*
* This routine deals with HTML data generation.
*
* @param ctx an initialized browse host context.
* @param dest the destination buffer.
* @param size the amount of bytes ``dest'' can hold.
*
* @return -1 on failure, zero at the end-of-file condition or if size
* was zero. On success, the amount of bytes copied to ``dest''
* is returned.
*/
static ssize_t
browse_host_read_html(struct special_upload *ctx,
void *const dest, size_t size)
{
static const char header[] =
"<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\r\n"
"<html>\r\n"
"<head>\r\n"
"<title>Browse Host</title>\r\n"
"</head>\r\n"
"<body>\r\n";
static const char trailer[] = "</ul>\r\n</body>\r\n</html>\r\n";
struct browse_host_upload *bh = cast_to_browse_host_upload(ctx);
char *p = dest;
g_assert(NULL != bh);
g_assert(NULL != dest);
g_assert(size <= INT_MAX);
g_assert(UNSIGNED(bh->state) < NUM_BH_STATES);
g_assert(bh->b_size <= INT_MAX);
g_assert(bh->b_offset <= bh->b_size);
do {
switch (bh->state) {
case BH_STATE_HEADER:
if (!bh->b_data) {
bh->b_data = header;
bh->b_size = CONST_STRLEN(header);
}
p += browse_host_read_data(bh, p, &size);
if (bh->b_size == bh->b_offset)
browse_host_next_state(bh, BH_STATE_LIBRARY_INFO);
break;
case BH_STATE_LIBRARY_INFO:
if (!bh->b_data) {
bh->w_buf_size = w_concat_strings(&bh->w_buf,
"<h1>", product_get_name(), "</h1>\r\n"
"<h3>", version_get_string(),
" sharing ",
uint64_to_string(shared_files_scanned()),
" file",
shared_files_scanned() == 1 ? "" : "s",
" ",
short_kb_size(shared_kbytes_scanned(),
GNET_PROPERTY(display_metric_units)),
" total</h3>\r\n"
"<ul>\r\n", (void *) 0);
bh->b_data = bh->w_buf;
bh->b_size = bh->w_buf_size - 1; /* minus trailing NUL */
bh->b_offset = 0;
}
p += browse_host_read_data(bh, p, &size);
if (bh->b_size == bh->b_offset)
browse_host_next_state(bh, BH_STATE_FILES);
break;
case BH_STATE_TRAILER:
if (!bh->b_data) {
bh->b_data = trailer;
bh->b_size = CONST_STRLEN(trailer);
}
p += browse_host_read_data(bh, p, &size);
if (bh->b_size == bh->b_offset)
browse_host_next_state(bh, BH_STATE_EOF);
break;
case BH_STATE_FILES:
if (bh->b_data && bh->b_size == bh->b_offset) {
g_assert(bh->w_buf == bh->b_data);
wfree(bh->w_buf, bh->w_buf_size);
bh->w_buf = NULL;
bh->w_buf_size = 0;
bh->b_data = NULL;
}
if (!bh->b_data) {
const shared_file_t *sf;
bh->file_index++;
sf = shared_file_sorted(bh->file_index);
if (!sf) {
if (bh->file_index > shared_files_scanned())
browse_host_next_state(bh, BH_STATE_TRAILER);
/* Skip holes in the file_index table */
} else if (SHARE_REBUILDING == sf) {
browse_host_next_state(bh, BH_STATE_REBUILDING);
} else {
const char * const name_nfc = shared_file_name_nfc(sf);
const filesize_t file_size = shared_file_size(sf);
size_t html_size;
char *html_name;
{
const char *dir;
char *name;
dir = shared_file_relative_path(sf);
if (dir) {
name = h_strconcat(dir, "/", name_nfc, (void *) 0);
} else {
name = deconstify_char(name_nfc);
}
html_size = 1 + html_escape(name, NULL, 0);
html_name = walloc(html_size);
html_escape(name, html_name, html_size);
if (name != name_nfc) {
HFREE_NULL(name);
}
}
if (sha1_hash_available(sf)) {
const struct sha1 *sha1 = shared_file_sha1(sf);
bh->w_buf_size = w_concat_strings(&bh->w_buf,
"<li><a href=\"/uri-res/N2R?urn:sha1:",
sha1_base32(sha1),
"\">", html_name, "</a> [",
short_html_size(file_size,
GNET_PROPERTY(display_metric_units)),
"]</li>\r\n",
(void *) 0);
} else {
char *escaped;
escaped = url_escape(name_nfc);
bh->w_buf_size = w_concat_strings(&bh->w_buf,
"<li><a href=\"/get/",
uint32_to_string(shared_file_index(sf)),
"/", escaped, "\">", html_name, "</a>"
" [",
short_html_size(file_size,
GNET_PROPERTY(display_metric_units)),
"]</li>\r\n", (void *) 0);
if (escaped != name_nfc) {
HFREE_NULL(escaped);
}
}
wfree(html_name, html_size);
bh->b_data = bh->w_buf;
bh->b_size = bh->w_buf_size - 1; /* minus trailing NUL */
bh->b_offset = 0;
}
}
if (bh->b_data)
p += browse_host_read_data(bh, p, &size);
break;
case BH_STATE_REBUILDING:
if (!bh->b_data) {
static const char msg[] =
"<li>"
"<b>"
"The library is currently being rebuild. Please, "
"try again in a moment."
"</b>"
"</li>";
bh->b_data = msg;
bh->b_size = CONST_STRLEN(msg);
}
p += browse_host_read_data(bh, p, &size);
if (bh->b_size == bh->b_offset)
browse_host_next_state(bh, BH_STATE_TRAILER);
break;
case BH_STATE_EOF:
return p - cast_to_char_ptr(dest);
case NUM_BH_STATES:
g_assert_not_reached();
}
} while (size > 0);
return p - cast_to_char_ptr(dest);
}
/**
* Add file to the current query hit.
*
* @return TRUE if we kept the file, FALSE if we did not include it in the hit.
*/
static bool
g2_build_qh2_add(struct g2_qh2_builder *ctx, const shared_file_t *sf)
{
const sha1_t *sha1;
g2_tree_t *h, *c;
shared_file_check(sf);
/*
* Make sure the file is still in the library.
*/
if (0 == shared_file_index(sf))
return FALSE;
/*
* On G2, the H/URN child is required, meaning we need the SHA1 at least.
*/
if (!sha1_hash_available(sf))
return FALSE;
/*
* Do not send duplicates, as determined by the SHA1 of the resource.
*
* A user may share several files with different names but the same SHA1,
* and if all of them are hits, we only want to send one instance.
*
* When generating hits for host-browsing, we do not care about duplicates
* and ctx->hs is NULL then.
*/
sha1 = shared_file_sha1(sf); /* This is an atom */
if (ctx->hs != NULL) {
if (hset_contains(ctx->hs, sha1))
return FALSE;
hset_insert(ctx->hs, sha1);
}
/*
* Create the "H" child and attach it to the current tree.
*/
if (NULL == ctx->t)
g2_build_qh2_start(ctx);
h = g2_tree_alloc_empty("H");
g2_tree_add_child(ctx->t, h);
/*
* URN -- Universal Resource Name
*
* If there is a known TTH, then we can generate a bitprint, otherwise
* we just convey the SHA1.
*/
{
const tth_t * const tth = shared_file_tth(sf);
char payload[SHA1_RAW_SIZE + TTH_RAW_SIZE + sizeof G2_URN_BITPRINT];
char *p = payload;
if (NULL == tth) {
p = mempcpy(p, G2_URN_SHA1, sizeof G2_URN_SHA1);
p += clamp_memcpy(p, sizeof payload - ptr_diff(p, payload),
sha1, SHA1_RAW_SIZE);
} else {
p = mempcpy(p, G2_URN_BITPRINT, sizeof G2_URN_BITPRINT);
p += clamp_memcpy(p, sizeof payload - ptr_diff(p, payload),
sha1, SHA1_RAW_SIZE);
p += clamp_memcpy(p, sizeof payload - ptr_diff(p, payload),
tth, TTH_RAW_SIZE);
}
g_assert(ptr_diff(p, payload) <= sizeof payload);
c = g2_tree_alloc_copy("URN", payload, ptr_diff(p, payload));
g2_tree_add_child(h, c);
}
/*
* URL -- empty to indicate that we share the file via uri-res.
*/
if (ctx->flags & QHIT_F_G2_URL) {
uint known;
uint16 csc;
c = g2_tree_alloc_empty("URL");
g2_tree_add_child(h, c);
/*
* CSC -- if we know alternate sources, indicate how many in "CSC".
*
* This child is only emitted when they requested "URL".
*/
known = dmesh_count(sha1);
csc = MIN(known, MAX_INT_VAL(uint16));
if (csc != 0) {
char payload[2];
poke_le16(payload, csc);
c = g2_tree_alloc_copy("CSC", payload, sizeof payload);
g2_tree_add_child(h, c);
}
/*
* PART -- if we only have a partial file, indicate how much we have.
*
* This child is only emitted when they requested "URL".
*/
if (shared_file_is_partial(sf) && !shared_file_is_finished(sf)) {
filesize_t available = shared_file_available(sf);
char payload[8]; /* If we have to encode file size as 64-bit */
uint32 av32;
time_t mtime = shared_file_modification_time(sf);
c = g2_tree_alloc_empty("PART");
g2_tree_add_child(h, c);
av32 = available;
if (av32 == available) {
/* Fits within a 32-bit quantity */
poke_le32(payload, av32);
g2_tree_set_payload(c, payload, sizeof av32, TRUE);
} else {
/* Encode as a 64-bit quantity then */
poke_le64(payload, available);
g2_tree_set_payload(c, payload, sizeof payload, TRUE);
}
/*
* GTKG extension: encode the last modification time of the
* partial file in an "MT" child. This lets the other party
* determine whether the host is still able to actively complete
* the file.
*/
poke_le32(payload, (uint32) mtime);
g2_tree_add_child(c,
g2_tree_alloc_copy("MT", payload, sizeof(uint32)));
}
/*
* CT -- creation time of the resource (GTKG extension).
*/
{
time_t create_time = shared_file_creation_time(sf);
if ((time_t) -1 != create_time) {
char payload[8];
int n;
create_time = MAX(0, create_time);
n = vlint_encode(create_time, payload);
g2_tree_add_child(h,
g2_tree_alloc_copy("CT", payload, n)); /* No trailing 0s */
}
}
}
/*
* DN -- distinguished name.
*
* Note that the presence of DN also governs the presence of SZ if the
* file length does not fit a 32-bit unsigned quantity.
*/
if (ctx->flags & QHIT_F_G2_DN) {
char payload[8]; /* If we have to encode file size as 64-bit */
uint32 fs32;
filesize_t fs = shared_file_size(sf);
const char *name;
const char *rp;
c = g2_tree_alloc_empty("DN");
fs32 = fs;
if (fs32 == fs) {
/* Fits within a 32-bit quantity */
poke_le32(payload, fs32);
g2_tree_set_payload(c, payload, sizeof fs32, TRUE);
} else {
/* Does not fit a 32-bit quantity, emit a SZ child */
poke_le64(payload, fs);
g2_tree_add_child(h,
g2_tree_alloc_copy("SZ", payload, sizeof payload));
}
name = shared_file_name_nfc(sf);
g2_tree_append_payload(c, name, shared_file_name_nfc_len(sf));
g2_tree_add_child(h, c);
/*
* GTKG extension: if there is a file path, expose it as a "P" child
* under the DN node.
*/
rp = shared_file_relative_path(sf);
if (rp != NULL) {
g2_tree_add_child(c, g2_tree_alloc_copy("P", rp, strlen(rp)));
}
}
/*
* GTKG extension: if they requested alt-locs in the /Q2/I with "A", then
* send them some known alt-locs in an "ALT" child.
*
* Note that these alt-locs can be for Gnutella hosts: since both Gnutella
* and G2 share a common HTTP-based file transfer mechanism with compatible
* extra headers, there is no need to handle them separately.
*/
if (ctx->flags & QHIT_F_G2_ALT) {
gnet_host_t hvec[G2_BUILD_QH2_MAX_ALT];
int hcnt = 0;
hcnt = dmesh_fill_alternate(sha1, hvec, N_ITEMS(hvec));
if (hcnt > 0) {
int i;
c = g2_tree_alloc_empty("ALT");
for (i = 0; i < hcnt; i++) {
host_addr_t addr;
uint16 port;
addr = gnet_host_get_addr(&hvec[i]);
port = gnet_host_get_port(&hvec[i]);
if (host_addr_is_ipv4(addr)) {
char payload[6];
host_ip_port_poke(payload, addr, port, NULL);
g2_tree_append_payload(c, payload, sizeof payload);
}
}
/*
* If the payload is still empty, then drop the "ALT" child.
* Otherwise, attach it to the "H" node.
*/
if (NULL == g2_tree_node_payload(c, NULL)) {
g2_tree_free_null(&c);
} else {
g2_tree_add_child(h, c);
}
}
}
/*
* Update the size of the query hit we're generating.
*/
ctx->current_size += g2_frame_serialize(h, NULL, 0);
return TRUE;
}