rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response
(unsigned char *buffer,int size, int max_response_bytes)
{
if (size<10) return NULL;
if (size>65536) return NULL;
if (max_response_bytes<10) return NULL;
if (max_response_bytes>1048576) return NULL;
int them_count=(size-10)/RHIZOME_BAR_BYTES;
/* We need to get a list of BARs that will fit into max_response_bytes when we
have summarised them into (1+RHIZOME_BAR_PREFIX_BYTES)-byte PUSH/PULL hints.
So we need an intermediate buffer that is somewhat larger to allow the actual
maximum response buffer to be completely filled. */
int max_intermediate_bytes
=10+((max_response_bytes-10)/(1+RHIZOME_BAR_PREFIX_BYTES))*RHIZOME_BAR_BYTES;
unsigned char usbuffer[max_intermediate_bytes];
rhizome_direct_bundle_cursor
*c=rhizome_direct_bundle_iterator(max_intermediate_bytes);
assert(c!=NULL);
if (rhizome_direct_bundle_iterator_unpickle_range(c,buffer,10))
{
DEBUGF("Couldn't unpickle range");
rhizome_direct_bundle_iterator_free(&c);
return NULL;
}
DEBUGF("unpickled size_high=%lld, limit_size_high=%lld",
c->size_high,c->limit_size_high);
DEBUGF("c->buffer_size=%d",c->buffer_size);
/* Get our list of BARs for the same cursor range */
int us_count=rhizome_direct_bundle_iterator_fill(c,-1);
DEBUGF("Found %d manifests in that range",us_count);
/* Transfer to a temporary buffer, so that we can overwrite
the cursor's buffer with the response data. */
bcopy(c->buffer,usbuffer,10+us_count*RHIZOME_BAR_BYTES);
c->buffer_offset_bytes=10;
c->buffer_used=0;
/* Iterate until we are through both lists.
Note that the responses are (1+RHIZOME_BAR_PREFIX_BYTES)-bytes each, much
smaller than the 32 bytes used by BARs, therefore the response will never be
bigger than the request, and so we don't need to worry about overflows. */
int them=0,us=0;
DEBUGF("themcount=%d, uscount=%d",them_count,us_count);
while(them<them_count||us<us_count)
{
DEBUGF("them=%d, us=%d",them,us);
unsigned char *them_bar=&buffer[10+them*RHIZOME_BAR_BYTES];
unsigned char *us_bar=&usbuffer[10+us*RHIZOME_BAR_BYTES];
int relation=0;
if (them<them_count&&us<us_count) {
relation=memcmp(them_bar,us_bar,RHIZOME_BAR_COMPARE_BYTES);
DEBUGF("relation = %d",relation);
dump("them BAR",them_bar,RHIZOME_BAR_BYTES);
dump("us BAR",us_bar,RHIZOME_BAR_BYTES);
}
else if (us==us_count) relation=-1; /* they have a bundle we don't have */
else if (them==them_count) relation=+1; /* we have a bundle they don't have */
else {
DEBUGF("This should never happen.");
break;
}
int who=0;
if (relation<0) {
/* They have a bundle that we don't have any version of.
Append 16-byte "please send" record consisting of 0x01 followed
by the eight-byte BID prefix from the BAR. */
c->buffer[c->buffer_offset_bytes+c->buffer_used]=0x01; /* Please send */
bcopy(&buffer[10+them*RHIZOME_BAR_BYTES+RHIZOME_BAR_PREFIX_OFFSET],
&c->buffer[c->buffer_offset_bytes+c->buffer_used+1],
RHIZOME_BAR_PREFIX_BYTES);
c->buffer_used+=1+RHIZOME_BAR_PREFIX_BYTES;
who=-1;
DEBUGF("They have previously unseen bundle %016llx*",
rhizome_bar_bidprefix_ll(&buffer[10+them*RHIZOME_BAR_BYTES]));
} else if (relation>0) {
/* We have a bundle that they don't have any version of
Append 16-byte "I have [newer]" record consisting of 0x02 followed
by the eight-byte BID prefix from the BAR. */
c->buffer[c->buffer_offset_bytes+c->buffer_used]=0x02; /* I have [newer] */
bcopy(&usbuffer[10+us*RHIZOME_BAR_BYTES+RHIZOME_BAR_PREFIX_OFFSET],
&c->buffer[c->buffer_offset_bytes+c->buffer_used+1],
RHIZOME_BAR_PREFIX_BYTES);
c->buffer_used+=1+RHIZOME_BAR_PREFIX_BYTES;
who=+1;
DEBUGF("We have previously unseen bundle %016llx*",
rhizome_bar_bidprefix_ll(&usbuffer[10+us*RHIZOME_BAR_BYTES]));
} else {
/* We each have a version of this bundle, so see whose is newer */
long long them_version
=rhizome_bar_version(&buffer[10+them*RHIZOME_BAR_BYTES]);
long long us_version
=rhizome_bar_version(&usbuffer[10+us*RHIZOME_BAR_BYTES]);
if (them_version>us_version) {
/* They have the newer version of the bundle */
c->buffer[c->buffer_offset_bytes+c->buffer_used]=0x01; /* Please send */
bcopy(&buffer[10+them*RHIZOME_BAR_BYTES+RHIZOME_BAR_PREFIX_OFFSET],
&c->buffer[c->buffer_offset_bytes+c->buffer_used+1],
RHIZOME_BAR_PREFIX_BYTES);
c->buffer_used+=1+RHIZOME_BAR_PREFIX_BYTES;
DEBUGF("They have newer version of bundle %016llx* (%lld versus %lld)",
rhizome_bar_bidprefix_ll(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&buffer[10+them*RHIZOME_BAR_BYTES]));
} else if (them_version<us_version) {
/* We have the newer version of the bundle */
c->buffer[c->buffer_offset_bytes+c->buffer_used]=0x02; /* I have [newer] */
bcopy(&usbuffer[10+us*RHIZOME_BAR_BYTES+RHIZOME_BAR_PREFIX_OFFSET],
&c->buffer[c->buffer_offset_bytes+c->buffer_used+1],
RHIZOME_BAR_PREFIX_BYTES);
c->buffer_used+=1+RHIZOME_BAR_PREFIX_BYTES;
DEBUGF("We have newer version of bundle %016llx* (%lld versus %lld)",
rhizome_bar_bidprefix_ll(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&usbuffer[10+us*RHIZOME_BAR_BYTES]),
rhizome_bar_version(&buffer[10+them*RHIZOME_BAR_BYTES]));
} else {
DEBUGF("We both have the same version of %016llx*",
rhizome_bar_bidprefix_ll(&buffer[10+them*RHIZOME_BAR_BYTES]));
}
}
/* Advance through lists accordingly */
switch(who) {
case -1: them++; break;
case +1: us++; break;
case 0: them++; us++; break;
}
}
return c;
}
void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
{
DEBUGF("Dispatch size_high=%lld",r->cursor->size_high);
rhizome_direct_transport_state_http *state = r->transport_specific_state;
int sock=socket(AF_INET, SOCK_STREAM, 0);
if (sock==-1) {
WHY_perror("socket");
goto end;
}
struct hostent *hostent;
hostent = gethostbyname(state->host);
if (!hostent) {
DEBUGF("could not resolve hostname");
goto end;
}
struct sockaddr_in addr;
addr.sin_family = AF_INET;
addr.sin_port = htons(state->port);
addr.sin_addr = *((struct in_addr *)hostent->h_addr);
bzero(&(addr.sin_zero),8);
if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) {
WHY_perror("connect");
close(sock);
goto end;
}
char boundary[20];
char buffer[8192];
strbuf bb = strbuf_local(boundary, sizeof boundary);
strbuf_sprintf(bb, "%08lx%08lx", random(), random());
assert(!strbuf_overrun(bb));
strbuf content_preamble = strbuf_alloca(200);
strbuf content_postamble = strbuf_alloca(40);
strbuf_sprintf(content_preamble,
"--%s\r\n"
"Content-Disposition: form-data; name=\"data\"; filename=\"IHAVEs\"\r\n"
"Content-Type: application/octet-stream\r\n"
"\r\n",
boundary
);
strbuf_sprintf(content_postamble, "\r\n--%s--\r\n", boundary);
assert(!strbuf_overrun(content_preamble));
assert(!strbuf_overrun(content_postamble));
int content_length = strbuf_len(content_preamble)
+ r->cursor->buffer_offset_bytes
+ r->cursor->buffer_used
+ strbuf_len(content_postamble);
strbuf request = strbuf_local(buffer, sizeof buffer);
strbuf_sprintf(request,
"POST /rhizome/enquiry HTTP/1.0\r\n"
"Content-Length: %d\r\n"
"Content-Type: multipart/form-data; boundary=%s\r\n"
"\r\n%s",
content_length, boundary, strbuf_str(content_preamble)
);
assert(!strbuf_overrun(request));
/* TODO: Refactor this code so that it uses our asynchronous framework.
*/
int len = strbuf_len(request);
int sent=0;
while(sent<len) {
DEBUGF("write(%d, %s, %d)", sock, alloca_toprint(-1, &buffer[sent], len-sent), len-sent);
int count=write(sock,&buffer[sent],len-sent);
if (count == -1) {
if (errno==EPIPE) goto rx;
WHYF_perror("write(%d)", len - sent);
close(sock);
goto end;
}
sent+=count;
}
len=r->cursor->buffer_offset_bytes+r->cursor->buffer_used;
sent=0;
while(sent<len) {
int count=write(sock,&r->cursor->buffer[sent],len-sent);
if (count == -1) {
if (errno == EPIPE)
goto rx;
WHYF_perror("write(%d)", count);
close(sock);
goto end;
}
sent+=count;
}
strbuf_reset(request);
strbuf_puts(request, strbuf_str(content_postamble));
len = strbuf_len(request);
sent=0;
while(sent<len) {
DEBUGF("write(%d, %s, %d)", sock, alloca_toprint(-1, &buffer[sent], len-sent), len-sent);
int count=write(sock,&buffer[sent],len-sent);
if (count == -1) {
if (errno==EPIPE) goto rx;
WHYF_perror("write(%d)", len - sent);
close(sock);
goto end;
}
sent+=count;
}
struct http_response_parts parts;
rx:
/* request sent, now get response back. */
if (receive_http_response(sock, buffer, sizeof buffer, &parts) == -1) {
close(sock);
goto end;
}
/* For some reason the response data gets overwritten during a push,
so we need to copy it, and use the copy instead. */
unsigned char *actionlist=alloca(parts.content_length);
bcopy(parts.content_start, actionlist, parts.content_length);
dump("response", actionlist, parts.content_length);
/* We now have the list of (1+RHIZOME_BAR_PREFIX_BYTES)-byte records that indicate
the list of BAR prefixes that differ between the two nodes. We can now action
those which are relevant, i.e., based on whether we are pushing, pulling or
synchronising (both).
I am currently undecided as to whether it is cleaner to have some general
rhizome direct function for doing that, or whether it just adds unnecessary
complication, and the responses should just be handled in here.
For now, I am just going to implement it in here, and we can generalise later.
*/
int i;
for(i=10;i<content_length;i+=(1+RHIZOME_BAR_PREFIX_BYTES))
{
int type=actionlist[i];
unsigned long long
bid_prefix_ll=rhizome_bar_bidprefix_ll((unsigned char *)&actionlist[i+1]);
DEBUGF("%s %016llx* @ 0x%x",type==1?"push":"pull",bid_prefix_ll,i);
if (type==2&&r->pullP) {
/* Need to fetch manifest. Once we have the manifest, then we can
use our normal bundle fetch routines from rhizome_fetch.c
Generate a request like: GET /rhizome/manifestbybar/<hex of bar>
and add it to our list of HTTP fetch requests, then watch
until the request is finished. That will give us the manifest.
Then as noted above, we can use that to pull the file down using
existing routines.
*/
if (!rhizome_fetch_request_manifest_by_prefix
(&addr,&actionlist[i+1],RHIZOME_BAR_PREFIX_BYTES,
1 /* import, getting file if needed */))
{
/* Fetching the manifest, and then using it to see if we want to
fetch the file for import is all handled asynchronously, so just
wait for it to finish. */
while(rhizome_file_fetch_queue_count) fd_poll();
}
} else if (type==1&&r->pushP) {
/* Form up the POST request to submit the appropriate bundle. */
/* Start by getting the manifest, which is the main thing we need, and also
gives us the information we need for sending any associated file. */
rhizome_manifest
*m=rhizome_direct_get_manifest(&actionlist[i+1],
RHIZOME_BAR_PREFIX_BYTES);
if (!m) {
WHY("This should never happen. The manifest exists, but when I went looking for it, it doesn't appear to be there.");
goto next_item;
}
/* Get filehash and size from manifest if present */
const char *id = rhizome_manifest_get(m, "id", NULL, 0);
DEBUGF("bundle id = '%s'",id);
const char *hash = rhizome_manifest_get(m, "filehash", NULL, 0);
DEBUGF("bundle file hash = '%s'",hash);
long long filesize = rhizome_manifest_get_ll(m, "filesize");
DEBUGF("file size = %lld",filesize);
/* We now have everything we need to compose the POST request and send it.
*/
char *template="POST /rhizome/import HTTP/1.0\r\n"
