int rhizome_manifest_check_file(rhizome_manifest *m_in)
{
long long gotfile = 0;
if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE ID='%s' and datavalid=1;", m_in->fileHexHash) != 1) {
WHYF("Failed to count files");
return 0;
}
if (gotfile) {
DEBUGF("Skipping file checks for bundle, as file is already in the database");
return 0;
}
/* Find out whether the payload is expected to be encrypted or not */
m_in->payloadEncryption=rhizome_manifest_get_ll(m_in, "crypt");
/* Check payload file is accessible and discover its length, then check that it matches
the file size stored in the manifest */
long long mfilesize = rhizome_manifest_get_ll(m_in, "filesize");
m_in->fileLength = 0;
if (m_in->dataFileName[0]) {
struct stat stat;
if (lstat(m_in->dataFileName,&stat) == -1) {
if (errno != ENOENT || mfilesize != 0)
return WHYF_perror("stat(%s)", m_in->dataFileName);
} else {
m_in->fileLength = stat.st_size;
}
}
if (debug & DEBUG_RHIZOME)
DEBUGF("filename=%s, fileLength=%lld", m_in->dataFileName, m_in->fileLength);
if (mfilesize != -1 && mfilesize != m_in->fileLength) {
WHYF("Manifest.filesize (%lld) != actual file size (%lld)", mfilesize, m_in->fileLength);
return -1;
}
/* If payload is empty, ensure manifest has not file hash, otherwis compute the hash of the
payload and check that it matches manifest. */
const char *mhexhash = rhizome_manifest_get(m_in, "filehash", NULL, 0);
if (m_in->fileLength != 0) {
char hexhashbuf[RHIZOME_FILEHASH_STRLEN + 1];
if (rhizome_hash_file(m_in,m_in->dataFileName, hexhashbuf))
return WHY("Could not hash file.");
memcpy(&m_in->fileHexHash[0], &hexhashbuf[0], sizeof hexhashbuf);
m_in->fileHashedP = 1;
if (!mhexhash) return WHY("manifest contains no file hash");
if (mhexhash && strcmp(m_in->fileHexHash, mhexhash)) {
WHYF("Manifest.filehash (%s) does not match payload hash (%s)", mhexhash, m_in->fileHexHash);
return -1;
}
} else {
if (mhexhash != NULL) {
WHYF("Manifest.filehash (%s) should be absent for empty payload", mhexhash);
return -1;
}
}
return 0;
}
int rhizome_manifest_check_sanity(rhizome_manifest *m_in)
{
/* Ensure manifest meets basic sanity checks. */
const char *service = rhizome_manifest_get(m_in, "service", NULL, 0);
const char *sender = rhizome_manifest_get(m_in, "sender", NULL, 0);
const char *recipient = rhizome_manifest_get(m_in, "recipient", NULL, 0);
if (service == NULL || !service[0])
return WHY("Manifest missing 'service' field");
if (rhizome_manifest_get_ll(m_in, "date") == -1)
return WHY("Manifest missing 'date' field");
if (strcasecmp(service, RHIZOME_SERVICE_FILE) == 0) {
const char *name = rhizome_manifest_get(m_in, "name", NULL, 0);
if (name == NULL)
return WHY("Manifest missing 'name' field");
} else if (strcasecmp(service, RHIZOME_SERVICE_MESHMS) == 0) {
if (sender == NULL || !sender[0])
return WHY("MeshMS Manifest missing 'sender' field");
if (!str_is_subscriber_id(sender))
return WHYF("MeshMS Manifest contains invalid 'sender' field: %s", sender);
if (recipient == NULL || !recipient[0])
return WHY("MeshMS Manifest missing 'recipient' field");
if (!str_is_subscriber_id(recipient))
return WHYF("MeshMS Manifest contains invalid 'recipient' field: %s", recipient);
} else {
return WHY("Invalid service type");
}
if (debug & DEBUG_RHIZOME)
DEBUGF("sender='%s'", sender ? sender : "(null)");
/* passes all sanity checks */
return 0;
}
/* Import a bundle from the inbox folder.
Check that the manifest prototype is valid, and if so, complete it, and sign it if required and possible.
Note that bundles can either be an ordinary bundle, or a group description.
Group specifications are simply manifests that have the "isagroup" variable set.
Groups get stored in the manifests table AND a reference included in the
grouplist table.
Groups are allowed to be listed as being members of other groups.
This allows a nested, i.e., multi-level group heirarchy where sub-groups will only
typically be discovered by joining the parent group. Probably not a bad way to do
things.
The file should be included in the specified rhizome groups, if possible.
(some groups may be closed groups that we do not have the private key for.)
*/
int rhizome_bundle_import(char *bundle,char *groups[], int ttl,
int verifyP, int checkFileP, int signP)
{
char filename[1024];
char manifestname[1024];
char *buffer;
snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path,bundle); filename[1023]=0;
snprintf(manifestname,1024,"%s/import/manifest.%s",rhizome_datastore_path,bundle); manifestname[1023]=0;
/* Open files */
rhizome_manifest *m=rhizome_read_manifest_file(manifestname,0 /* file not buffer */,
RHIZOME_VERIFY);
if (!m) return WHY("Could not read manifest file.");
char hexhash[SHA512_DIGEST_STRING_LENGTH];
/* work out time to live */
if (ttl<0) ttl=0; if (ttl>254) ttl=254; m->ttl=ttl;
/* Keep associated file name handy for later */
m->dataFileName=strdup(filename);
struct stat stat;
if (lstat(filename,&stat)) {
return WHY("Could not stat() associated file");
m->fileLength=stat.st_size;
}
if (checkFileP||signP) {
if (rhizome_hash_file(filename,hexhash))
{ rhizome_manifest_free(m); return WHY("Could not hash file."); }
bcopy(&hexhash[0],&m->fileHexHash[0],SHA512_DIGEST_STRING_LENGTH);
m->fileHashedP=1;
}
if (verifyP)
{
/* Make sure hashes match.
Make sure that no signature verification errors were spotted on loading. */
int verifyErrors=0;
char *mhexhash;
if (checkFileP) {
if ((mhexhash=rhizome_manifest_get(m,"filehash",NULL,0))!=NULL)
if (strcmp(hexhash,mhexhash)) verifyErrors++; }
if (m->errors) verifyErrors+=m->errors;
if (verifyErrors) {
rhizome_manifest_free(m);
unlink(manifestname);
unlink(filename);
return WHY("Errors encountered verifying bundle manifest");
}
}
if (!verifyP) {
if ((buffer=rhizome_manifest_get(m,"id",NULL,0))!=NULL) {
/* No bundle id (256 bit random string being a public key in the NaCl CryptoSign crypto system),
so create one, and keep the private key handy. */
printf("manifest does not have an id\n");
rhizome_manifest_createid(m);
/* The ID is implicit in transit, but we need to store it in the file,
so that reimporting manifests on receiver nodes works easily.
We might implement something that strips the id variable out of the
manifest when sending it, or some other scheme to avoid sending all
the extra bytes. */
rhizome_manifest_set(m,"id",rhizome_bytes_to_hex(m->cryptoSignPublic,crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES));
} else {
/* An ID was specified, so remember it, and look for the private key if
we have it stowed away */
rhizome_hex_to_bytes(buffer,m->cryptoSignPublic,
crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2);
if (!rhizome_find_keypair_bytes(m->cryptoSignPublic,m->cryptoSignSecret))
m->haveSecret=1;
}
rhizome_manifest_set(m,"filehash",hexhash);
if (rhizome_manifest_get(m,"version",NULL,0)==NULL)
/* Version not set, so set one */
rhizome_manifest_set_ll(m,"version",overlay_gettime_ms());
rhizome_manifest_set_ll(m,"first_byte",0);
rhizome_manifest_set_ll(m,"last_byte",rhizome_file_size(filename));
}
/* Discard if it is older than the most recent known version */
long long storedversion = sqlite_exec_int64("SELECT version from manifests where id='%s';",rhizome_bytes_to_hex(m->cryptoSignPublic,crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES));
if (storedversion>rhizome_manifest_get_ll(m,"version"))
{
rhizome_manifest_free(m);
return WHY("Newer version exists");
}
/* Add group memberships */
int i;
if (groups) for(i=0;groups[i];i++) rhizome_manifest_add_group(m,groups[i]);
if (rhizome_manifest_finalise(m,signP)) {
return WHY("Failed to finalise manifest.\n");
}
/* Write manifest back to disk */
if (rhizome_write_manifest_file(m,manifestname)) {
rhizome_manifest_free(m);
return WHY("Could not write manifest file.");
}
/* Okay, it is written, and can be put directly into the rhizome database now */
int r=rhizome_store_bundle(m,filename);
if (!r) {
// XXX For testing unlink(manifestname);
unlink(filename);
return 0;
}
return WHY("rhizome_store_bundle() failed.");
}
/*
Store the specified manifest into the sqlite database.
We assume that sufficient space has been made for us.
The manifest should be finalised, and so we don't need to
look at the underlying manifest file, but can just write m->manifest_data
as a blob.
associated_filename needs to be read in and stored as a blob. Hopefully that
can be done in pieces so that we don't have memory exhaustion issues on small
architectures. However, we do know it's hash apriori from m, and so we can
skip loading the file in if it is already stored. mmap() apparently works on
Linux FAT file systems, and is probably the best choice since it doesn't need
all pages to be in RAM at the same time.
SQLite does allow modifying of blobs once stored in the database.
The trick is to insert the blob as all zeroes using a special function, and then
substitute bytes in the blog progressively.
We need to also need to create the appropriate row(s) in the MANIFESTS, FILES,
FILEMANIFESTS and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well.
*/
int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename)
{
char sqlcmd[1024];
const char *cmdtail;
char *manifestid=rhizome_bytes_to_hex(m->cryptoSignPublic,crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
if (!m->finalised) return WHY("Manifest was not finalised");
/* remove any old version of the manifest */
if (sqlite_exec_int64("SELECT COUNT(*) FROM MANIFESTS WHERE id='%s';",manifestid)>0)
{
/* Manifest already exists.
Remove old manifest entry, and replace with new one.
But we do need to check if the file referenced by the old one is still needed,
and if it's priority is right */
sqlite_exec_int64("DELETE FROM MANIFESTS WHERE id='%s';",manifestid);
char sql[1024];
sqlite3_stmt *statement;
snprintf(sql,1024,"SELECT fileid from filemanifests where manifestid='%s';",
manifestid);
if (sqlite3_prepare_v2(rhizome_db,sql,strlen(sql)+1,&statement,NULL)!=SQLITE_OK)
{
WHY("sqlite3_prepare_v2() failed");
WHY(sql);
WHY(sqlite3_errmsg(rhizome_db));
}
else
{
while ( sqlite3_step(statement)== SQLITE_ROW)
{
const unsigned char *fileid;
if (sqlite3_column_type(statement,0)==SQLITE_TEXT) {
fileid=sqlite3_column_text(statement,0);
rhizome_update_file_priority((char *)fileid);
}
}
sqlite3_finalize(statement);
}
sqlite_exec_int64("DELETE FROM FILEMANIFESTS WHERE manifestid='%s';",manifestid);
}
/* Store manifest */
WHY("*** Writing into manifests table");
snprintf(sqlcmd,1024,
"INSERT INTO MANIFESTS(id,manifest,version,inserttime,bar) VALUES('%s',?,%lld,%lld,?);",
manifestid,m->version,overlay_gettime_ms());
if (m->haveSecret) {
if (rhizome_store_keypair_bytes(m->cryptoSignPublic,m->cryptoSignSecret))
{
WHY("*** Insert into manifests failed (-1).");
return WHY("Failed to store key pair.");
}
} else {
/* We don't have the secret for this manifest, so only allow updates if
the self-signature is valid */
if (!m->selfSigned) {
WHY("*** Insert into manifests failed (-2).");
return WHY("Manifest is not signed, and I don't have the key. Manifest might be forged or corrupt.");
}
}
sqlite3_stmt *statement;
if (sqlite3_prepare_v2(rhizome_db,sqlcmd,strlen(sqlcmd)+1,&statement,&cmdtail)
!= SQLITE_OK) {
sqlite3_finalize(statement);
WHY("*** Insert into manifests failed.");
return WHY(sqlite3_errmsg(rhizome_db));
}
/* Bind manifest data to data field */
if (sqlite3_bind_blob(statement,1,m->manifestdata,m->manifest_bytes,SQLITE_TRANSIENT)!=SQLITE_OK)
{
sqlite3_finalize(statement);
WHY("*** Insert into manifests failed (2).");
return WHY(sqlite3_errmsg(rhizome_db));
}
/* Bind BAR to data field */
unsigned char bar[RHIZOME_BAR_BYTES];
rhizome_manifest_to_bar(m,bar);
if (sqlite3_bind_blob(statement,2,bar,RHIZOME_BAR_BYTES,SQLITE_TRANSIENT)
!=SQLITE_OK)
{
sqlite3_finalize(statement);
WHY("*** Insert into manifests failed (3).");
return WHY(sqlite3_errmsg(rhizome_db));
}
if (rhizome_finish_sqlstatement(statement)) {
WHY("*** Insert into manifests failed (4).");
return WHY("SQLite3 failed to insert row for manifest");
}
else
WHY("*** Insert into manifests apparently worked.");
/* Create relationship between file and manifest */
long long r=sqlite_exec_int64("INSERT INTO FILEMANIFESTS(manifestid,fileid) VALUES('%s','%s');",
manifestid,
m->fileHexHash);
if (r<0) {
WHY(sqlite3_errmsg(rhizome_db));
return WHY("SQLite3 failed to insert row in filemanifests.");
}
/* Create relationships to groups */
if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) {
/* This manifest is a group, so add entry to group list.
Created group is not automatically subscribed to, however. */
int closed=rhizome_manifest_get_ll(m,"closedgroup");
if (closed<1) closed=0;
int ciphered=rhizome_manifest_get_ll(m,"cipheredgroup");
if (ciphered<1) ciphered=0;
sqlite_exec_int64("delete from grouplist where id='%s';",manifestid);
int storedP
=sqlite_exec_int64("insert into grouplist(id,closed,ciphered,priority) VALUES('%s',%d,%d,%d);",
manifestid,closed,ciphered,RHIZOME_PRIORITY_DEFAULT);
if (storedP<0) return WHY("Failed to insert group manifest into grouplist table.");
}
{
int g;
int dud=0;
for(g=0;g<m->group_count;g++)
{
if (sqlite_exec_int64("INSERT INTO GROUPMEMBERSHIPS(manifestid,groupid) VALUES('%s','%s');",
manifestid, m->groups[g])<0)
dud++;
}
if (dud>0) return WHY("Failed to create one or more group associations");
}
/* Store the file */
if (m->fileLength>0)
if (rhizome_store_file(associated_filename,m->fileHexHash,m->fileHighestPriority))
return WHY("Could not store associated file");
/* Get things consistent */
sqlite3_exec(rhizome_db,"COMMIT;",NULL,NULL,NULL);
return 0;
}
int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
{
if (debug & DEBUG_RHIZOME)
DEBUGF("rhizome_add_manifest(m_in=%p, ttl=%d)",m_in, ttl);
if (m_in->finalised==0)
return WHY("Manifest must be finalised before being stored");
/* Store time to live, clamped to within legal range */
m_in->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
if (rhizome_manifest_check_sanity(m_in))
return WHY("Sanity checks on manifest failed");
if (rhizome_manifest_check_file(m_in))
return WHY("File does not belong to this manifest");
/* Get manifest version number. */
m_in->version = rhizome_manifest_get_ll(m_in, "version");
if (m_in->version==-1)
return WHY("Manifest must have a version number");
/* Supply manifest version number if missing, so we can do the version check below */
if (m_in->version == -1) {
m_in->version = gettime_ms();
rhizome_manifest_set_ll(m_in, "version", m_in->version);
}
/* If the manifest already has an ID */
char id[SID_STRLEN + 1];
if (rhizome_manifest_get(m_in, "id", id, SID_STRLEN + 1)) {
str_toupper_inplace(id);
/* Discard the new manifest unless it is newer than the most recent known version with the same ID */
long long storedversion = -1;
switch (sqlite_exec_int64(&storedversion, "SELECT version from manifests where id='%s';", id)) {
case -1:
return WHY("Select failed");
case 0:
if (debug & DEBUG_RHIZOME) DEBUG("No existing manifest");
break;
case 1:
if (debug & DEBUG_RHIZOME) DEBUGF("Found existing version=%lld, new version=%lld", storedversion, m_in->version);
if (m_in->version < storedversion)
return WHY("Newer version exists");
if (m_in->version == storedversion)
return WHY("Same version of manifest exists, not adding");
break;
default:
return WHY("Select found too many rows!");
}
} else {
/* no manifest ID */
return WHY("Manifest does not have an ID");
}
/* Okay, it is written, and can be put directly into the rhizome database now */
if (rhizome_store_bundle(m_in) == -1)
return WHY("rhizome_store_bundle() failed.");
// This message used in tests; do not modify or remove.
const char *service = rhizome_manifest_get(m_in, "service", NULL, 0);
INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%lld",
service ? service : "NULL",
alloca_tohex_sid(m_in->cryptoSignPublic),
m_in->version
);
monitor_announce_bundle(m_in);
return 0;
}
void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
{
DEBUGF("Dispatch size_high=%lld",r->cursor->size_high);
rhizome_direct_transport_state_http *state = r->transport_specific_state;
int sock=socket(AF_INET, SOCK_STREAM, 0);
if (sock==-1) {
WHY_perror("socket");
goto end;
}
struct hostent *hostent;
hostent = gethostbyname(state->host);
if (!hostent) {
DEBUGF("could not resolve hostname");
goto end;
}
struct sockaddr_in addr;
addr.sin_family = AF_INET;
addr.sin_port = htons(state->port);
addr.sin_addr = *((struct in_addr *)hostent->h_addr);
bzero(&(addr.sin_zero),8);
if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) {
WHY_perror("connect");
close(sock);
goto end;
}
char boundary[20];
char buffer[8192];
strbuf bb = strbuf_local(boundary, sizeof boundary);
strbuf_sprintf(bb, "%08lx%08lx", random(), random());
assert(!strbuf_overrun(bb));
strbuf content_preamble = strbuf_alloca(200);
strbuf content_postamble = strbuf_alloca(40);
strbuf_sprintf(content_preamble,
"--%s\r\n"
"Content-Disposition: form-data; name=\"data\"; filename=\"IHAVEs\"\r\n"
"Content-Type: application/octet-stream\r\n"
"\r\n",
boundary
);
strbuf_sprintf(content_postamble, "\r\n--%s--\r\n", boundary);
assert(!strbuf_overrun(content_preamble));
assert(!strbuf_overrun(content_postamble));
int content_length = strbuf_len(content_preamble)
+ r->cursor->buffer_offset_bytes
+ r->cursor->buffer_used
+ strbuf_len(content_postamble);
strbuf request = strbuf_local(buffer, sizeof buffer);
strbuf_sprintf(request,
"POST /rhizome/enquiry HTTP/1.0\r\n"
"Content-Length: %d\r\n"
"Content-Type: multipart/form-data; boundary=%s\r\n"
"\r\n%s",
content_length, boundary, strbuf_str(content_preamble)
);
assert(!strbuf_overrun(request));
/* TODO: Refactor this code so that it uses our asynchronous framework.
*/
int len = strbuf_len(request);
int sent=0;
while(sent<len) {
DEBUGF("write(%d, %s, %d)", sock, alloca_toprint(-1, &buffer[sent], len-sent), len-sent);
int count=write(sock,&buffer[sent],len-sent);
if (count == -1) {
if (errno==EPIPE) goto rx;
WHYF_perror("write(%d)", len - sent);
close(sock);
goto end;
}
sent+=count;
}
len=r->cursor->buffer_offset_bytes+r->cursor->buffer_used;
sent=0;
while(sent<len) {
int count=write(sock,&r->cursor->buffer[sent],len-sent);
if (count == -1) {
if (errno == EPIPE)
goto rx;
WHYF_perror("write(%d)", count);
close(sock);
goto end;
}
sent+=count;
}
strbuf_reset(request);
strbuf_puts(request, strbuf_str(content_postamble));
len = strbuf_len(request);
sent=0;
while(sent<len) {
DEBUGF("write(%d, %s, %d)", sock, alloca_toprint(-1, &buffer[sent], len-sent), len-sent);
int count=write(sock,&buffer[sent],len-sent);
if (count == -1) {
if (errno==EPIPE) goto rx;
WHYF_perror("write(%d)", len - sent);
close(sock);
goto end;
}
sent+=count;
}
struct http_response_parts parts;
rx:
/* request sent, now get response back. */
if (receive_http_response(sock, buffer, sizeof buffer, &parts) == -1) {
close(sock);
goto end;
}
/* For some reason the response data gets overwritten during a push,
so we need to copy it, and use the copy instead. */
unsigned char *actionlist=alloca(parts.content_length);
bcopy(parts.content_start, actionlist, parts.content_length);
dump("response", actionlist, parts.content_length);
/* We now have the list of (1+RHIZOME_BAR_PREFIX_BYTES)-byte records that indicate
the list of BAR prefixes that differ between the two nodes. We can now action
those which are relevant, i.e., based on whether we are pushing, pulling or
synchronising (both).
I am currently undecided as to whether it is cleaner to have some general
rhizome direct function for doing that, or whether it just adds unnecessary
complication, and the responses should just be handled in here.
For now, I am just going to implement it in here, and we can generalise later.
*/
int i;
for(i=10;i<content_length;i+=(1+RHIZOME_BAR_PREFIX_BYTES))
{
int type=actionlist[i];
unsigned long long
bid_prefix_ll=rhizome_bar_bidprefix_ll((unsigned char *)&actionlist[i+1]);
DEBUGF("%s %016llx* @ 0x%x",type==1?"push":"pull",bid_prefix_ll,i);
if (type==2&&r->pullP) {
/* Need to fetch manifest. Once we have the manifest, then we can
use our normal bundle fetch routines from rhizome_fetch.c
Generate a request like: GET /rhizome/manifestbybar/<hex of bar>
and add it to our list of HTTP fetch requests, then watch
until the request is finished. That will give us the manifest.
Then as noted above, we can use that to pull the file down using
existing routines.
*/
if (!rhizome_fetch_request_manifest_by_prefix
(&addr,&actionlist[i+1],RHIZOME_BAR_PREFIX_BYTES,
1 /* import, getting file if needed */))
{
/* Fetching the manifest, and then using it to see if we want to
fetch the file for import is all handled asynchronously, so just
wait for it to finish. */
while(rhizome_file_fetch_queue_count) fd_poll();
}
} else if (type==1&&r->pushP) {
/* Form up the POST request to submit the appropriate bundle. */
/* Start by getting the manifest, which is the main thing we need, and also
gives us the information we need for sending any associated file. */
rhizome_manifest
*m=rhizome_direct_get_manifest(&actionlist[i+1],
RHIZOME_BAR_PREFIX_BYTES);
if (!m) {
WHY("This should never happen. The manifest exists, but when I went looking for it, it doesn't appear to be there.");
goto next_item;
}
/* Get filehash and size from manifest if present */
const char *id = rhizome_manifest_get(m, "id", NULL, 0);
DEBUGF("bundle id = '%s'",id);
const char *hash = rhizome_manifest_get(m, "filehash", NULL, 0);
DEBUGF("bundle file hash = '%s'",hash);
long long filesize = rhizome_manifest_get_ll(m, "filesize");
DEBUGF("file size = %lld",filesize);
/* We now have everything we need to compose the POST request and send it.
*/
char *template="POST /rhizome/import HTTP/1.0\r\n"
