long long rhizome_database_used_bytes()
{
long long db_page_size=sqlite_exec_int64("PRAGMA page_size;");
long long db_page_count=sqlite_exec_int64("PRAGMA page_count;");
long long db_free_page_count=sqlite_exec_int64("PRAGMA free_count;");
return db_page_size*(db_page_count-db_free_page_count);
}
int rhizome_update_file_priority(char *fileid)
{
/* Drop if no references */
int referrers=sqlite_exec_int64("SELECT COUNT(*) FROM FILEMANIFESTS WHERE fileid='%s';",fileid);
if (referrers==0)
rhizome_drop_stored_file(fileid,RHIZOME_PRIORITY_HIGHEST+1);
if (referrers>0) {
/* It has referrers, so workout the highest priority of any referrer */
int highestPriority=sqlite_exec_int64("SELECT max(grouplist.priority) FROM MANIFESTS,FILEMANIFESTS,GROUPMEMBERSHIPS,GROUPLIST where manifests.id=filemanifests.manifestid AND groupmemberships.manifestid=manifests.id AND groupmemberships.groupid=grouplist.id AND filemanifests.fileid='%s';",fileid);
if (highestPriority>=0)
sqlite_exec_int64("UPDATE files set highestPriority=%d WHERE id='%s';",
highestPriority,fileid);
}
return 0;
}
int rhizome_manifest_check_file(rhizome_manifest *m_in)
{
long long gotfile = 0;
if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE ID='%s' and datavalid=1;", m_in->fileHexHash) != 1) {
WHYF("Failed to count files");
return 0;
}
if (gotfile) {
DEBUGF("Skipping file checks for bundle, as file is already in the database");
return 0;
}
/* Find out whether the payload is expected to be encrypted or not */
m_in->payloadEncryption=rhizome_manifest_get_ll(m_in, "crypt");
/* Check payload file is accessible and discover its length, then check that it matches
the file size stored in the manifest */
long long mfilesize = rhizome_manifest_get_ll(m_in, "filesize");
m_in->fileLength = 0;
if (m_in->dataFileName[0]) {
struct stat stat;
if (lstat(m_in->dataFileName,&stat) == -1) {
if (errno != ENOENT || mfilesize != 0)
return WHYF_perror("stat(%s)", m_in->dataFileName);
} else {
m_in->fileLength = stat.st_size;
}
}
if (debug & DEBUG_RHIZOME)
DEBUGF("filename=%s, fileLength=%lld", m_in->dataFileName, m_in->fileLength);
if (mfilesize != -1 && mfilesize != m_in->fileLength) {
WHYF("Manifest.filesize (%lld) != actual file size (%lld)", mfilesize, m_in->fileLength);
return -1;
}
/* If payload is empty, ensure manifest has not file hash, otherwis compute the hash of the
payload and check that it matches manifest. */
const char *mhexhash = rhizome_manifest_get(m_in, "filehash", NULL, 0);
if (m_in->fileLength != 0) {
char hexhashbuf[RHIZOME_FILEHASH_STRLEN + 1];
if (rhizome_hash_file(m_in,m_in->dataFileName, hexhashbuf))
return WHY("Could not hash file.");
memcpy(&m_in->fileHexHash[0], &hexhashbuf[0], sizeof hexhashbuf);
m_in->fileHashedP = 1;
if (!mhexhash) return WHY("manifest contains no file hash");
if (mhexhash && strcmp(m_in->fileHexHash, mhexhash)) {
WHYF("Manifest.filehash (%s) does not match payload hash (%s)", mhexhash, m_in->fileHexHash);
return -1;
}
} else {
if (mhexhash != NULL) {
WHYF("Manifest.filehash (%s) should be absent for empty payload", mhexhash);
return -1;
}
}
return 0;
}
int rhizome_make_space(int group_priority, long long bytes)
{
sqlite3_stmt *statement;
/* Asked for impossibly large amount */
if (bytes>=(rhizome_space-65536)) return -1;
long long db_used=rhizome_database_used_bytes();
/* If there is already enough space now, then do nothing more */
if (db_used<=(rhizome_space-bytes-65536)) return 0;
/* Okay, not enough space, so free up some. */
char sql[1024];
snprintf(sql,1024,"select id,length from files where highestpriority<%d order by descending length",group_priority);
if(sqlite3_prepare_v2(rhizome_db,sql, -1, &statement, NULL) != SQLITE_OK )
{
fprintf(stderr,"SQLite error running query '%s': %s\n",sql,sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
sqlite3_close(rhizome_db);
rhizome_db=NULL;
exit(-1);
}
while ( bytes>(rhizome_space-65536-rhizome_database_used_bytes()) && sqlite3_step(statement) == SQLITE_ROW)
{
/* Make sure we can drop this blob, and if so drop it, and recalculate number of bytes required */
const unsigned char *id;
long long length;
/* Get values */
if (sqlite3_column_type(statement, 0)==SQLITE_TEXT) id=sqlite3_column_text(statement, 0);
else {
fprintf(stderr,"Incorrect type in id column of files table.\n");
continue; }
if (sqlite3_column_type(statement, 1)==SQLITE_INTEGER) length=sqlite3_column_int(statement, 1);
else {
fprintf(stderr,"Incorrect type in length column of files table.\n");
continue; }
/* Try to drop this file from storage, discarding any references that do not trump the priority of this
request. The query done earlier should ensure this, but it doesn't hurt to be paranoid, and it also
protects against inconsistency in the database. */
rhizome_drop_stored_file((char *)id,group_priority+1);
}
sqlite3_finalize(statement);
long long equal_priority_larger_file_space_used = sqlite_exec_int64("SELECT COUNT(length) FROM FILES WHERE highestpriority=%d and length>%lld",group_priority,bytes);
/* XXX Get rid of any equal priority files that are larger than this one */
/* XXX Get rid of any higher priority files that are not relevant in this time or location */
/* Couldn't make space */
return WHY("Incomplete");
}
/* Drop the specified file from storage, and any manifests that reference it,
provided that none of those manifests are being retained at a higher priority
than the maximum specified here. */
int rhizome_drop_stored_file(char *id,int maximum_priority)
{
char sql[1024];
sqlite3_stmt *statement;
int cannot_drop=0;
if (strlen(id)>70) return -1;
snprintf(sql,1024,"select manifests.id from manifests,filemanifests where manifests.id==filemanifests.manifestid and filemanifests.fileid='%s'",
id);
if(sqlite3_prepare_v2(rhizome_db,sql, -1, &statement, NULL) != SQLITE_OK )
{
fprintf(stderr,"SQLite error running query '%s': %s\n",sql,sqlite3_errmsg(rhizome_db));
sqlite3_finalize(statement);
sqlite3_close(rhizome_db);
rhizome_db=NULL;
return WHY("Could not drop stored file");
}
while ( sqlite3_step(statement) == SQLITE_ROW)
{
/* Find manifests for this file */
const unsigned char *id;
if (sqlite3_column_type(statement, 0)==SQLITE_TEXT) id=sqlite3_column_text(statement, 0);
else {
fprintf(stderr,"Incorrect type in id column of manifests table.\n");
continue; }
/* Check that manifest is not part of a higher priority group.
If so, we cannot drop the manifest or the file.
However, we will keep iterating, as we can still drop any other manifests pointing to this file
that are lower priority, and thus free up a little space. */
if (rhizome_manifest_priority((char *)id)>maximum_priority) {
cannot_drop=1;
} else {
printf("removing stale filemanifests, manifests, groupmemberships\n");
sqlite_exec_int64("delete from filemanifests where manifestid='%s';",id);
sqlite_exec_int64("delete from manifests where manifestid='%s';",id);
sqlite_exec_int64("delete from keypairs where public='%s';",id);
sqlite_exec_int64("delete from groupmemberships where manifestid='%s';",id);
}
}
sqlite3_finalize(statement);
if (!cannot_drop) {
printf("cleaning up filemanifests, manifests\n");
sqlite_exec_int64("delete from filemanifests where fileid='%s';",id);
sqlite_exec_int64("delete from files where id='%s';",id);
}
return 0;
}
/* Import a bundle from the inbox folder.
Check that the manifest prototype is valid, and if so, complete it, and sign it if required and possible.
Note that bundles can either be an ordinary bundle, or a group description.
Group specifications are simply manifests that have the "isagroup" variable set.
Groups get stored in the manifests table AND a reference included in the
grouplist table.
Groups are allowed to be listed as being members of other groups.
This allows a nested, i.e., multi-level group heirarchy where sub-groups will only
typically be discovered by joining the parent group. Probably not a bad way to do
things.
The file should be included in the specified rhizome groups, if possible.
(some groups may be closed groups that we do not have the private key for.)
*/
int rhizome_bundle_import(char *bundle,char *groups[], int ttl,
int verifyP, int checkFileP, int signP)
{
char filename[1024];
char manifestname[1024];
char *buffer;
snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path,bundle); filename[1023]=0;
snprintf(manifestname,1024,"%s/import/manifest.%s",rhizome_datastore_path,bundle); manifestname[1023]=0;
/* Open files */
rhizome_manifest *m=rhizome_read_manifest_file(manifestname,0 /* file not buffer */,
RHIZOME_VERIFY);
if (!m) return WHY("Could not read manifest file.");
char hexhash[SHA512_DIGEST_STRING_LENGTH];
/* work out time to live */
if (ttl<0) ttl=0; if (ttl>254) ttl=254; m->ttl=ttl;
/* Keep associated file name handy for later */
m->dataFileName=strdup(filename);
struct stat stat;
if (lstat(filename,&stat)) {
return WHY("Could not stat() associated file");
m->fileLength=stat.st_size;
}
if (checkFileP||signP) {
if (rhizome_hash_file(filename,hexhash))
{ rhizome_manifest_free(m); return WHY("Could not hash file."); }
bcopy(&hexhash[0],&m->fileHexHash[0],SHA512_DIGEST_STRING_LENGTH);
m->fileHashedP=1;
}
if (verifyP)
{
/* Make sure hashes match.
Make sure that no signature verification errors were spotted on loading. */
int verifyErrors=0;
char *mhexhash;
if (checkFileP) {
if ((mhexhash=rhizome_manifest_get(m,"filehash",NULL,0))!=NULL)
if (strcmp(hexhash,mhexhash)) verifyErrors++; }
if (m->errors) verifyErrors+=m->errors;
if (verifyErrors) {
rhizome_manifest_free(m);
unlink(manifestname);
unlink(filename);
return WHY("Errors encountered verifying bundle manifest");
}
}
if (!verifyP) {
if ((buffer=rhizome_manifest_get(m,"id",NULL,0))!=NULL) {
/* No bundle id (256 bit random string being a public key in the NaCl CryptoSign crypto system),
so create one, and keep the private key handy. */
printf("manifest does not have an id\n");
rhizome_manifest_createid(m);
/* The ID is implicit in transit, but we need to store it in the file,
so that reimporting manifests on receiver nodes works easily.
We might implement something that strips the id variable out of the
manifest when sending it, or some other scheme to avoid sending all
the extra bytes. */
rhizome_manifest_set(m,"id",rhizome_bytes_to_hex(m->cryptoSignPublic,crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES));
} else {
/* An ID was specified, so remember it, and look for the private key if
we have it stowed away */
rhizome_hex_to_bytes(buffer,m->cryptoSignPublic,
crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2);
if (!rhizome_find_keypair_bytes(m->cryptoSignPublic,m->cryptoSignSecret))
m->haveSecret=1;
}
rhizome_manifest_set(m,"filehash",hexhash);
if (rhizome_manifest_get(m,"version",NULL,0)==NULL)
/* Version not set, so set one */
rhizome_manifest_set_ll(m,"version",overlay_gettime_ms());
rhizome_manifest_set_ll(m,"first_byte",0);
rhizome_manifest_set_ll(m,"last_byte",rhizome_file_size(filename));
}
/* Discard if it is older than the most recent known version */
long long storedversion = sqlite_exec_int64("SELECT version from manifests where id='%s';",rhizome_bytes_to_hex(m->cryptoSignPublic,crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES));
if (storedversion>rhizome_manifest_get_ll(m,"version"))
{
rhizome_manifest_free(m);
return WHY("Newer version exists");
}
/* Add group memberships */
int i;
if (groups) for(i=0;groups[i];i++) rhizome_manifest_add_group(m,groups[i]);
if (rhizome_manifest_finalise(m,signP)) {
return WHY("Failed to finalise manifest.\n");
}
/* Write manifest back to disk */
if (rhizome_write_manifest_file(m,manifestname)) {
rhizome_manifest_free(m);
return WHY("Could not write manifest file.");
}
/* Okay, it is written, and can be put directly into the rhizome database now */
int r=rhizome_store_bundle(m,filename);
if (!r) {
// XXX For testing unlink(manifestname);
unlink(filename);
return 0;
}
return WHY("rhizome_store_bundle() failed.");
}
static int rhizome_server_parse_http_request(rhizome_http_request *r)
{
/* Switching to writing, so update the call-back */
r->alarm.poll.events=POLLOUT;
watch(&r->alarm);
// Start building up a response.
r->request_type = 0;
// Parse the HTTP "GET" line.
char *path = NULL;
size_t pathlen = 0;
if (str_startswith(r->request, "GET ", &path)) {
char *p;
// This loop is guaranteed to terminate before the end of the buffer, because we know that the
// buffer contains at least "\n\n" and maybe "\r\n\r\n" at the end of the header block.
for (p = path; !isspace(*p); ++p)
;
pathlen = p - path;
if ( str_startswith(p, " HTTP/1.", &p)
&& (str_startswith(p, "0", &p) || str_startswith(p, "1", &p))
&& (str_startswith(p, "\r\n", &p) || str_startswith(p, "\n", &p))
)
path[pathlen] = '\0';
else
path = NULL;
}
if (path) {
char *id = NULL;
INFOF("RHIZOME HTTP SERVER, GET %s", alloca_toprint(1024, path, pathlen));
if (strcmp(path, "/favicon.ico") == 0) {
r->request_type = RHIZOME_HTTP_REQUEST_FAVICON;
rhizome_server_http_response_header(r, 200, "image/vnd.microsoft.icon", favicon_len);
} else if (strcmp(path, "/rhizome/groups") == 0) {
/* Return the list of known groups */
rhizome_server_sql_query_http_response(r, "id", "groups", "from groups", 32, 1);
} else if (strcmp(path, "/rhizome/files") == 0) {
/* Return the list of known files */
rhizome_server_sql_query_http_response(r, "id", "files", "from files", 32, 1);
} else if (strcmp(path, "/rhizome/bars") == 0) {
/* Return the list of known BARs */
rhizome_server_sql_query_http_response(r, "bar", "manifests", "from manifests", 32, 0);
} else if (str_startswith(path, "/rhizome/file/", &id)) {
/* Stream the specified payload */
if (!rhizome_str_is_file_hash(id)) {
rhizome_server_simple_http_response(r, 400, "<html><h1>Invalid payload ID</h1></html>\r\n");
} else {
// TODO: Check for Range: header and return 206 if returning partial content
str_toupper_inplace(id);
long long rowid = -1;
sqlite_exec_int64(&rowid, "select rowid from files where id='%s';", id);
if (rowid >= 0 && sqlite3_blob_open(rhizome_db, "main", "files", "data", rowid, 0, &r->blob) != SQLITE_OK)
rowid = -1;
if (rowid == -1) {
rhizome_server_simple_http_response(r, 404, "<html><h1>Payload not found</h1></html>\r\n");
} else {
r->source_index = 0;
r->blob_end = sqlite3_blob_bytes(r->blob);
rhizome_server_http_response_header(r, 200, "application/binary", r->blob_end - r->source_index);
r->request_type |= RHIZOME_HTTP_REQUEST_BLOB;
}
}
} else if (str_startswith(path, "/rhizome/manifest/", &id)) {
// TODO: Stream the specified manifest
rhizome_server_simple_http_response(r, 500, "<html><h1>Not implemented</h1></html>\r\n");
} else {
rhizome_server_simple_http_response(r, 404, "<html><h1>Not found</h1></html>\r\n");
}
} else {
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("Received malformed HTTP request: %s", alloca_toprint(120, (const char *)r->request, r->request_length));
rhizome_server_simple_http_response(r, 400, "<html><h1>Malformed request</h1></html>\r\n");
}
/* Try sending data immediately. */
rhizome_server_http_send_bytes(r);
return 0;
}
static int rhizome_server_sql_query_http_response(rhizome_http_request *r,
char *column,char *table,char *query_body,
int bytes_per_row,int dehexP)
{
/* Run the provided SQL query progressively and return the values of the first
column it returns. As the result list may be very long, we will add the
LIMIT <skip>,<count> clause to do it piece by piece.
Otherwise, the response is prefixed by a 256 byte header, including the public
key of the sending node, and allowing space for information about encryption of
the body, although encryption is not yet implemented here.
*/
if (r->buffer == NULL || r->buffer_size < 16384) {
if (r->buffer)
free(r->buffer);
r->buffer_size = 16384;
r->buffer = malloc(r->buffer_size);
if (r->buffer == NULL) {
r->buffer_size = 0;
WHY_perror("malloc");
return WHY("Cannot send response, out of memory");
}
}
r->buffer_length=0;
r->buffer_offset=0;
r->source_record_size=bytes_per_row;
r->source_count = 0;
sqlite_exec_int64(&r->source_count, "SELECT COUNT(*) %s", query_body);
/* Work out total response length */
long long response_bytes=256+r->source_count*r->source_record_size;
rhizome_server_http_response_header(r, 200, "servalproject.org/rhizome-list", response_bytes);
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("headers consumed %d bytes", r->buffer_length);
/* Clear and prepare response header */
bzero(&r->buffer[r->buffer_length],256);
r->buffer[r->buffer_length]=0x01; /* type of response (list) */
r->buffer[r->buffer_length+1]=0x01; /* version of response */
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("Found %lld records",r->source_count);
/* Number of records we intend to return */
r->buffer[r->buffer_length+4]=(r->source_count>>0)&0xff;
r->buffer[r->buffer_length+5]=(r->source_count>>8)&0xff;
r->buffer[r->buffer_length+6]=(r->source_count>>16)&0xff;
r->buffer[r->buffer_length+7]=(r->source_count>>24)&0xff;
r->buffer_length+=256;
/* copy our public key in to bytes 32+ */
// TODO get out public key (SID) from keyring and copy into response packet
/* build templated query */
strbuf b = strbuf_local(r->source, sizeof r->source);
strbuf_sprintf(b, "SELECT %s,rowid %s", column, query_body);
if (strbuf_overrun(b))
WHYF("SQL query overrun: %s", strbuf_str(b));
r->source_index=0;
r->source_flags=dehexP;
DEBUGF("buffer_length=%d",r->buffer_length);
/* Populate spare space in buffer with rows of data */
return rhizome_server_sql_query_fill_buffer(r, table, column);
}
int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
{
int pass;
int bytes=e->sizeLimit-e->length;
int overhead=1+8+1+3+1+1+1; /* maximum overhead */
int slots=(bytes-overhead)/RHIZOME_BAR_BYTES;
if (slots>30) slots=30;
int slots_used=0;
int bytes_used=0;
int bytes_available=bytes-overhead-1 /* one byte held for expanding RFS */;
int bundles_advertised=0;
if (slots<1) return WHY("No room for node advertisements");
if (!rhizome_db) return WHY("Rhizome not enabled");
if (ob_append_byte(e,OF_TYPE_RHIZOME_ADVERT))
return WHY("could not add rhizome bundle advertisement header");
ob_append_byte(e,1); /* TTL */
int rfs_offset=e->length; /* remember where the RFS byte gets stored
so that we can patch it later */
ob_append_byte(e,1+8+1+1+1+RHIZOME_BAR_BYTES*slots_used/* RFS */);
/* Stuff in dummy address fields */
ob_append_byte(e,OA_CODE_BROADCAST);
{ int i; for(i=0;i<8;i++) ob_append_byte(e,random()&0xff); } /* BPI for broadcast */
ob_append_byte(e,OA_CODE_PREVIOUS);
ob_append_byte(e,OA_CODE_SELF);
/* Randomly choose whether to advertise manifests or BARs first. */
int skipmanifests=random()&1;
/* Version of rhizome advert block:
1 = manifests then BARs,
2 = BARs only */
ob_append_byte(e,1+skipmanifests);
/* XXX Should add priority bundles here.
XXX Should prioritise bundles for subscribed groups, Serval-authorised files
etc over common bundles.
XXX Should wait a while after going through bundle list so that we don't waste
CPU on db queries if there are not many bundles. Actually, we probably just
shouldn't be sending bundles blindly on every tick.
XXX How do we indicate group membership with BARs? Or do groups actively poll?
*/
if (debug&DEBUG_RHIZOME)
WHY("Group handling not completely thought out here yet.");
/* Get number of bundles available if required */
bundles_available=sqlite_exec_int64("SELECT COUNT(BAR) FROM MANIFESTS;");
if (bundles_available==-1||(bundle_offset[0]>=bundles_available))
bundle_offset[0]=0;
if (bundles_available==-1||(bundle_offset[1]>=bundles_available))
bundle_offset[1]=0;
for(pass=skipmanifests;pass<2;pass++)
{
sqlite3_stmt *statement;
char query[1024];
switch(pass) {
case 0: /* Full manifests */
snprintf(query,1024,"SELECT MANIFEST,ROWID FROM MANIFESTS LIMIT %d,%d",
bundle_offset[pass],slots);
break;
case 1: /* BARs */
snprintf(query,1024,"SELECT BAR,ROWID FROM MANIFESTS LIMIT %d,%d",
bundle_offset[pass],slots);
break;
}
switch (sqlite3_prepare_v2(rhizome_db,query,-1,&statement,NULL))
{
case SQLITE_OK: case SQLITE_DONE: case SQLITE_ROW:
break;
default:
sqlite3_finalize(statement);
sqlite3_close(rhizome_db);
rhizome_db=NULL;
WHY(query);
WHY(sqlite3_errmsg(rhizome_db));
return WHY("Could not prepare sql statement for fetching BARs for advertisement.");
}
while((bytes_used<bytes_available)&&(sqlite3_step(statement)==SQLITE_ROW)&&
(e->length+RHIZOME_BAR_BYTES<=e->sizeLimit))
{
sqlite3_blob *blob;
int column_type=sqlite3_column_type(statement, 0);
switch(column_type) {
case SQLITE_BLOB:
if (sqlite3_blob_open(rhizome_db,"main","manifests",
pass?"bar":"manifest",
sqlite3_column_int64(statement,1) /* rowid */,
0 /* read only */,&blob)!=SQLITE_OK)
{
WHY("Couldn't open blob");
continue;
}
int blob_bytes=sqlite3_blob_bytes(blob);
if (pass&&(blob_bytes!=RHIZOME_BAR_BYTES)) {
if (debug&DEBUG_RHIZOME)
fprintf(stderr,"Found a BAR that is the wrong size - ignoring\n");
continue;
}
/* Only include manifests that are <=1KB inline.
Longer ones are only advertised by BAR */
if (blob_bytes>1024) continue;
int overhead=0;
if (!pass) overhead=2;
if (ob_makespace(e,overhead+blob_bytes)) {
if (debug&DEBUG_RHIZOME)
fprintf(stderr,"Stopped cramming %s into Rhizome advertisement frame.\n",
pass?"BARs":"manifests");
break;
}
if (!pass) {
/* put manifest length field and manifest ID */
e->bytes[e->length]=(blob_bytes>>8)&0xff;
e->bytes[e->length+1]=(blob_bytes>>0)&0xff;
if (debug&DEBUG_RHIZOME)
fprintf(stderr,"length bytes written at offset 0x%x\n",e->length);
}
if (sqlite3_blob_read(blob,&e->bytes[e->length+overhead],blob_bytes,0)
!=SQLITE_OK) {
if (debug&DEBUG_RHIZOME) WHY("Couldn't read from blob");
sqlite3_blob_close(blob);
continue;
}
e->length+=overhead+blob_bytes;
bytes_used+=overhead+blob_bytes;
bundles_advertised++;
sqlite3_blob_close(blob);
}
}
sqlite3_finalize(statement);
if (!pass)
{
/* Mark end of whole manifests by writing 0xff, which is more than the MSB
of a manifest's length is allowed to be. */
ob_append_byte(e,0xff);
}
}
/* The following function just stores the file (or silently returns if it already
exists).
The relationships of manifests to this file are the responsibility of the
caller. */
int rhizome_store_file(char *file,char *hash,int priority) {
int fd=open(file,O_RDONLY);
if (fd<0) return WHY("Could not open associated file");
struct stat stat;
if (fstat(fd,&stat)) {
close(fd);
return WHY("Could not stat() associated file");
}
unsigned char *addr =
mmap(NULL, stat.st_size, PROT_READ, MAP_FILE|MAP_SHARED, fd, 0);
if (addr==MAP_FAILED) {
close(fd);
return WHY("mmap() of associated file failed.");
}
/* Get hash of file if not supplied */
char hexhash[SHA512_DIGEST_STRING_LENGTH];
if (!hash)
{
/* Hash the file */
SHA512_CTX c;
SHA512_Init(&c);
SHA512_Update(&c,addr,stat.st_size);
SHA512_End(&c,hexhash);
hash=hexhash;
}
/* INSERT INTO FILES(id as text, data blob, length integer, highestpriority integer).
BUT, we have to do this incrementally so that we can handle blobs larger than available memory.
This is possible using:
int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
That binds an all zeroes blob to a field. We can then populate the data by
opening a handle to the blob using:
int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset);
*/
char sqlcmd[1024];
const char *cmdtail;
/* See if the file is already stored, and if so, don't bother storing it again */
int count=sqlite_exec_int64("SELECT COUNT(*) FROM FILES WHERE id='%s' AND datavalid<>0;",hash);
if (count==1) {
/* File is already stored, so just update the highestPriority field if required. */
long long storedPriority = sqlite_exec_int64("SELECT highestPriority FROM FILES WHERE id='%s' AND datavalid!=0",hash);
if (storedPriority<priority)
{
snprintf(sqlcmd,1024,"UPDATE FILES SET highestPriority=%d WHERE id='%s';",
priority,hash);
if (sqlite3_exec(rhizome_db,sqlcmd,NULL,NULL,NULL)!=SQLITE_OK) {
close(fd);
WHY(sqlite3_errmsg(rhizome_db));
return WHY("SQLite failed to update highestPriority field for stored file.");
}
}
close(fd);
return 0;
} else if (count>1) {
/* This should never happen! */
return WHY("Duplicate records for a file in the rhizome database. Database probably corrupt.");
}
/* Okay, so there are no records that match, but we should delete any half-baked record (with datavalid=0) so that the insert below doesn't fail.
Don't worry about the return result, since it might not delete any records. */
sqlite3_exec(rhizome_db,"DELETE FROM FILES WHERE datavalid=0;",NULL,NULL,NULL);
snprintf(sqlcmd,1024,"INSERT INTO FILES(id,data,length,highestpriority,datavalid) VALUES('%s',?,%lld,%d,0);",
hash,(long long)stat.st_size,priority);
sqlite3_stmt *statement;
if (sqlite3_prepare_v2(rhizome_db,sqlcmd,strlen(sqlcmd)+1,&statement,&cmdtail)
!= SQLITE_OK)
{
close(fd);
sqlite3_finalize(statement);
return WHY(sqlite3_errmsg(rhizome_db));
}
/* Bind appropriate sized zero-filled blob to data field */
int dud=0;
int r;
if ((r=sqlite3_bind_zeroblob(statement,1,stat.st_size))!=SQLITE_OK)
{
dud++;
WHY("sqlite3_bind_zeroblob() failed");
WHY(sqlite3_errmsg(rhizome_db));
}
/* Do actual insert, and abort if it fails */
if (!dud)
switch(sqlite3_step(statement)) {
case SQLITE_OK: case SQLITE_ROW: case SQLITE_DONE:
break;
default:
dud++;
WHY("sqlite3_step() failed");
WHY(sqlite3_errmsg(rhizome_db));
}
if (sqlite3_finalize(statement)) dud++;
if (dud) {
close(fd);
if (sqlite3_finalize(statement)!=SQLITE_OK)
{
WHY("sqlite3_finalize() failed");
WHY(sqlite3_errmsg(rhizome_db));
}
return WHY("SQLite3 failed to insert row for file");
}
/* Get rowid for inserted row, so that we can modify the blob */
int rowid=sqlite3_last_insert_rowid(rhizome_db);
if (rowid<1) {
close(fd);
WHY(sqlite3_errmsg(rhizome_db));
return WHY("SQLite3 failed return rowid of inserted row");
}
sqlite3_blob *blob;
if (sqlite3_blob_open(rhizome_db,"main","FILES","data",rowid,
1 /* read/write */,
&blob) != SQLITE_OK)
{
WHY(sqlite3_errmsg(rhizome_db));
close(fd);
sqlite3_blob_close(blob);
return WHY("SQLite3 failed to open file blob for writing");
}
{
long long i;
for(i=0;i<stat.st_size;i+=65536)
{
int n=65536;
if (i+n>stat.st_size) n=stat.st_size-i;
if (sqlite3_blob_write(blob,&addr[i],n,i) !=SQLITE_OK) dud++;
}
}
close(fd);
sqlite3_blob_close(blob);
/* Mark file as up-to-date */
sqlite_exec_int64("UPDATE FILES SET datavalid=1 WHERE id='%s';",
hash);
if (dud) {
WHY(sqlite3_errmsg(rhizome_db));
return WHY("SQLite3 failed write all blob data");
}
printf("stored file\n");
return 0;
}
/*
Store the specified manifest into the sqlite database.
We assume that sufficient space has been made for us.
The manifest should be finalised, and so we don't need to
look at the underlying manifest file, but can just write m->manifest_data
as a blob.
associated_filename needs to be read in and stored as a blob. Hopefully that
can be done in pieces so that we don't have memory exhaustion issues on small
architectures. However, we do know it's hash apriori from m, and so we can
skip loading the file in if it is already stored. mmap() apparently works on
Linux FAT file systems, and is probably the best choice since it doesn't need
all pages to be in RAM at the same time.
SQLite does allow modifying of blobs once stored in the database.
The trick is to insert the blob as all zeroes using a special function, and then
substitute bytes in the blog progressively.
We need to also need to create the appropriate row(s) in the MANIFESTS, FILES,
FILEMANIFESTS and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well.
*/
int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename)
{
char sqlcmd[1024];
const char *cmdtail;
char *manifestid=rhizome_bytes_to_hex(m->cryptoSignPublic,crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
if (!m->finalised) return WHY("Manifest was not finalised");
/* remove any old version of the manifest */
if (sqlite_exec_int64("SELECT COUNT(*) FROM MANIFESTS WHERE id='%s';",manifestid)>0)
{
/* Manifest already exists.
Remove old manifest entry, and replace with new one.
But we do need to check if the file referenced by the old one is still needed,
and if it's priority is right */
sqlite_exec_int64("DELETE FROM MANIFESTS WHERE id='%s';",manifestid);
char sql[1024];
sqlite3_stmt *statement;
snprintf(sql,1024,"SELECT fileid from filemanifests where manifestid='%s';",
manifestid);
if (sqlite3_prepare_v2(rhizome_db,sql,strlen(sql)+1,&statement,NULL)!=SQLITE_OK)
{
WHY("sqlite3_prepare_v2() failed");
WHY(sql);
WHY(sqlite3_errmsg(rhizome_db));
}
else
{
while ( sqlite3_step(statement)== SQLITE_ROW)
{
const unsigned char *fileid;
if (sqlite3_column_type(statement,0)==SQLITE_TEXT) {
fileid=sqlite3_column_text(statement,0);
rhizome_update_file_priority((char *)fileid);
}
}
sqlite3_finalize(statement);
}
sqlite_exec_int64("DELETE FROM FILEMANIFESTS WHERE manifestid='%s';",manifestid);
}
/* Store manifest */
WHY("*** Writing into manifests table");
snprintf(sqlcmd,1024,
"INSERT INTO MANIFESTS(id,manifest,version,inserttime,bar) VALUES('%s',?,%lld,%lld,?);",
manifestid,m->version,overlay_gettime_ms());
if (m->haveSecret) {
if (rhizome_store_keypair_bytes(m->cryptoSignPublic,m->cryptoSignSecret))
{
WHY("*** Insert into manifests failed (-1).");
return WHY("Failed to store key pair.");
}
} else {
/* We don't have the secret for this manifest, so only allow updates if
the self-signature is valid */
if (!m->selfSigned) {
WHY("*** Insert into manifests failed (-2).");
return WHY("Manifest is not signed, and I don't have the key. Manifest might be forged or corrupt.");
}
}
sqlite3_stmt *statement;
if (sqlite3_prepare_v2(rhizome_db,sqlcmd,strlen(sqlcmd)+1,&statement,&cmdtail)
!= SQLITE_OK) {
sqlite3_finalize(statement);
WHY("*** Insert into manifests failed.");
return WHY(sqlite3_errmsg(rhizome_db));
}
/* Bind manifest data to data field */
if (sqlite3_bind_blob(statement,1,m->manifestdata,m->manifest_bytes,SQLITE_TRANSIENT)!=SQLITE_OK)
{
sqlite3_finalize(statement);
WHY("*** Insert into manifests failed (2).");
return WHY(sqlite3_errmsg(rhizome_db));
}
/* Bind BAR to data field */
unsigned char bar[RHIZOME_BAR_BYTES];
rhizome_manifest_to_bar(m,bar);
if (sqlite3_bind_blob(statement,2,bar,RHIZOME_BAR_BYTES,SQLITE_TRANSIENT)
!=SQLITE_OK)
{
sqlite3_finalize(statement);
WHY("*** Insert into manifests failed (3).");
return WHY(sqlite3_errmsg(rhizome_db));
}
if (rhizome_finish_sqlstatement(statement)) {
WHY("*** Insert into manifests failed (4).");
return WHY("SQLite3 failed to insert row for manifest");
}
else
WHY("*** Insert into manifests apparently worked.");
/* Create relationship between file and manifest */
long long r=sqlite_exec_int64("INSERT INTO FILEMANIFESTS(manifestid,fileid) VALUES('%s','%s');",
manifestid,
m->fileHexHash);
if (r<0) {
WHY(sqlite3_errmsg(rhizome_db));
return WHY("SQLite3 failed to insert row in filemanifests.");
}
/* Create relationships to groups */
if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) {
/* This manifest is a group, so add entry to group list.
Created group is not automatically subscribed to, however. */
int closed=rhizome_manifest_get_ll(m,"closedgroup");
if (closed<1) closed=0;
int ciphered=rhizome_manifest_get_ll(m,"cipheredgroup");
if (ciphered<1) ciphered=0;
sqlite_exec_int64("delete from grouplist where id='%s';",manifestid);
int storedP
=sqlite_exec_int64("insert into grouplist(id,closed,ciphered,priority) VALUES('%s',%d,%d,%d);",
manifestid,closed,ciphered,RHIZOME_PRIORITY_DEFAULT);
if (storedP<0) return WHY("Failed to insert group manifest into grouplist table.");
}
{
int g;
int dud=0;
for(g=0;g<m->group_count;g++)
{
if (sqlite_exec_int64("INSERT INTO GROUPMEMBERSHIPS(manifestid,groupid) VALUES('%s','%s');",
manifestid, m->groups[g])<0)
dud++;
}
if (dud>0) return WHY("Failed to create one or more group associations");
}
/* Store the file */
if (m->fileLength>0)
if (rhizome_store_file(associated_filename,m->fileHexHash,m->fileHighestPriority))
return WHY("Could not store associated file");
/* Get things consistent */
sqlite3_exec(rhizome_db,"COMMIT;",NULL,NULL,NULL);
return 0;
}
/* XXX Requires a messy join that might be slow. */
int rhizome_manifest_priority(char *id)
{
long long result = sqlite_exec_int64("select max(grouplist.priorty) from grouplist,manifests,groupmemberships where manifests.id='%s' and grouplist.id=groupmemberships.groupid and groupmemberships.manifestid=manifests.id;",id);
return result;
}
int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
{
if (debug & DEBUG_RHIZOME)
DEBUGF("rhizome_add_manifest(m_in=%p, ttl=%d)",m_in, ttl);
if (m_in->finalised==0)
return WHY("Manifest must be finalised before being stored");
/* Store time to live, clamped to within legal range */
m_in->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
if (rhizome_manifest_check_sanity(m_in))
return WHY("Sanity checks on manifest failed");
if (rhizome_manifest_check_file(m_in))
return WHY("File does not belong to this manifest");
/* Get manifest version number. */
m_in->version = rhizome_manifest_get_ll(m_in, "version");
if (m_in->version==-1)
return WHY("Manifest must have a version number");
/* Supply manifest version number if missing, so we can do the version check below */
if (m_in->version == -1) {
m_in->version = gettime_ms();
rhizome_manifest_set_ll(m_in, "version", m_in->version);
}
/* If the manifest already has an ID */
char id[SID_STRLEN + 1];
if (rhizome_manifest_get(m_in, "id", id, SID_STRLEN + 1)) {
str_toupper_inplace(id);
/* Discard the new manifest unless it is newer than the most recent known version with the same ID */
long long storedversion = -1;
switch (sqlite_exec_int64(&storedversion, "SELECT version from manifests where id='%s';", id)) {
case -1:
return WHY("Select failed");
case 0:
if (debug & DEBUG_RHIZOME) DEBUG("No existing manifest");
break;
case 1:
if (debug & DEBUG_RHIZOME) DEBUGF("Found existing version=%lld, new version=%lld", storedversion, m_in->version);
if (m_in->version < storedversion)
return WHY("Newer version exists");
if (m_in->version == storedversion)
return WHY("Same version of manifest exists, not adding");
break;
default:
return WHY("Select found too many rows!");
}
} else {
/* no manifest ID */
return WHY("Manifest does not have an ID");
}
/* Okay, it is written, and can be put directly into the rhizome database now */
if (rhizome_store_bundle(m_in) == -1)
return WHY("rhizome_store_bundle() failed.");
// This message used in tests; do not modify or remove.
const char *service = rhizome_manifest_get(m_in, "service", NULL, 0);
INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%lld",
service ? service : "NULL",
alloca_tohex_sid(m_in->cryptoSignPublic),
m_in->version
);
monitor_announce_bundle(m_in);
return 0;
}
