int SLNSubmissionStoreBatch(SLNSubmissionRef const *const list, size_t const count) {
if(!count) return 0;
// Session permissions were already checked when the sub was created.
SLNRepoRef const repo = SLNSessionGetRepo(list[0]->session);
DB_env *db = NULL;
SLNRepoDBOpen(repo, &db);
DB_txn *txn = NULL;
int rc = db_txn_begin(db, NULL, DB_RDWR, &txn);
if(rc < 0) {
SLNRepoDBClose(repo, &db);
return rc;
}
uint64_t sortID = 0;
rc = DB_NOTFOUND;
for(size_t i = 0; i < count; i++) {
if(!list[i]) continue;
assert(repo == SLNSessionGetRepo(list[i]->session));
rc = SLNSubmissionStore(list[i], txn);
if(rc < 0) break;
uint64_t const metaFileID = list[i]->metaFileID;
if(metaFileID > sortID) sortID = metaFileID;
}
if(rc >= 0) {
rc = db_txn_commit(txn); txn = NULL;
} else {
db_txn_abort(txn); txn = NULL;
}
SLNRepoDBClose(repo, &db);
if(rc >= 0) SLNRepoSubmissionEmit(repo, sortID);
return rc;
}
int SLNSessionGetFileInfo(SLNSessionRef const session, strarg_t const URI, SLNFileInfo *const info) {
DB_env *db = NULL;
int rc = SLNSessionDBOpen(session, SLN_RDONLY, &db);
if(rc < 0) return rc;
DB_txn *txn = NULL;
rc = db_txn_begin(db, NULL, DB_RDONLY, &txn);
if(rc < 0) {
SLNSessionDBClose(session, &db);
return rc;
}
DB_cursor *cursor;
rc = db_txn_cursor(txn, &cursor);
assert(!rc);
DB_range fileIDs[1];
SLNURIAndFileIDRange1(fileIDs, txn, URI);
DB_val URIAndFileID_key[1];
rc = db_cursor_firstr(cursor, fileIDs, URIAndFileID_key, NULL, +1);
DB_val file_val[1];
if(rc >= 0) {
strarg_t URI2;
uint64_t fileID;
SLNURIAndFileIDKeyUnpack(URIAndFileID_key, txn, &URI2, &fileID);
assert(0 == strcmp(URI, URI2));
if(info) {
DB_val fileID_key[1];
SLNFileByIDKeyPack(fileID_key, txn, fileID);
rc = db_get(txn, fileID_key, file_val);
}
}
if(rc < 0) {
db_txn_abort(txn); txn = NULL;
SLNSessionDBClose(session, &db);
return rc;
}
if(info) {
// Clear padding for later assert_zeroed.
memset(info, 0, sizeof(*info));
strarg_t const internalHash = db_read_string(file_val, txn);
strarg_t const type = db_read_string(file_val, txn);
uint64_t const size = db_read_uint64(file_val);
info->hash = strdup(internalHash);
info->path = SLNRepoCopyInternalPath(SLNSessionGetRepo(session), internalHash);
info->type = strdup(type);
info->size = size;
if(!info->hash || !info->path || !info->type) {
SLNFileInfoCleanup(info);
db_txn_abort(txn); txn = NULL;
SLNSessionDBClose(session, &db);
return DB_ENOMEM;
}
}
db_txn_abort(txn); txn = NULL;
SLNSessionDBClose(session, &db);
return 0;
}
int SLNSubmissionCreate(SLNSessionRef const session, strarg_t const knownURI, SLNSubmissionRef *const out) {
assert(out);
if(!SLNSessionHasPermission(session, SLN_WRONLY)) return UV_EACCES;
SLNSubmissionRef sub = calloc(1, sizeof(struct SLNSubmission));
if(!sub) return UV_ENOMEM;
int rc = 0;
sub->session = session;
if(knownURI) {
sub->knownURI = strdup(knownURI);
if(!sub->knownURI) rc = UV_ENOMEM;
if(rc < 0) goto cleanup;
}
sub->type = NULL;
sub->tmppath = SLNRepoCopyTempPath(SLNSessionGetRepo(session));
if(!sub->tmppath) rc = UV_ENOMEM;
if(rc < 0) goto cleanup;
rc = async_fs_open_mkdirp(sub->tmppath, O_CREAT | O_EXCL | O_RDWR, 0400);
if(rc < 0) goto cleanup;
sub->tmpfile = rc;
sub->hasher = SLNHasherCreate(sub->type);
if(!sub->hasher) rc = UV_ENOMEM;
if(rc < 0) goto cleanup;
sub->metaFileID = 0;
*out = sub; sub = NULL;
cleanup:
SLNSubmissionFree(&sub);
return rc;
}
int SLNFilterWriteURIs(SLNFilterRef const filter, SLNSessionRef const session, SLNFilterPosition *const pos, bool const meta, uint64_t const max, bool const wait, SLNFilterWriteCB const writecb, void *ctx) {
uint64_t remaining = max;
for(;;) {
ssize_t const count = SLNFilterWriteURIBatch(filter, session, pos, meta, remaining, writecb, ctx);
if(count < 0) return count;
remaining -= count;
if(!remaining) return 0;
if(!count) break;
}
if(!wait || pos->dir < 0) return 0;
SLNRepoRef const repo = SLNSessionGetRepo(session);
for(;;) {
uint64_t const timeout = uv_now(async_loop)+(1000 * 30);
int rc = SLNRepoSubmissionWait(repo, pos->sortID, timeout);
if(UV_ETIMEDOUT == rc) {
uv_buf_t const parts[] = { uv_buf_init((char *)STR_LEN("\r\n")) };
rc = writecb(ctx, parts, numberof(parts));
if(rc < 0) break;
continue;
}
assert(rc >= 0); // TODO: Handle cancellation?
for(;;) {
ssize_t const count = SLNFilterWriteURIBatch(filter, session, pos, meta, remaining, writecb, ctx);
if(count < 0) return count;
remaining -= count;
if(!remaining) return 0;
}
}
return 0;
}
int SLNSessionGetValueForField(SLNSessionRef const session, str_t value[], size_t const max, strarg_t const fileURI, strarg_t const field) {
if(!SLNSessionHasPermission(session, SLN_RDONLY)) return DB_EACCES;
if(!field) return DB_EINVAL;
if(max) value[0] = '\0';
int rc = 0;
DB_cursor *metafiles = NULL;
DB_cursor *values = NULL;
SLNRepoRef const repo = SLNSessionGetRepo(session);
DB_env *db = NULL;
SLNRepoDBOpen(repo, &db);
DB_txn *txn = NULL;
rc = db_txn_begin(db, NULL, DB_RDONLY, &txn);
if(rc < 0) goto done;
rc = db_cursor_open(txn, &metafiles);
if(rc < 0) goto done;
rc = db_cursor_open(txn, &values);
if(rc < 0) goto done;
DB_range metaFileIDs[1];
SLNTargetURIAndMetaFileIDRange1(metaFileIDs, txn, fileURI);
DB_val metaFileID_key[1];
rc = db_cursor_firstr(metafiles, metaFileIDs, metaFileID_key, NULL, +1);
if(rc < 0 && DB_NOTFOUND != rc) goto done;
for(; rc >= 0; rc = db_cursor_nextr(metafiles, metaFileIDs, metaFileID_key, NULL, +1)) {
strarg_t u;
uint64_t metaFileID;
SLNTargetURIAndMetaFileIDKeyUnpack(metaFileID_key, txn, &u, &metaFileID);
assert(0 == strcmp(fileURI, u));
DB_range vrange[1];
SLNMetaFileIDFieldAndValueRange2(vrange, txn, metaFileID, field);
DB_val value_val[1];
rc = db_cursor_firstr(values, vrange, value_val, NULL, +1);
if(rc < 0 && DB_NOTFOUND != rc) goto done;
for(; rc >= 0; rc = db_cursor_nextr(values, vrange, value_val, NULL, +1)) {
uint64_t m;
strarg_t f, v;
SLNMetaFileIDFieldAndValueKeyUnpack(value_val, txn, &m, &f, &v);
assert(metaFileID == m);
assert(0 == strcmp(field, f));
if(!v) continue;
if(0 == strcmp("", v)) continue;
size_t const len = strlen(v);
memcpy(value, v, MIN(len, max-1));
value[MIN(len, max-1)] = '\0';
goto done;
}
}
done:
db_cursor_close(values); values = NULL;
db_cursor_close(metafiles); metafiles = NULL;
db_txn_abort(txn); txn = NULL;
SLNRepoDBClose(repo, &db);
return rc;
}
ssize_t SLNFilterCopyURIs(SLNFilterRef const filter, SLNSessionRef const session, SLNFilterPosition *const pos, int const dir, bool const meta, str_t *URIs[], size_t const max) {
assert(URIs);
if(!SLNSessionHasPermission(session, SLN_RDONLY)) return DB_EACCES;
if(0 == pos->dir) return DB_EINVAL;
if(0 == dir) return DB_EINVAL;
if(0 == max) return 0;
DB_env *db = NULL;
DB_txn *txn = NULL;
ssize_t rc = 0;
SLNRepoRef const repo = SLNSessionGetRepo(session);
SLNRepoDBOpen(repo, &db);
rc = db_txn_begin(db, NULL, DB_RDONLY, &txn);
if(rc < 0) goto cleanup;
rc = SLNFilterPrepare(filter, txn);
if(rc < 0) goto cleanup;
rc = SLNFilterSeekToPosition(filter, pos, txn);
if(rc < 0) goto cleanup;
int const stepdir = pos->dir * dir;
size_t i = 0;
for(; i < max; i++) {
size_t const x = stepdir > 0 ? i : max-1-i;
rc = SLNFilterGetPosition(filter, pos, txn);
if(DB_NOTFOUND == rc) {
rc = 0;
break;
}
rc = SLNFilterCopyURI(filter, pos->fileID, meta, txn, &URIs[x]);
if(rc < 0) goto cleanup;
assert(URIs[x]);
SLNFilterStep(filter, pos->dir);
}
// The results should always be in the first `i` slots, even when
// filling them in reverse order.
if(stepdir < 0) {
memmove(URIs+0, URIs+(max-i), sizeof(*URIs) * i);
}
assert(rc >= 0);
rc = i;
cleanup:
db_txn_abort(txn); txn = NULL;
SLNRepoDBClose(repo, &db);
return rc;
}
int SLNSubmissionGetFileInfo(SLNSubmissionRef const sub, SLNFileInfo *const info) {
if(!sub) return UV_EINVAL;
if(!sub->internalHash) return UV_EINVAL;
SLNRepoRef const repo = SLNSessionGetRepo(sub->session);
info->hash = strdup(sub->internalHash);
info->path = SLNRepoCopyInternalPath(repo, sub->internalHash);
info->type = strdup(sub->type);
info->size = sub->size;
if(!info->hash || !info->path || !info->type) {
SLNFileInfoCleanup(info);
return UV_ENOMEM;
}
return 0;
}
int SLNFilterWriteURIs(SLNFilterRef const filter, SLNSessionRef const session, SLNFilterPosition *const pos, bool const meta, uint64_t const max, bool const wait, SLNFilterWriteCB const writecb, SLNFilterFlushCB const flushcb, void *ctx) {
uint64_t remaining = max;
for(;;) {
ssize_t const count = SLNFilterWriteURIBatch(filter, session, pos, meta, remaining, writecb, ctx);
if(count < 0) return count;
remaining -= count;
if(!remaining) return 0;
if(!count) break;
}
if(!wait || pos->dir < 0) return 0;
SLNRepoRef const repo = SLNSessionGetRepo(session);
for(;;) {
int rc = flushcb ? flushcb(ctx) : 0;
if(rc < 0) return rc;
uint64_t latest = pos->sortID;
uint64_t const timeout = uv_now(async_loop)+(1000 * 30);
rc = SLNRepoSubmissionWait(repo, &latest, timeout);
if(UV_ETIMEDOUT == rc) {
uv_buf_t const parts[] = { uv_buf_init((char *)STR_LEN("\r\n")) };
rc = writecb(ctx, parts, numberof(parts));
if(rc < 0) break;
continue;
}
assert(rc >= 0); // TODO: Handle cancellation?
for(;;) {
ssize_t const count = SLNFilterWriteURIBatch(filter, session, pos, meta, remaining, writecb, ctx);
if(count < 0) return count;
remaining -= count;
if(!remaining) return 0;
if(count < BATCH_SIZE) break;
}
// This is how far we scanned, even if we didn't find anything.
if(pos->sortID < latest) {
pos->sortID = latest;
pos->fileID = 0;
}
}
return 0;
}
int SLNSessionCreateUser(SLNSessionRef const session, DB_txn *const txn, strarg_t const username, strarg_t const password) {
SLNRepoRef const repo = SLNSessionGetRepo(session);
SLNMode const mode = SLNRepoGetRegistrationMode(repo);
return SLNSessionCreateUserInternal(session, txn, username, password, mode);
}
void SLNSessionDBClose(SLNSessionRef const session, DB_env **const dbptr) {
assert(session);
SLNRepoDBClose(SLNSessionGetRepo(session), dbptr);
}
int SLNSessionDBOpen(SLNSessionRef const session, SLNMode const mode, DB_env **const dbptr) {
if(!SLNSessionHasPermission(session, mode)) return DB_EACCES;
assert(session);
SLNRepoDBOpenUnsafe(SLNSessionGetRepo(session), dbptr);
return 0;
}
SLNRepoRef SLNSubmissionGetRepo(SLNSubmissionRef const sub) {
if(!sub) return NULL;
return SLNSessionGetRepo(sub->session);
}
static void SLNFilterResultsWrite(SLNSessionRef const session, SLNFilterRef const filter, SLNFilterOpts *const opts, HTTPConnectionRef const conn) {
// TODO: Accept count and use it for the total number of results.
opts->count = 0;
// We're sending a series of batches, so reversing one batch
// doesn't make sense.
opts->outdir = opts->dir;
static strarg_t const fields[] = { "wait" };
str_t *values[numberof(fields)] = {};
QSValuesParse(qs, values, fields, numberof(fields));
bool const wait = parse_wait(values[0]);
QSValuesCleanup(values, numberof(values));
// I'm aware that we're abusing HTTP for sending real-time push data.
// I'd also like to support WebSocket at some point, but this is simpler
// and frankly probably more widely supported.
// Note that the protocol doesn't really break even if this data is
// cached. It DOES break if a proxy tries to buffer the whole response
// before passing it back to the client. I'd be curious to know whether
// such proxies still exist in 2015.
HTTPConnectionWriteResponse(conn, 200, "OK");
HTTPConnectionWriteHeader(conn, "Transfer-Encoding", "chunked");
HTTPConnectionWriteHeader(conn,
"Content-Type", "text/uri-list; charset=utf-8");
HTTPConnectionWriteHeader(conn, "Cache-Control", "no-store");
HTTPConnectionWriteHeader(conn, "Vary", "*");
HTTPConnectionBeginBody(conn);
int rc;
for(;;) {
rc = sendURIBatch(session, filter, opts, conn);
if(DB_NOTFOUND == rc) break;
if(DB_SUCCESS == rc) continue;
fprintf(stderr, "Query error: %s\n", db_strerror(rc));
goto cleanup;
}
if(!wait || opts->dir < 0) goto cleanup;
SLNRepoRef const repo = SLNSessionGetRepo(session);
for(;;) {
uint64_t const timeout = uv_now(async_loop)+(1000 * 30);
rc = SLNRepoSubmissionWait(repo, opts->sortID, timeout);
if(UV_ETIMEDOUT == rc) {
uv_buf_t const parts[] = { uv_buf_init((char *)STR_LEN("\r\n")) };
rc = HTTPConnectionWriteChunkv(conn, parts, numberof(parts));
if(rc < 0) break;
continue;
}
assert(rc >= 0); // TODO: Handle cancellation?
for(;;) {
rc = sendURIBatch(session, filter, opts, conn);
if(DB_NOTFOUND == rc) break;
if(DB_SUCCESS == rc) continue;
fprintf(stderr, "Query error: %s\n", db_strerror(rc));
goto cleanup;
}
}
cleanup:
HTTPConnectionWriteChunkEnd(conn);
HTTPConnectionEnd(conn);
SLNFilterOptsCleanup(opts);
}
