int SLNFilterWriteURIs(SLNFilterRef const filter, SLNSessionRef const session, SLNFilterPosition *const pos, bool const meta, uint64_t const max, bool const wait, SLNFilterWriteCB const writecb, void *ctx) {
uint64_t remaining = max;
for(;;) {
ssize_t const count = SLNFilterWriteURIBatch(filter, session, pos, meta, remaining, writecb, ctx);
if(count < 0) return count;
remaining -= count;
if(!remaining) return 0;
if(!count) break;
}
if(!wait || pos->dir < 0) return 0;
SLNRepoRef const repo = SLNSessionGetRepo(session);
for(;;) {
uint64_t const timeout = uv_now(async_loop)+(1000 * 30);
int rc = SLNRepoSubmissionWait(repo, pos->sortID, timeout);
if(UV_ETIMEDOUT == rc) {
uv_buf_t const parts[] = { uv_buf_init((char *)STR_LEN("\r\n")) };
rc = writecb(ctx, parts, numberof(parts));
if(rc < 0) break;
continue;
}
assert(rc >= 0); // TODO: Handle cancellation?
for(;;) {
ssize_t const count = SLNFilterWriteURIBatch(filter, session, pos, meta, remaining, writecb, ctx);
if(count < 0) return count;
remaining -= count;
if(!remaining) return 0;
}
}
return 0;
}
int SLNFilterWriteURIs(SLNFilterRef const filter, SLNSessionRef const session, SLNFilterPosition *const pos, bool const meta, uint64_t const max, bool const wait, SLNFilterWriteCB const writecb, SLNFilterFlushCB const flushcb, void *ctx) {
uint64_t remaining = max;
for(;;) {
ssize_t const count = SLNFilterWriteURIBatch(filter, session, pos, meta, remaining, writecb, ctx);
if(count < 0) return count;
remaining -= count;
if(!remaining) return 0;
if(!count) break;
}
if(!wait || pos->dir < 0) return 0;
SLNRepoRef const repo = SLNSessionGetRepo(session);
for(;;) {
int rc = flushcb ? flushcb(ctx) : 0;
if(rc < 0) return rc;
uint64_t latest = pos->sortID;
uint64_t const timeout = uv_now(async_loop)+(1000 * 30);
rc = SLNRepoSubmissionWait(repo, &latest, timeout);
if(UV_ETIMEDOUT == rc) {
uv_buf_t const parts[] = { uv_buf_init((char *)STR_LEN("\r\n")) };
rc = writecb(ctx, parts, numberof(parts));
if(rc < 0) break;
continue;
}
assert(rc >= 0); // TODO: Handle cancellation?
for(;;) {
ssize_t const count = SLNFilterWriteURIBatch(filter, session, pos, meta, remaining, writecb, ctx);
if(count < 0) return count;
remaining -= count;
if(!remaining) return 0;
if(count < BATCH_SIZE) break;
}
// This is how far we scanned, even if we didn't find anything.
if(pos->sortID < latest) {
pos->sortID = latest;
pos->fileID = 0;
}
}
return 0;
}
static void SLNFilterResultsWrite(SLNSessionRef const session, SLNFilterRef const filter, SLNFilterOpts *const opts, HTTPConnectionRef const conn) {
// TODO: Accept count and use it for the total number of results.
opts->count = 0;
// We're sending a series of batches, so reversing one batch
// doesn't make sense.
opts->outdir = opts->dir;
static strarg_t const fields[] = { "wait" };
str_t *values[numberof(fields)] = {};
QSValuesParse(qs, values, fields, numberof(fields));
bool const wait = parse_wait(values[0]);
QSValuesCleanup(values, numberof(values));
// I'm aware that we're abusing HTTP for sending real-time push data.
// I'd also like to support WebSocket at some point, but this is simpler
// and frankly probably more widely supported.
// Note that the protocol doesn't really break even if this data is
// cached. It DOES break if a proxy tries to buffer the whole response
// before passing it back to the client. I'd be curious to know whether
// such proxies still exist in 2015.
HTTPConnectionWriteResponse(conn, 200, "OK");
HTTPConnectionWriteHeader(conn, "Transfer-Encoding", "chunked");
HTTPConnectionWriteHeader(conn,
"Content-Type", "text/uri-list; charset=utf-8");
HTTPConnectionWriteHeader(conn, "Cache-Control", "no-store");
HTTPConnectionWriteHeader(conn, "Vary", "*");
HTTPConnectionBeginBody(conn);
int rc;
for(;;) {
rc = sendURIBatch(session, filter, opts, conn);
if(DB_NOTFOUND == rc) break;
if(DB_SUCCESS == rc) continue;
fprintf(stderr, "Query error: %s\n", db_strerror(rc));
goto cleanup;
}
if(!wait || opts->dir < 0) goto cleanup;
SLNRepoRef const repo = SLNSessionGetRepo(session);
for(;;) {
uint64_t const timeout = uv_now(async_loop)+(1000 * 30);
rc = SLNRepoSubmissionWait(repo, opts->sortID, timeout);
if(UV_ETIMEDOUT == rc) {
uv_buf_t const parts[] = { uv_buf_init((char *)STR_LEN("\r\n")) };
rc = HTTPConnectionWriteChunkv(conn, parts, numberof(parts));
if(rc < 0) break;
continue;
}
assert(rc >= 0); // TODO: Handle cancellation?
for(;;) {
rc = sendURIBatch(session, filter, opts, conn);
if(DB_NOTFOUND == rc) break;
if(DB_SUCCESS == rc) continue;
fprintf(stderr, "Query error: %s\n", db_strerror(rc));
goto cleanup;
}
}
cleanup:
HTTPConnectionWriteChunkEnd(conn);
HTTPConnectionEnd(conn);
SLNFilterOptsCleanup(opts);
}
