static void reader(SLNPullRef const pull) {
HTTPConnectionRef conn = NULL;
int rc;
for(;;) {
if(pull->stop) goto stop;
str_t URI[URI_MAX];
async_mutex_lock(pull->connlock);
rc = HTTPConnectionReadBodyLine(pull->conn, URI, sizeof(URI));
if(rc < 0) {
for(;;) {
if(pull->stop) break;
if(reconnect(pull) >= 0) break;
if(pull->stop) break;
async_sleep(1000 * 5);
}
async_mutex_unlock(pull->connlock);
continue;
}
if('#' == URI[0]) { // Comment line.
async_mutex_unlock(pull->connlock);
continue;
}
async_mutex_lock(pull->mutex);
while(pull->count + 1 > QUEUE_SIZE) {
async_cond_wait(pull->cond, pull->mutex);
if(pull->stop) {
async_mutex_unlock(pull->mutex);
async_mutex_unlock(pull->connlock);
goto stop;
}
}
size_t pos = (pull->cur + pull->count) % QUEUE_SIZE;
pull->count += 1;
async_mutex_unlock(pull->mutex);
async_mutex_unlock(pull->connlock);
for(;;) {
if(import(pull, URI, pos, &conn) >= 0) break;
if(pull->stop) goto stop;
async_sleep(1000 * 5);
}
}
stop:
HTTPConnectionFree(&conn);
async_mutex_lock(pull->mutex);
assertf(pull->stop, "Reader ended early");
assert(pull->tasks > 0);
pull->tasks--;
async_cond_broadcast(pull->cond);
async_mutex_unlock(pull->mutex);
}
static int reconnect(SLNPullRef const pull) {
int rc;
HTTPConnectionFree(&pull->conn);
rc = HTTPConnectionCreateOutgoing(pull->host, 0, &pull->conn);
if(rc < 0) {
alogf("Pull couldn't connect to %s (%s)\n", pull->host, sln_strerror(rc));
return rc;
}
// str_t path[URI_MAX];
// str_t *query_encoded = NULL;
// if(pull->query) query_encoded = QSEscape(pull->query, strlen(pull->query), true);
// snprintf(path, sizeof(path), "/sln/query-obsolete?q=%s", query_encoded ?: "");
// FREE(&query_encoded);
HTTPConnectionWriteRequest(pull->conn, HTTP_GET, "/sln/all", pull->host);
// TODO
// - New API /sln/query and /sln/metafiles
// - Pagination ?start=[last URI seen]
// - Error handling
// - Query string formatter
HTTPConnectionWriteHeader(pull->conn, "Cookie", pull->cookie);
HTTPConnectionBeginBody(pull->conn);
rc = HTTPConnectionEnd(pull->conn);
if(rc < 0) {
alogf("Pull couldn't connect to %s (%s)\n", pull->host, sln_strerror(rc));
return rc;
}
int const status = HTTPConnectionReadResponseStatus(pull->conn);
if(status < 0) {
alogf("Pull connection error: %s\n", sln_strerror(status));
return status;
}
if(403 == status) {
alogf("Pull connection authentication failed\n");
return UV_EACCES;
}
if(status < 200 || status >= 300) {
alogf("Pull connection error: %d\n", status);
return UV_EPROTO;
}
// TODO: All this does is scan past the headers.
// We don't actually use them...
HTTPHeadersRef headers;
rc = HTTPHeadersCreateFromConnection(pull->conn, &headers);
assert(rc >= 0); // TODO
HTTPHeadersFree(&headers);
/* rc = HTTPConnectionReadHeaders(pull->conn, NULL, NULL, 0);
if(rc < 0) {
alogf("Pull connection error %s\n", sln_strerror(rc));
return rc;
}*/
return 0;
}
int HTTPConnectionCreateIncoming(uv_stream_t *const socket, unsigned const flags, HTTPConnectionRef *const out) {
HTTPConnectionRef conn = calloc(1, sizeof(struct HTTPConnection));
if(!conn) return UV_ENOMEM;
int rc = uv_tcp_init(async_loop, conn->stream);
if(rc < 0) goto cleanup;
rc = uv_accept(socket, (uv_stream_t *)conn->stream);
if(rc < 0) goto cleanup;
http_parser_init(conn->parser, HTTP_REQUEST);
conn->parser->data = conn;
*out = conn; conn = NULL;
cleanup:
HTTPConnectionFree(&conn);
return rc;
}
void SLNPullStop(SLNPullRef const pull) {
if(!pull) return;
if(pull->stop) return;
async_mutex_lock(pull->mutex);
pull->stop = true;
async_cond_broadcast(pull->cond);
while(pull->tasks > 0) {
async_cond_wait(pull->cond, pull->mutex);
}
async_mutex_unlock(pull->mutex);
HTTPConnectionFree(&pull->conn);
for(size_t i = 0; i < QUEUE_SIZE; ++i) {
SLNSubmissionFree(&pull->queue[i]);
pull->filled[i] = false;
}
pull->cur = 0;
pull->count = 0;
}
static int import(SLNPullRef const pull, strarg_t const URI, size_t const pos, HTTPConnectionRef *const conn) {
if(!pull) return 0;
// TODO: Even if there's nothing to do, we have to enqueue something to fill up our reserved slots. I guess it's better than doing a lot of work inside the connection lock, but there's got to be a better way.
SLNSubmissionRef sub = NULL;
HTTPHeadersRef headers = NULL;
if(!URI) goto enqueue;
str_t algo[SLN_ALGO_SIZE];
str_t hash[SLN_HASH_SIZE];
if(SLNParseURI(URI, algo, hash) < 0) goto enqueue;
int rc = SLNSessionGetFileInfo(pull->session, URI, NULL);
if(rc >= 0) goto enqueue;
db_assertf(DB_NOTFOUND == rc, "Database error: %s", sln_strerror(rc));
// TODO: We're logging out of order when we do it like this...
// alogf("Pulling %s\n", URI);
if(!*conn) {
rc = HTTPConnectionCreateOutgoing(pull->host, 0, conn);
if(rc < 0) {
alogf("Pull import connection error: %s\n", sln_strerror(rc));
goto fail;
}
}
str_t *path = aasprintf("/sln/file/%s/%s", algo, hash);
if(!path) {
alogf("Pull aasprintf error\n");
goto fail;
}
rc = HTTPConnectionWriteRequest(*conn, HTTP_GET, path, pull->host);
assert(rc >= 0); // TODO
FREE(&path);
HTTPConnectionWriteHeader(*conn, "Cookie", pull->cookie);
HTTPConnectionBeginBody(*conn);
rc = HTTPConnectionEnd(*conn);
if(rc < 0) {
alogf("Pull import request error: %s\n", sln_strerror(rc));
goto fail;
}
int const status = HTTPConnectionReadResponseStatus(*conn);
if(status < 0) {
alogf("Pull import response error: %s\n", sln_strerror(status));
goto fail;
}
if(status < 200 || status >= 300) {
alogf("Pull import status error: %d\n", status);
goto fail;
}
rc = HTTPHeadersCreateFromConnection(*conn, &headers);
assert(rc >= 0); // TODO
/* if(rc < 0) {
alogf("Pull import headers error %s\n", sln_strerror(rc));
goto fail;
}*/
strarg_t const type = HTTPHeadersGet(headers, "content-type");
rc = SLNSubmissionCreate(pull->session, URI, &sub);
if(rc < 0) {
alogf("Pull submission error: %s\n", sln_strerror(rc));
goto fail;
}
rc = SLNSubmissionSetType(sub, type);
if(rc < 0) {
alogf("Pull submission type error: %s\n", sln_strerror(rc));
goto fail;
}
for(;;) {
if(pull->stop) goto fail;
uv_buf_t buf[1] = {};
rc = HTTPConnectionReadBody(*conn, buf);
if(rc < 0) {
alogf("Pull download error: %s\n", sln_strerror(rc));
goto fail;
}
if(0 == buf->len) break;
rc = SLNSubmissionWrite(sub, (byte_t *)buf->base, buf->len);
if(rc < 0) {
alogf("Pull write error\n");
goto fail;
}
}
rc = SLNSubmissionEnd(sub);
if(rc < 0) {
alogf("Pull submission error: %s\n", sln_strerror(rc));
goto fail;
}
enqueue:
HTTPHeadersFree(&headers);
async_mutex_lock(pull->mutex);
pull->queue[pos] = sub; sub = NULL;
pull->filled[pos] = true;
async_cond_broadcast(pull->cond);
async_mutex_unlock(pull->mutex);
return 0;
fail:
HTTPHeadersFree(&headers);
SLNSubmissionFree(&sub);
HTTPConnectionFree(conn);
return -1;
}
int HTTPServerListen(HTTPServerRef const server, strarg_t const address, strarg_t const port) {
if(!server) return 0;
assertf(!server->socket->data, "HTTPServer already listening");
int rc;
rc = uv_tcp_init(async_loop, server->socket);
if(rc < 0) return rc;
server->socket->data = server;
struct addrinfo const hints = {
.ai_flags = AI_V4MAPPED | AI_ADDRCONFIG | AI_NUMERICSERV | AI_PASSIVE,
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM,
.ai_protocol = 0, // ???
};
struct addrinfo *info;
rc = async_getaddrinfo(address, port, &hints, &info);
if(rc < 0) {
HTTPServerClose(server);
return rc;
}
int bound = 0;
rc = 0;
for(struct addrinfo *each = info; each; each = each->ai_next) {
rc = uv_tcp_bind(server->socket, each->ai_addr, 0);
if(rc >= 0) bound++;
}
uv_freeaddrinfo(info);
if(!bound) {
HTTPServerClose(server);
if(rc < 0) return rc;
return UV_EADDRNOTAVAIL;
}
rc = uv_listen((uv_stream_t *)server->socket, 511, connection_cb);
if(rc < 0) {
HTTPServerClose(server);
return rc;
}
return 0;
}
int HTTPServerListenSecure(HTTPServerRef const server, strarg_t const address, strarg_t const port, struct tls **const tlsptr) {
if(!server) return 0;
int rc = HTTPServerListen(server, address, port);
if(rc < 0) return rc;
server->secure = *tlsptr; *tlsptr = NULL;
return 0;
}
void HTTPServerClose(HTTPServerRef const server) {
if(!server) return;
if(!server->socket->data) return;
if(server->secure) tls_close(server->secure);
tls_free(server->secure); server->secure = NULL;
async_close((uv_handle_t *)server->socket);
}
static void connection(uv_stream_t *const socket) {
HTTPServerRef const server = socket->data;
HTTPConnectionRef conn;
int rc = HTTPConnectionCreateIncomingSecure(socket, server->secure, 0, &conn);
if(rc < 0) {
fprintf(stderr, "HTTP server connection error %s\n", uv_strerror(rc));
return;
}
assert(conn);
for(;;) {
server->listener(server->context, server, conn);
rc = HTTPConnectionDrainMessage(conn);
if(rc < 0) break;
}
HTTPConnectionFree(&conn);
}
int HTTPConnectionCreateOutgoing(strarg_t const domain, unsigned const flags, HTTPConnectionRef *const out) {
str_t host[1023+1];
str_t service[15+1];
host[0] = '\0';
service[0] = '\0';
int matched = sscanf(domain, "%1023[^:]:%15[0-9]", host, service);
if(matched < 1) return UV_EINVAL;
if('\0' == host[0]) return UV_EINVAL;
static struct addrinfo const hints = {
.ai_flags = AI_V4MAPPED | AI_ADDRCONFIG | AI_NUMERICSERV,
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM,
.ai_protocol = 0, // ???
};
struct addrinfo *info = NULL;
HTTPConnectionRef conn = NULL;
int rc;
rc = async_getaddrinfo(host, service[0] ? service : "80", &hints, &info);
if(rc < 0) goto cleanup;
conn = calloc(1, sizeof(struct HTTPConnection));
if(!conn) rc = UV_ENOMEM;
if(rc < 0) goto cleanup;
rc = UV_EADDRNOTAVAIL;
for(struct addrinfo *each = info; each; each = each->ai_next) {
rc = uv_tcp_init(async_loop, conn->stream);
if(rc < 0) break;
rc = async_tcp_connect(conn->stream, each->ai_addr);
if(rc >= 0) break;
async_close((uv_handle_t *)conn->stream);
}
if(rc < 0) goto cleanup;
http_parser_init(conn->parser, HTTP_RESPONSE);
conn->parser->data = conn;
*out = conn; conn = NULL;
cleanup:
uv_freeaddrinfo(info); info = NULL;
HTTPConnectionFree(&conn);
return rc;
}
void HTTPConnectionFree(HTTPConnectionRef *const connptr) {
HTTPConnectionRef conn = *connptr;
if(!conn) return;
async_close((uv_handle_t *)conn->stream);
// http_parser does not need to be freed, closed or destroyed.
memset(conn->parser, 0, sizeof(*conn->parser));
FREE(&conn->buf);
*conn->raw = uv_buf_init(NULL, 0);
conn->type = HTTPNothing;
*conn->out = uv_buf_init(NULL, 0);
conn->flags = 0;
assert_zeroed(conn, 1);
FREE(connptr); conn = NULL;
}
