static void reader(SLNPullRef const pull) { HTTPConnectionRef conn = NULL; int rc; for(;;) { if(pull->stop) goto stop; str_t URI[URI_MAX]; async_mutex_lock(pull->connlock); rc = HTTPConnectionReadBodyLine(pull->conn, URI, sizeof(URI)); if(rc < 0) { for(;;) { if(pull->stop) break; if(reconnect(pull) >= 0) break; if(pull->stop) break; async_sleep(1000 * 5); } async_mutex_unlock(pull->connlock); continue; } if('#' == URI[0]) { // Comment line. async_mutex_unlock(pull->connlock); continue; } async_mutex_lock(pull->mutex); while(pull->count + 1 > QUEUE_SIZE) { async_cond_wait(pull->cond, pull->mutex); if(pull->stop) { async_mutex_unlock(pull->mutex); async_mutex_unlock(pull->connlock); goto stop; } } size_t pos = (pull->cur + pull->count) % QUEUE_SIZE; pull->count += 1; async_mutex_unlock(pull->mutex); async_mutex_unlock(pull->connlock); for(;;) { if(import(pull, URI, pos, &conn) >= 0) break; if(pull->stop) goto stop; async_sleep(1000 * 5); } } stop: HTTPConnectionFree(&conn); async_mutex_lock(pull->mutex); assertf(pull->stop, "Reader ended early"); assert(pull->tasks > 0); pull->tasks--; async_cond_broadcast(pull->cond); async_mutex_unlock(pull->mutex); }
static int reconnect(SLNPullRef const pull) { int rc; HTTPConnectionFree(&pull->conn); rc = HTTPConnectionCreateOutgoing(pull->host, 0, &pull->conn); if(rc < 0) { alogf("Pull couldn't connect to %s (%s)\n", pull->host, sln_strerror(rc)); return rc; } // str_t path[URI_MAX]; // str_t *query_encoded = NULL; // if(pull->query) query_encoded = QSEscape(pull->query, strlen(pull->query), true); // snprintf(path, sizeof(path), "/sln/query-obsolete?q=%s", query_encoded ?: ""); // FREE(&query_encoded); HTTPConnectionWriteRequest(pull->conn, HTTP_GET, "/sln/all", pull->host); // TODO // - New API /sln/query and /sln/metafiles // - Pagination ?start=[last URI seen] // - Error handling // - Query string formatter HTTPConnectionWriteHeader(pull->conn, "Cookie", pull->cookie); HTTPConnectionBeginBody(pull->conn); rc = HTTPConnectionEnd(pull->conn); if(rc < 0) { alogf("Pull couldn't connect to %s (%s)\n", pull->host, sln_strerror(rc)); return rc; } int const status = HTTPConnectionReadResponseStatus(pull->conn); if(status < 0) { alogf("Pull connection error: %s\n", sln_strerror(status)); return status; } if(403 == status) { alogf("Pull connection authentication failed\n"); return UV_EACCES; } if(status < 200 || status >= 300) { alogf("Pull connection error: %d\n", status); return UV_EPROTO; } // TODO: All this does is scan past the headers. // We don't actually use them... HTTPHeadersRef headers; rc = HTTPHeadersCreateFromConnection(pull->conn, &headers); assert(rc >= 0); // TODO HTTPHeadersFree(&headers); /* rc = HTTPConnectionReadHeaders(pull->conn, NULL, NULL, 0); if(rc < 0) { alogf("Pull connection error %s\n", sln_strerror(rc)); return rc; }*/ return 0; }
int HTTPConnectionCreateIncoming(uv_stream_t *const socket, unsigned const flags, HTTPConnectionRef *const out) { HTTPConnectionRef conn = calloc(1, sizeof(struct HTTPConnection)); if(!conn) return UV_ENOMEM; int rc = uv_tcp_init(async_loop, conn->stream); if(rc < 0) goto cleanup; rc = uv_accept(socket, (uv_stream_t *)conn->stream); if(rc < 0) goto cleanup; http_parser_init(conn->parser, HTTP_REQUEST); conn->parser->data = conn; *out = conn; conn = NULL; cleanup: HTTPConnectionFree(&conn); return rc; }
void SLNPullStop(SLNPullRef const pull) { if(!pull) return; if(pull->stop) return; async_mutex_lock(pull->mutex); pull->stop = true; async_cond_broadcast(pull->cond); while(pull->tasks > 0) { async_cond_wait(pull->cond, pull->mutex); } async_mutex_unlock(pull->mutex); HTTPConnectionFree(&pull->conn); for(size_t i = 0; i < QUEUE_SIZE; ++i) { SLNSubmissionFree(&pull->queue[i]); pull->filled[i] = false; } pull->cur = 0; pull->count = 0; }
static int import(SLNPullRef const pull, strarg_t const URI, size_t const pos, HTTPConnectionRef *const conn) { if(!pull) return 0; // TODO: Even if there's nothing to do, we have to enqueue something to fill up our reserved slots. I guess it's better than doing a lot of work inside the connection lock, but there's got to be a better way. SLNSubmissionRef sub = NULL; HTTPHeadersRef headers = NULL; if(!URI) goto enqueue; str_t algo[SLN_ALGO_SIZE]; str_t hash[SLN_HASH_SIZE]; if(SLNParseURI(URI, algo, hash) < 0) goto enqueue; int rc = SLNSessionGetFileInfo(pull->session, URI, NULL); if(rc >= 0) goto enqueue; db_assertf(DB_NOTFOUND == rc, "Database error: %s", sln_strerror(rc)); // TODO: We're logging out of order when we do it like this... // alogf("Pulling %s\n", URI); if(!*conn) { rc = HTTPConnectionCreateOutgoing(pull->host, 0, conn); if(rc < 0) { alogf("Pull import connection error: %s\n", sln_strerror(rc)); goto fail; } } str_t *path = aasprintf("/sln/file/%s/%s", algo, hash); if(!path) { alogf("Pull aasprintf error\n"); goto fail; } rc = HTTPConnectionWriteRequest(*conn, HTTP_GET, path, pull->host); assert(rc >= 0); // TODO FREE(&path); HTTPConnectionWriteHeader(*conn, "Cookie", pull->cookie); HTTPConnectionBeginBody(*conn); rc = HTTPConnectionEnd(*conn); if(rc < 0) { alogf("Pull import request error: %s\n", sln_strerror(rc)); goto fail; } int const status = HTTPConnectionReadResponseStatus(*conn); if(status < 0) { alogf("Pull import response error: %s\n", sln_strerror(status)); goto fail; } if(status < 200 || status >= 300) { alogf("Pull import status error: %d\n", status); goto fail; } rc = HTTPHeadersCreateFromConnection(*conn, &headers); assert(rc >= 0); // TODO /* if(rc < 0) { alogf("Pull import headers error %s\n", sln_strerror(rc)); goto fail; }*/ strarg_t const type = HTTPHeadersGet(headers, "content-type"); rc = SLNSubmissionCreate(pull->session, URI, &sub); if(rc < 0) { alogf("Pull submission error: %s\n", sln_strerror(rc)); goto fail; } rc = SLNSubmissionSetType(sub, type); if(rc < 0) { alogf("Pull submission type error: %s\n", sln_strerror(rc)); goto fail; } for(;;) { if(pull->stop) goto fail; uv_buf_t buf[1] = {}; rc = HTTPConnectionReadBody(*conn, buf); if(rc < 0) { alogf("Pull download error: %s\n", sln_strerror(rc)); goto fail; } if(0 == buf->len) break; rc = SLNSubmissionWrite(sub, (byte_t *)buf->base, buf->len); if(rc < 0) { alogf("Pull write error\n"); goto fail; } } rc = SLNSubmissionEnd(sub); if(rc < 0) { alogf("Pull submission error: %s\n", sln_strerror(rc)); goto fail; } enqueue: HTTPHeadersFree(&headers); async_mutex_lock(pull->mutex); pull->queue[pos] = sub; sub = NULL; pull->filled[pos] = true; async_cond_broadcast(pull->cond); async_mutex_unlock(pull->mutex); return 0; fail: HTTPHeadersFree(&headers); SLNSubmissionFree(&sub); HTTPConnectionFree(conn); return -1; }
int HTTPServerListen(HTTPServerRef const server, strarg_t const address, strarg_t const port) { if(!server) return 0; assertf(!server->socket->data, "HTTPServer already listening"); int rc; rc = uv_tcp_init(async_loop, server->socket); if(rc < 0) return rc; server->socket->data = server; struct addrinfo const hints = { .ai_flags = AI_V4MAPPED | AI_ADDRCONFIG | AI_NUMERICSERV | AI_PASSIVE, .ai_family = AF_UNSPEC, .ai_socktype = SOCK_STREAM, .ai_protocol = 0, // ??? }; struct addrinfo *info; rc = async_getaddrinfo(address, port, &hints, &info); if(rc < 0) { HTTPServerClose(server); return rc; } int bound = 0; rc = 0; for(struct addrinfo *each = info; each; each = each->ai_next) { rc = uv_tcp_bind(server->socket, each->ai_addr, 0); if(rc >= 0) bound++; } uv_freeaddrinfo(info); if(!bound) { HTTPServerClose(server); if(rc < 0) return rc; return UV_EADDRNOTAVAIL; } rc = uv_listen((uv_stream_t *)server->socket, 511, connection_cb); if(rc < 0) { HTTPServerClose(server); return rc; } return 0; } int HTTPServerListenSecure(HTTPServerRef const server, strarg_t const address, strarg_t const port, struct tls **const tlsptr) { if(!server) return 0; int rc = HTTPServerListen(server, address, port); if(rc < 0) return rc; server->secure = *tlsptr; *tlsptr = NULL; return 0; } void HTTPServerClose(HTTPServerRef const server) { if(!server) return; if(!server->socket->data) return; if(server->secure) tls_close(server->secure); tls_free(server->secure); server->secure = NULL; async_close((uv_handle_t *)server->socket); } static void connection(uv_stream_t *const socket) { HTTPServerRef const server = socket->data; HTTPConnectionRef conn; int rc = HTTPConnectionCreateIncomingSecure(socket, server->secure, 0, &conn); if(rc < 0) { fprintf(stderr, "HTTP server connection error %s\n", uv_strerror(rc)); return; } assert(conn); for(;;) { server->listener(server->context, server, conn); rc = HTTPConnectionDrainMessage(conn); if(rc < 0) break; } HTTPConnectionFree(&conn); }
int HTTPConnectionCreateOutgoing(strarg_t const domain, unsigned const flags, HTTPConnectionRef *const out) { str_t host[1023+1]; str_t service[15+1]; host[0] = '\0'; service[0] = '\0'; int matched = sscanf(domain, "%1023[^:]:%15[0-9]", host, service); if(matched < 1) return UV_EINVAL; if('\0' == host[0]) return UV_EINVAL; static struct addrinfo const hints = { .ai_flags = AI_V4MAPPED | AI_ADDRCONFIG | AI_NUMERICSERV, .ai_family = AF_UNSPEC, .ai_socktype = SOCK_STREAM, .ai_protocol = 0, // ??? }; struct addrinfo *info = NULL; HTTPConnectionRef conn = NULL; int rc; rc = async_getaddrinfo(host, service[0] ? service : "80", &hints, &info); if(rc < 0) goto cleanup; conn = calloc(1, sizeof(struct HTTPConnection)); if(!conn) rc = UV_ENOMEM; if(rc < 0) goto cleanup; rc = UV_EADDRNOTAVAIL; for(struct addrinfo *each = info; each; each = each->ai_next) { rc = uv_tcp_init(async_loop, conn->stream); if(rc < 0) break; rc = async_tcp_connect(conn->stream, each->ai_addr); if(rc >= 0) break; async_close((uv_handle_t *)conn->stream); } if(rc < 0) goto cleanup; http_parser_init(conn->parser, HTTP_RESPONSE); conn->parser->data = conn; *out = conn; conn = NULL; cleanup: uv_freeaddrinfo(info); info = NULL; HTTPConnectionFree(&conn); return rc; } void HTTPConnectionFree(HTTPConnectionRef *const connptr) { HTTPConnectionRef conn = *connptr; if(!conn) return; async_close((uv_handle_t *)conn->stream); // http_parser does not need to be freed, closed or destroyed. memset(conn->parser, 0, sizeof(*conn->parser)); FREE(&conn->buf); *conn->raw = uv_buf_init(NULL, 0); conn->type = HTTPNothing; *conn->out = uv_buf_init(NULL, 0); conn->flags = 0; assert_zeroed(conn, 1); FREE(connptr); conn = NULL; }