void Test(const std::wstring& path, const std::string& search_text) {
std::unique_ptr<TaskQueue> task_queue(new TaskQueueImpl());
std::unique_ptr<TaskQueue> io_task_queue(new IOTaskQueue());
std::thread io_thread([&io_task_queue] { io_task_queue->Run(); });
Report report;
RegexSearchFactoryImpl factory(search_text, &report);
ProcessingManager consumer(
std::thread::hardware_concurrency(), std::thread::hardware_concurrency(),
std::thread::hardware_concurrency(), task_queue.get(),
io_task_queue.get(), &factory, path);
consumer.Start();
task_queue->Run();
io_task_queue->PostQuitTask();
io_thread.join();
}
void worker()
{
while (true) {
int socketfd = task_queue().pop();
char recv_buff[1024];
std::vector<char> request_buff;
std::vector<char> response_buff;
while (true) {
int read_cnt = recv(socketfd, recv_buff,
sizeof(recv_buff), MSG_NOSIGNAL);
if (-1 == read_cnt)
break;
if (0 == read_cnt)
break;
request_buff.insert(request_buff.end(),
recv_buff, recv_buff + read_cnt);
if (process_http_request(request_buff, response_buff)) {
ssize_t total_cnt = 0;
ssize_t sent_cnt = 0;
while (total_cnt != response_buff.size()) {
sent_cnt = send(socketfd, &response_buff[0] + total_cnt,
response_buff.size() - total_cnt, MSG_NOSIGNAL);
if (sent_cnt != -1)
total_cnt += sent_cnt;
}
shutdown(socketfd, SHUT_RDWR);
}
}
}
}
/* This function is called from the protocol layer accept() in order to
* instanciate a new session on behalf of a given listener and frontend. It
* returns a positive value upon success, 0 if the connection can be ignored,
* or a negative value upon critical failure. The accepted file descriptor is
* closed if we return <= 0. If no handshake is needed, it immediately tries
* to instanciate a new stream. The created connection's owner points to the
* new session until the upper layers are created.
*/
int session_accept_fd(struct listener *l, int cfd, struct sockaddr_storage *addr)
{
struct connection *cli_conn;
struct proxy *p = l->bind_conf->frontend;
struct session *sess;
int ret;
ret = -1; /* assume unrecoverable error by default */
if (unlikely((cli_conn = conn_new()) == NULL))
goto out_close;
conn_prepare(cli_conn, l->proto, l->bind_conf->xprt);
cli_conn->handle.fd = cfd;
cli_conn->addr.from = *addr;
cli_conn->flags |= CO_FL_ADDR_FROM_SET;
cli_conn->target = &l->obj_type;
cli_conn->proxy_netns = l->netns;
conn_ctrl_init(cli_conn);
/* wait for a PROXY protocol header */
if (l->options & LI_O_ACC_PROXY) {
cli_conn->flags |= CO_FL_ACCEPT_PROXY;
conn_sock_want_recv(cli_conn);
}
/* wait for a NetScaler client IP insertion protocol header */
if (l->options & LI_O_ACC_CIP) {
cli_conn->flags |= CO_FL_ACCEPT_CIP;
conn_sock_want_recv(cli_conn);
}
conn_xprt_want_recv(cli_conn);
if (conn_xprt_init(cli_conn) < 0)
goto out_free_conn;
sess = session_new(p, l, &cli_conn->obj_type);
if (!sess)
goto out_free_conn;
conn_set_owner(cli_conn, sess, NULL);
/* now evaluate the tcp-request layer4 rules. We only need a session
* and no stream for these rules.
*/
if ((l->options & LI_O_TCP_L4_RULES) && !tcp_exec_l4_rules(sess)) {
/* let's do a no-linger now to close with a single RST. */
setsockopt(cfd, SOL_SOCKET, SO_LINGER, (struct linger *) &nolinger, sizeof(struct linger));
ret = 0; /* successful termination */
goto out_free_sess;
}
/* monitor-net and health mode are processed immediately after TCP
* connection rules. This way it's possible to block them, but they
* never use the lower data layers, they send directly over the socket,
* as they were designed for. We first flush the socket receive buffer
* in order to avoid emission of an RST by the system. We ignore any
* error.
*/
if (unlikely((p->mode == PR_MODE_HEALTH) ||
((l->options & LI_O_CHK_MONNET) &&
addr->ss_family == AF_INET &&
(((struct sockaddr_in *)addr)->sin_addr.s_addr & p->mon_mask.s_addr) == p->mon_net.s_addr))) {
/* we have 4 possibilities here :
* - HTTP mode, from monitoring address => send "HTTP/1.0 200 OK"
* - HEALTH mode with HTTP check => send "HTTP/1.0 200 OK"
* - HEALTH mode without HTTP check => just send "OK"
* - TCP mode from monitoring address => just close
*/
if (l->proto->drain)
l->proto->drain(cfd);
if (p->mode == PR_MODE_HTTP ||
(p->mode == PR_MODE_HEALTH && (p->options2 & PR_O2_CHK_ANY) == PR_O2_HTTP_CHK))
send(cfd, "HTTP/1.0 200 OK\r\n\r\n", 19, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE);
else if (p->mode == PR_MODE_HEALTH)
send(cfd, "OK\n", 3, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE);
ret = 0;
goto out_free_sess;
}
/* Adjust some socket options */
if (l->addr.ss_family == AF_INET || l->addr.ss_family == AF_INET6) {
setsockopt(cfd, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof(one));
if (p->options & PR_O_TCP_CLI_KA)
setsockopt(cfd, SOL_SOCKET, SO_KEEPALIVE, (char *) &one, sizeof(one));
if (p->options & PR_O_TCP_NOLING)
fdtab[cfd].linger_risk = 1;
#if defined(TCP_MAXSEG)
if (l->maxseg < 0) {
/* we just want to reduce the current MSS by that value */
int mss;
socklen_t mss_len = sizeof(mss);
if (getsockopt(cfd, IPPROTO_TCP, TCP_MAXSEG, &mss, &mss_len) == 0) {
mss += l->maxseg; /* remember, it's < 0 */
setsockopt(cfd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss));
}
}
#endif
}
if (global.tune.client_sndbuf)
setsockopt(cfd, SOL_SOCKET, SO_SNDBUF, &global.tune.client_sndbuf, sizeof(global.tune.client_sndbuf));
if (global.tune.client_rcvbuf)
setsockopt(cfd, SOL_SOCKET, SO_RCVBUF, &global.tune.client_rcvbuf, sizeof(global.tune.client_rcvbuf));
/* OK, now either we have a pending handshake to execute with and then
* we must return to the I/O layer, or we can proceed with the end of
* the stream initialization. In case of handshake, we also set the I/O
* timeout to the frontend's client timeout and register a task in the
* session for this purpose. The connection's owner is left to the
* session during this period.
*
* At this point we set the relation between sess/task/conn this way :
*
* +----------------- task
* | |
* orig -- sess <-- context |
* | ^ | |
* v | | |
* conn -- owner ---> task <-----+
*/
if (cli_conn->flags & (CO_FL_HANDSHAKE | CO_FL_EARLY_SSL_HS)) {
if (unlikely((sess->task = task_new(tid_bit)) == NULL))
goto out_free_sess;
conn_set_xprt_done_cb(cli_conn, conn_complete_session);
sess->task->context = sess;
sess->task->nice = l->nice;
sess->task->process = session_expire_embryonic;
sess->task->expire = tick_add_ifset(now_ms, p->timeout.client);
task_queue(sess->task);
return 1;
}
/* OK let's complete stream initialization since there is no handshake */
if (conn_complete_session(cli_conn) >= 0)
return 1;
/* error unrolling */
out_free_sess:
/* prevent call to listener_release during session_free. It will be
* done below, for all errors. */
sess->listener = NULL;
session_free(sess);
out_free_conn:
conn_stop_tracking(cli_conn);
conn_xprt_close(cli_conn);
conn_free(cli_conn);
out_close:
listener_release(l);
if (ret < 0 && l->bind_conf->xprt == xprt_get(XPRT_RAW) && p->mode == PR_MODE_HTTP) {
/* critical error, no more memory, try to emit a 500 response */
struct chunk *err_msg = &p->errmsg[HTTP_ERR_500];
if (!err_msg->str)
err_msg = &http_err_chunks[HTTP_ERR_500];
send(cfd, err_msg->str, err_msg->len, MSG_DONTWAIT|MSG_NOSIGNAL);
}
if (fdtab[cfd].owner)
fd_delete(cfd);
else
close(cfd);
return ret;
}
/* This function is called from the protocol layer accept() in order to
* instanciate a new session on behalf of a given listener and frontend. It
* returns a positive value upon success, 0 if the connection can be ignored,
* or a negative value upon critical failure. The accepted file descriptor is
* closed if we return <= 0. If no handshake is needed, it immediately tries
* to instanciate a new stream.
*/
int session_accept_fd(struct listener *l, int cfd, struct sockaddr_storage *addr)
{
struct connection *cli_conn;
struct proxy *p = l->frontend;
struct session *sess;
struct stream *strm;
struct task *t;
int ret;
ret = -1; /* assume unrecoverable error by default */
if (unlikely((cli_conn = conn_new()) == NULL))
goto out_close;
conn_prepare(cli_conn, l->proto, l->xprt);
cli_conn->t.sock.fd = cfd;
cli_conn->addr.from = *addr;
cli_conn->flags |= CO_FL_ADDR_FROM_SET;
cli_conn->target = &l->obj_type;
cli_conn->proxy_netns = l->netns;
conn_ctrl_init(cli_conn);
/* wait for a PROXY protocol header */
if (l->options & LI_O_ACC_PROXY) {
cli_conn->flags |= CO_FL_ACCEPT_PROXY;
conn_sock_want_recv(cli_conn);
}
conn_data_want_recv(cli_conn);
if (conn_xprt_init(cli_conn) < 0)
goto out_free_conn;
sess = session_new(p, l, &cli_conn->obj_type);
if (!sess)
goto out_free_conn;
p->feconn++;
/* This session was accepted, count it now */
if (p->feconn > p->fe_counters.conn_max)
p->fe_counters.conn_max = p->feconn;
proxy_inc_fe_conn_ctr(l, p);
/* now evaluate the tcp-request layer4 rules. We only need a session
* and no stream for these rules.
*/
if ((l->options & LI_O_TCP_RULES) && !tcp_exec_req_rules(sess)) {
/* let's do a no-linger now to close with a single RST. */
setsockopt(cfd, SOL_SOCKET, SO_LINGER, (struct linger *) &nolinger, sizeof(struct linger));
ret = 0; /* successful termination */
goto out_free_sess;
}
/* monitor-net and health mode are processed immediately after TCP
* connection rules. This way it's possible to block them, but they
* never use the lower data layers, they send directly over the socket,
* as they were designed for. We first flush the socket receive buffer
* in order to avoid emission of an RST by the system. We ignore any
* error.
*/
if (unlikely((p->mode == PR_MODE_HEALTH) ||
((l->options & LI_O_CHK_MONNET) &&
addr->ss_family == AF_INET &&
(((struct sockaddr_in *)addr)->sin_addr.s_addr & p->mon_mask.s_addr) == p->mon_net.s_addr))) {
/* we have 4 possibilities here :
* - HTTP mode, from monitoring address => send "HTTP/1.0 200 OK"
* - HEALTH mode with HTTP check => send "HTTP/1.0 200 OK"
* - HEALTH mode without HTTP check => just send "OK"
* - TCP mode from monitoring address => just close
*/
if (l->proto->drain)
l->proto->drain(cfd);
if (p->mode == PR_MODE_HTTP ||
(p->mode == PR_MODE_HEALTH && (p->options2 & PR_O2_CHK_ANY) == PR_O2_HTTP_CHK))
send(cfd, "HTTP/1.0 200 OK\r\n\r\n", 19, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE);
else if (p->mode == PR_MODE_HEALTH)
send(cfd, "OK\n", 3, MSG_DONTWAIT|MSG_NOSIGNAL|MSG_MORE);
ret = 0;
goto out_free_sess;
}
/* Adjust some socket options */
if (l->addr.ss_family == AF_INET || l->addr.ss_family == AF_INET6) {
setsockopt(cfd, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof(one));
if (p->options & PR_O_TCP_CLI_KA)
setsockopt(cfd, SOL_SOCKET, SO_KEEPALIVE, (char *) &one, sizeof(one));
if (p->options & PR_O_TCP_NOLING)
fdtab[cfd].linger_risk = 1;
#if defined(TCP_MAXSEG)
if (l->maxseg < 0) {
/* we just want to reduce the current MSS by that value */
int mss;
socklen_t mss_len = sizeof(mss);
if (getsockopt(cfd, IPPROTO_TCP, TCP_MAXSEG, &mss, &mss_len) == 0) {
mss += l->maxseg; /* remember, it's < 0 */
setsockopt(cfd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss));
}
}
#endif
}
if (global.tune.client_sndbuf)
setsockopt(cfd, SOL_SOCKET, SO_SNDBUF, &global.tune.client_sndbuf, sizeof(global.tune.client_sndbuf));
if (global.tune.client_rcvbuf)
setsockopt(cfd, SOL_SOCKET, SO_RCVBUF, &global.tune.client_rcvbuf, sizeof(global.tune.client_rcvbuf));
if (unlikely((t = task_new()) == NULL))
goto out_free_sess;
t->context = sess;
t->nice = l->nice;
/* OK, now either we have a pending handshake to execute with and
* then we must return to the I/O layer, or we can proceed with the
* end of the stream initialization. In case of handshake, we also
* set the I/O timeout to the frontend's client timeout.
*
* At this point we set the relation between sess/task/conn this way :
*
* orig -- sess <-- context
* | |
* v |
* conn -- owner ---> task
*/
if (cli_conn->flags & CO_FL_HANDSHAKE) {
conn_attach(cli_conn, t, &sess_conn_cb);
t->process = session_expire_embryonic;
t->expire = tick_add_ifset(now_ms, p->timeout.client);
task_queue(t);
cli_conn->flags |= CO_FL_INIT_DATA | CO_FL_WAKE_DATA;
return 1;
}
/* OK let's complete stream initialization since there is no handshake */
cli_conn->flags |= CO_FL_CONNECTED;
/* we want the connection handler to notify the stream interface about updates. */
cli_conn->flags |= CO_FL_WAKE_DATA;
/* if logs require transport layer information, note it on the connection */
if (sess->fe->to_log & LW_XPRT)
cli_conn->flags |= CO_FL_XPRT_TRACKED;
session_count_new(sess);
strm = stream_new(sess, t, &cli_conn->obj_type);
if (!strm)
goto out_free_task;
strm->target = sess->listener->default_target;
strm->req.analysers = sess->listener->analysers;
return 1;
out_free_task:
task_free(t);
out_free_sess:
p->feconn--;
session_free(sess);
out_free_conn:
cli_conn->flags &= ~CO_FL_XPRT_TRACKED;
conn_xprt_close(cli_conn);
conn_free(cli_conn);
out_close:
if (ret < 0 && l->xprt == &raw_sock && p->mode == PR_MODE_HTTP) {
/* critical error, no more memory, try to emit a 500 response */
struct chunk *err_msg = &p->errmsg[HTTP_ERR_500];
if (!err_msg->str)
err_msg = &http_err_chunks[HTTP_ERR_500];
send(cfd, err_msg->str, err_msg->len, MSG_DONTWAIT|MSG_NOSIGNAL);
}
if (fdtab[cfd].owner)
fd_delete(cfd);
else
close(cfd);
return ret;
}
/*
* This is the proxy management task. It enables proxies when there are enough
* free sessions, or stops them when the table is full. It is designed to be
* called as a task which is woken up upon stopping or when rate limiting must
* be enforced.
*/
struct task *manage_proxy(struct task *t)
{
struct proxy *p = t->context;
int next = TICK_ETERNITY;
unsigned int wait;
/* We should periodically try to enable listeners waiting for a
* global resource here.
*/
/* first, let's check if we need to stop the proxy */
if (unlikely(stopping && p->state != PR_STSTOPPED)) {
int t;
t = tick_remain(now_ms, p->stop_time);
if (t == 0) {
Warning("Proxy %s stopped (FE: %lld conns, BE: %lld conns).\n",
p->id, p->fe_counters.cum_conn, p->be_counters.cum_conn);
send_log(p, LOG_WARNING, "Proxy %s stopped (FE: %lld conns, BE: %lld conns).\n",
p->id, p->fe_counters.cum_conn, p->be_counters.cum_conn);
stop_proxy(p);
/* try to free more memory */
pool_gc2();
}
else {
next = tick_first(next, p->stop_time);
}
}
/* the rest below is just for frontends */
if (!(p->cap & PR_CAP_FE))
goto out;
/* check the various reasons we may find to block the frontend */
if (unlikely(p->feconn >= p->maxconn)) {
if (p->state == PR_STREADY)
p->state = PR_STFULL;
goto out;
}
/* OK we have no reason to block, so let's unblock if we were blocking */
if (p->state == PR_STFULL)
p->state = PR_STREADY;
if (p->fe_sps_lim &&
(wait = next_event_delay(&p->fe_sess_per_sec, p->fe_sps_lim, 0))) {
/* we're blocking because a limit was reached on the number of
* requests/s on the frontend. We want to re-check ASAP, which
* means in 1 ms before estimated expiration date, because the
* timer will have settled down.
*/
next = tick_first(next, tick_add(now_ms, wait));
goto out;
}
/* The proxy is not limited so we can re-enable any waiting listener */
if (!LIST_ISEMPTY(&p->listener_queue))
dequeue_all_listeners(&p->listener_queue);
out:
t->expire = next;
task_queue(t);
return t;
}
