void bstr_builder_clear(bstr_builder_t *bb) {
// Do nothing if the list is empty
if (htp_list_size(bb->pieces) == 0) return;
for (size_t i = 0, n = htp_list_size(bb->pieces); i < n; i++) {
bstr *b = htp_list_get(bb->pieces, i);
bstr_free(b);
}
htp_list_clear(bb->pieces);
}
/**
* Free stream data.
*
* @param[in] sd
*/
void free_stream_data(stream_data *sd) {
if (sd == NULL) return;
// Free stream chunks, if any
if (sd->chunks != NULL) {
for (int i = 0, n = htp_list_size(sd->chunks); i < n; i++) {
chunk_t *chunk = htp_list_get(sd->chunks, i);
free(chunk->data);
free(chunk);
}
htp_list_destroy(sd->chunks);
sd->chunked = NULL;
}
// Free inbound chunks, if any
if (sd->inbound_chunks != NULL) {
for (int i = 0, n = htp_list_size(sd->inbound_chunks); i < n; i++) {
chunk_t *chunk = htp_list_get(sd->inbound_chunkds, i);
free(chunk->data);
free(chunk);
}
htp_list_destroy(sd->inbound_chunks);
sd->inbound_chunks = NULL;
}
// Free outbound chunks, if any
if (sd->outbound_chunks != NULL) {
for (int i = 0, n = htp_list_size(sd->outbound_chunks); i < n; i++) {
chunk_t *chunk = htp_list_get(sd->outbound_chunkds, i);
free(chunk->data);
free(chunk);
}
htp_list_destroy(sd->outbound_chunks);
sd->outbound_chunks = NULL;
}
// Close the stream file, if we have it open
if (sd->fd != -1) {
close(sd->fd);
}
free(sd);
}
/**
* Process as much buffered inbound and outbound data as possible
* (in that order)
*
* @param[in] sd
*/
void process_stored_stream_data(stream_data *sd) {
int loop = 0;
do {
// Reset loop
loop = 0;
// Send as much inbound data as possible
while((sd->connp->in_status == HTP_STREAM_DATA)&&(htp_list_size(sd->inbound_chunks) > 0)) {
chunk_t *chunk = (chunk_t *)htp_list_get(sd->inbound_chunks, 0);
int rc = htp_connp_req_data(sd->connp, 0, chunk->data + chunk->consumed, chunk->len - chunk->consumed);
if (rc == HTP_STREAM_DATA) {
// The entire chunk was consumed
htp_list_shift(sd->inbound_chunks);
free(chunk->data);
free(chunk);
} else {
// Partial consumption
chunk->consumed = htp_connp_req_data_consumed(sd->connp);
}
}
// Send as much outbound data as possible
while((sd->connp->out_status == HTP_STREAM_DATA)&&(htp_list_size(sd->outbound_chunks) > 0)) {
chunk_t *chunk = (chunk_t *)htp_list_get(sd->outbound_chunks, 0);
int rc = htp_connp_res_data(sd->connp, 0, chunk->data + chunk->consumed, chunk->len - chunk->consumed);
if (rc == HTP_STREAM_DATA) {
// The entire chunk was consumed
htp_list_shift(sd->outbound_chunks);
free(chunk->data);
free(chunk);
} else {
// Partial consumption
chunk->consumed = htp_connp_res_data_consumed(sd->connp);
}
// Whenever we send some outbound data to the parser,
// we need to go back and try to send inbound data
// if we have it.
loop = 1;
}
} while(loop);
}
void htp_hook_destroy(htp_hook_t *hook) {
if (hook == NULL) return;
for (size_t i = 0, n = htp_list_size(hook->callbacks); i < n; i++) {
free((htp_callback_t *) htp_list_get(hook->callbacks, i));
}
htp_list_array_destroy(hook->callbacks);
free(hook);
}
void *htp_table_get_index(const htp_table_t *table, size_t idx, bstr **key) {
if (table == NULL) return NULL;
if (idx >= htp_list_size(table->list)) return NULL;
if (key != NULL) {
*key = htp_list_get(table->list, idx * 2);
}
return htp_list_get(table->list, (idx * 2) + 1);
}
void htp_conn_destroy(htp_conn_t *conn) {
if (conn == NULL) return;
if (conn->transactions != NULL) {
// Destroy individual transactions. Do note that iterating
// using the iterator does not work here because some of the
// list element may be NULL (and with the iterator it is impossible
// to distinguish a NULL element from the end of the list).
for (size_t i = 0, n = htp_list_size(conn->transactions); i < n; i++) {
htp_tx_t *tx = htp_list_get(conn->transactions, i);
if (tx != NULL) {
htp_tx_destroy_incomplete(tx);
}
}
htp_list_destroy(conn->transactions);
conn->transactions = NULL;
}
if (conn->messages != NULL) {
// Destroy individual messages.
for (size_t i = 0, n = htp_list_size(conn->messages); i < n; i++) {
htp_log_t *l = htp_list_get(conn->messages, i);
free((void *) l->msg);
free(l);
}
htp_list_destroy(conn->messages);
conn->messages = NULL;
}
if (conn->server_addr != NULL) {
free(conn->server_addr);
}
if (conn->client_addr != NULL) {
free(conn->client_addr);
}
free(conn);
}
bstr *bstr_builder_to_str(const bstr_builder_t *bb) {
size_t len = 0;
// Determine the size of the string
for (size_t i = 0, n = htp_list_size(bb->pieces); i < n; i++) {
bstr *b = htp_list_get(bb->pieces, i);
len += bstr_len(b);
}
// Allocate string
bstr *bnew = bstr_alloc(len);
if (bnew == NULL) return NULL;
// Determine the size of the string
for (size_t i = 0, n = htp_list_size(bb->pieces); i < n; i++) {
bstr *b = htp_list_get(bb->pieces, i);
bstr_add_noex(bnew, b);
}
return bnew;
}
void bstr_builder_destroy(bstr_builder_t *bb) {
if (bb == NULL) return;
// Destroy any pieces we might have
for (size_t i = 0, n = htp_list_size(bb->pieces); i < n; i++) {
bstr *b = htp_list_get(bb->pieces, i);
bstr_free(b);
}
htp_list_destroy(bb->pieces);
free(bb);
}
htp_status_t htp_conn_remove_tx(htp_conn_t *conn, const htp_tx_t *tx) {
if ((tx == NULL) || (conn == NULL)) return HTP_ERROR;
if (conn->transactions == NULL) return HTP_ERROR;
for (size_t i = 0, n = htp_list_size(conn->transactions); i < n; i++) {
htp_tx_t *candidate_tx = htp_list_get(conn->transactions, i);
if (tx == candidate_tx) {
htp_list_replace(conn->transactions, i, NULL);
return HTP_OK;
}
}
return HTP_DECLINED;
}
/*
* XXX: Not sure I like mucking around in the transaction to get the method,
* but I'm not sure of a better way.
*/
static PyObject *htpy_connp_get_method(PyObject *self, PyObject *args) {
PyObject *ret;
htp_tx_t *tx = NULL;
tx = htp_list_get(((htpy_connp *) self)->connp->conn->transactions, htp_list_size(((htpy_connp *) self)->connp->conn->transactions) - 1);
if (!tx || !tx->request_method) {
PyErr_SetString(htpy_error, "Missing transaction or request method.");
return NULL;
}
ret = Py_BuildValue("s#", bstr_ptr(tx->request_method), bstr_len(tx->request_method));
return ret;
}
static PyObject *htpy_connp_get_response_status(PyObject *self, PyObject *args) {
PyObject *ret;
htp_tx_t *tx = NULL;
tx = htp_list_get(((htpy_connp *) self)->connp->conn->transactions, htp_list_size(((htpy_connp *) self)->connp->conn->transactions) - 1);
if (!tx) {
PyErr_SetString(htpy_error, "Missing transaction.");
return NULL;
}
ret = Py_BuildValue("i", tx->response_status_number);
return ret;
}
void htp_table_clear(htp_table_t *table) {
if (table == NULL) return;
// Free the table keys, but only if we're managing them.
if ((table->alloc_type == HTP_TABLE_KEYS_COPIED)||(table->alloc_type == HTP_TABLE_KEYS_ADOPTED)) {
bstr *key = NULL;
for (size_t i = 0, n = htp_list_size(table->list); i < n; i += 2) {
key = htp_list_get(table->list, i);
bstr_free(key);
}
}
htp_list_clear(table->list);
}
/**
* Invoked every time LibHTP wants to log.
*
* @param[in] log
*/
int callback_log(htp_log_t *log) {
stream_data *sd = (stream_data *)htp_connp_get_user_data(log->connp);
if ((sd->log_level == -1)||(sd->log_level > log->level)) {
sd->log_level = log->level;
}
if (log->code != 0) {
fprintf(stderr, "[#%d/%d][%d][code %d][file %s][line %d] %s\n", sd->id, sd->req_count,
log->level, log->code, log->file, log->line, log->msg);
} else {
fprintf(stderr, "[#%d/%d][%d][file %s][line %d] %s\n", sd->id, sd->req_count,
log->level, log->file, log->line, log->msg);
}
// If this is the first time a log message was generated for this connection,
// start writing the entire thing to a file on disk.
if (sd->fd == -1) {
char filename[256];
// TODO Use IP addresses and ports in filename
snprintf(filename, 255, "conn-%d.t", sd->id);
sd->fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR);
if (sd->fd == -1) {
fprintf(stderr, "Failed to create file %s: %s\n", filename, strerror(errno));
exit(1);
}
// Write to disk the data we have in memory
for (int i = 0, n = htp_list_size(sd->chunks); i < n; i++) {
chunk_t *chunk = htp_list_get(sd->chunks, i);
if (sd->chunk_counter != 0) {
write(sd->fd, "\r\n", 2);
}
if (sd->direction == chunk->direction) {
write(sd->fd, ">>>\r\n", 5);
} else {
write(sd->fd, "<<<\r\n", 5);
}
write(sd->fd, chunk->data, chunk->len);
sd->chunk_counter++;
}
}
}
void *htp_table_get_mem(const htp_table_t *table, const void *key, size_t key_len) {
if ((table == NULL)||(key == NULL)) return NULL;
// Iterate through the list, comparing
// keys with the parameter, return data if found.
for (size_t i = 0, n = htp_list_size(table->list); i < n; i += 2) {
bstr *key_candidate = htp_list_get(table->list, i);
void *element = htp_list_get(table->list, i + 1);
if (bstr_cmp_mem_nocase(key_candidate, key, key_len) == 0) {
return element;
}
}
return NULL;
}
htp_hook_t *htp_hook_copy(const htp_hook_t *hook) {
if (hook == NULL) return NULL;
htp_hook_t *copy = htp_hook_create();
if (copy == NULL) return NULL;
for (size_t i = 0, n = htp_list_size(hook->callbacks); i < n; i++) {
htp_callback_t *callback = htp_list_get(hook->callbacks, i);
if (htp_hook_register(©, callback->fn) != HTP_OK) {
htp_hook_destroy(copy);
return NULL;
}
}
return copy;
}
htp_tx_t *htp_connp_tx_create(htp_connp_t *connp) {
// Detect pipelining
if (htp_list_size(connp->conn->transactions) > connp->out_next_tx_index) {
connp->conn->flags |= HTP_CONN_PIPELINED;
}
htp_tx_t *tx = htp_tx_create(connp);
if (tx == NULL) return NULL;
connp->in_tx = tx;
htp_list_add(connp->conn->transactions, tx);
htp_connp_in_reset(connp);
return tx;
}
htp_status_t htp_hook_run_all(htp_hook_t *hook, void *user_data) {
if (hook == NULL) return HTP_OK;
// Loop through the registered callbacks, giving each a chance to run.
for (size_t i = 0, n = htp_list_size(hook->callbacks); i < n; i++) {
htp_callback_t *callback = htp_list_get(hook->callbacks, i);
htp_status_t rc = callback->fn(user_data);
// A hook can return HTP_OK to say that it did some work,
// or HTP_DECLINED to say that it did no work. Anything else
// is treated as an error.
if ((rc != HTP_OK) && (rc != HTP_DECLINED)) {
return rc;
}
}
return HTP_OK;
}
/**
* Finalize Multipart processing.
*
* @param[in] d
* @return HTP_OK on success, HTP_ERROR on failure.
*/
htp_status_t htp_ch_multipart_callback_request_body_data(htp_tx_data_t *d) {
htp_tx_t *tx = d->tx;
// Check that we were not invoked again after the finalization.
if (tx->request_mpartp->gave_up_data == 1) return HTP_ERROR;
if (d->data != NULL) {
// Process one chunk of data.
htp_mpartp_parse(tx->request_mpartp, d->data, d->len);
} else {
// Finalize parsing.
htp_mpartp_finalize(tx->request_mpartp);
htp_multipart_t *body = htp_mpartp_get_multipart(tx->request_mpartp);
for (size_t i = 0, n = htp_list_size(body->parts); i < n; i++) {
htp_multipart_part_t *part = htp_list_get(body->parts, i);
// Use text parameters.
if (part->type == MULTIPART_PART_TEXT) {
htp_param_t *param = calloc(1, sizeof (htp_param_t));
if (param == NULL) return HTP_ERROR;
param->name = part->name;
param->value = part->value;
param->source = HTP_SOURCE_BODY;
param->parser_id = HTP_PARSER_MULTIPART;
param->parser_data = part;
if (htp_tx_req_add_param(tx, param) != HTP_OK) {
free(param);
return HTP_ERROR;
}
}
}
// Tell the parser that it no longer owns names
// and values of MULTIPART_PART_TEXT parts.
tx->request_mpartp->gave_up_data = 1;
}
return HTP_OK;
}
htp_status_t htp_hook_run_one(htp_hook_t *hook, void *user_data) {
if (hook == NULL) return HTP_DECLINED;
for (size_t i = 0, n = htp_list_size(hook->callbacks); i < n; i++) {
htp_callback_t *callback = htp_list_get(hook->callbacks, i);
htp_status_t rc = callback->fn(user_data);
// A hook can return HTP_DECLINED to say that it did no work,
// and we'll ignore that. If we see HTP_OK or anything else,
// we stop processing (because it was either a successful
// handling or an error).
if (rc != HTP_DECLINED) {
// Return HTP_OK or an error.
return rc;
}
}
// No hook wanted to process the callback.
return HTP_DECLINED;
}
size_t bstr_builder_size(const bstr_builder_t *bb) {
return htp_list_size(bb->pieces);
}
static htp_status_t htp_tx_process_request_headers(htp_tx_t *tx) {
// Remember how many header lines there were before trailers.
tx->request_header_lines_no_trailers = htp_list_size(tx->request_header_lines);
// Determine if we have a request body, and how it is packaged.
htp_header_t *cl = htp_table_get_c(tx->request_headers, "content-length");
htp_header_t *te = htp_table_get_c(tx->request_headers, "transfer-encoding");
// Check for the Transfer-Encoding header, which would indicate a chunked request body.
if (te != NULL) {
// Make sure it contains "chunked" only.
if (bstr_cmp_c(te->value, "chunked") != 0) {
// Invalid T-E header value.
tx->flags |= HTP_INVALID_CHUNKING;
htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
"Invalid T-E value in request");
}
// Chunked encoding is a HTTP/1.1 feature. Check that some other protocol is not
// used. The flag will also be set if the protocol could not be parsed.
//
// TODO IIS 7.0, for example, would ignore the T-E header when it
// it is used with a protocol below HTTP 1.1.
if (tx->request_protocol_number < HTP_PROTOCOL_1_1) {
tx->flags |= HTP_INVALID_CHUNKING;
}
// If the T-E header is present we are going to use it.
tx->request_transfer_coding = HTP_CODING_CHUNKED;
// We are still going to check for the presence of C-L.
if (cl != NULL) {
// This is a violation of the RFC.
tx->flags |= HTP_REQUEST_SMUGGLING;
}
} else if (cl != NULL) {
// We have a request body of known length.
tx->request_transfer_coding = HTP_CODING_IDENTITY;
// Check for a folded C-L header.
if (cl->flags & HTP_FIELD_FOLDED) {
tx->flags |= HTP_REQUEST_SMUGGLING;
}
// Check for multiple C-L headers.
if (cl->flags & HTP_FIELD_REPEATED) {
tx->flags |= HTP_REQUEST_SMUGGLING;
}
// Get body length.
tx->request_content_length = htp_parse_content_length(cl->value);
if (tx->request_content_length < 0) {
htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Invalid C-L field in request");
return HTP_ERROR;
}
} else {
// No body.
tx->request_transfer_coding = HTP_CODING_NO_BODY;
}
// Check for PUT requests, which we need to treat as file uploads.
if (tx->request_method_number == HTP_M_PUT) {
if (htp_tx_req_has_body(tx)) {
// Prepare to treat PUT request body as a file.
tx->connp->put_file = calloc(1, sizeof (htp_file_t));
if (tx->connp->put_file == NULL) return HTP_ERROR;
tx->connp->put_file->source = HTP_FILE_PUT;
} else {
// TODO Warn about PUT request without a body.
}
return HTP_OK;
}
// Host resolution
htp_header_t *h = htp_table_get_c(tx->request_headers, "host");
if (h == NULL) {
// No host information in the headers.
// HTTP/1.1 requires host information in the headers.
if (tx->request_protocol_number >= HTP_PROTOCOL_1_1) {
tx->flags |= HTP_HOST_MISSING;
htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0,
"Host information in request headers required by HTTP/1.1");
}
} else {
// Host information available in the headers.
bstr *hostname;
int port;
if (htp_parse_hostport(h->value, &hostname, &port, &(tx->flags)) != HTP_OK) return HTP_ERROR;
// Is there host information in the URI?
if (tx->parsed_uri->hostname == NULL) {
// There is no host information in the URI. Place the
// hostname from the headers into the parsed_uri structure.
tx->parsed_uri->hostname = hostname;
tx->parsed_uri->port_number = port;
} else {
if ((bstr_cmp_nocase(hostname, tx->parsed_uri->hostname) != 0) || (port != tx->parsed_uri->port_number)) {
// The host information is different in the
// headers and the URI. The HTTP RFC states that
// we should ignore the header copy.
tx->flags |= HTP_HOST_AMBIGUOUS;
htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Host information ambiguous");
}
bstr_free(hostname);
}
}
// Parse the Content-Type header.
htp_header_t *ct = htp_table_get_c(tx->request_headers, "content-type");
if (ct != NULL) {
if (htp_parse_ct_header(ct->value, &tx->request_content_type) != HTP_OK) return HTP_ERROR;
}
// Parse cookies.
if (tx->connp->cfg->parse_request_cookies) {
htp_parse_cookies_v0(tx->connp);
}
// Parse authentication information.
if (tx->connp->cfg->parse_request_http_authentication) {
htp_parse_authorization(tx->connp);
}
// Run hook REQUEST_HEADERS.
int rc = htp_hook_run_all(tx->connp->cfg->hook_request_headers, tx->connp);
if (rc != HTP_OK) return rc;
return HTP_OK;
}
void htp_tx_destroy(htp_tx_t *tx) {
bstr_free(tx->request_line);
bstr_free(tx->request_line_raw);
bstr_free(tx->request_method);
bstr_free(tx->request_uri);
bstr_free(tx->request_uri_normalized);
bstr_free(tx->request_protocol);
bstr_free(tx->request_headers_sep);
if (tx->parsed_uri != NULL) {
bstr_free(tx->parsed_uri->scheme);
bstr_free(tx->parsed_uri->username);
bstr_free(tx->parsed_uri->password);
bstr_free(tx->parsed_uri->hostname);
bstr_free(tx->parsed_uri->port);
bstr_free(tx->parsed_uri->path);
bstr_free(tx->parsed_uri->query);
bstr_free(tx->parsed_uri->fragment);
free(tx->parsed_uri);
}
if (tx->parsed_uri_incomplete != NULL) {
bstr_free(tx->parsed_uri_incomplete->scheme);
bstr_free(tx->parsed_uri_incomplete->username);
bstr_free(tx->parsed_uri_incomplete->password);
bstr_free(tx->parsed_uri_incomplete->hostname);
bstr_free(tx->parsed_uri_incomplete->port);
bstr_free(tx->parsed_uri_incomplete->path);
bstr_free(tx->parsed_uri_incomplete->query);
bstr_free(tx->parsed_uri_incomplete->fragment);
free(tx->parsed_uri_incomplete);
}
// Destroy request_header_lines.
if (tx->request_header_lines != NULL) {
for (int i = 0, n = htp_list_size(tx->request_header_lines); i < n; i++) {
htp_header_line_t *hl = htp_list_get(tx->request_header_lines, i);
bstr_free(hl->line);
// No need to destroy hl->header because
// htp_header_line_t does not own it.
free(hl);
}
htp_list_destroy(tx->request_header_lines);
tx->request_header_lines = NULL;
}
// Destroy request_headers.
if (tx->request_headers != NULL) {
htp_header_t *h = NULL;
for (int i = 0, n = htp_table_size(tx->request_headers); i < n; i++) {
h = htp_table_get_index(tx->request_headers, i, NULL);
bstr_free(h->name);
bstr_free(h->value);
free(h);
}
htp_table_destroy(tx->request_headers);
}
if (tx->request_headers_raw != NULL) {
bstr_free(tx->request_headers_raw);
}
if (tx->response_headers_raw != NULL) {
bstr_free(tx->response_headers_raw);
}
bstr_free(tx->response_line);
bstr_free(tx->response_line_raw);
bstr_free(tx->response_protocol);
bstr_free(tx->response_status);
bstr_free(tx->response_message);
bstr_free(tx->response_headers_sep);
// Destroy response_header_lines.
if (tx->response_header_lines != NULL) {
for (int i = 0, n = htp_list_size(tx->response_header_lines); i < n; i++) {
htp_header_line_t *hl = htp_list_get(tx->response_header_lines, i);
bstr_free(hl->line);
// No need to destroy hl->header because
// htp_header_line_t does not own it.
free(hl);
}
htp_list_destroy(tx->response_header_lines);
tx->response_header_lines = NULL;
}
// Destroy response headers.
if (tx->response_headers != NULL) {
htp_header_t *h = NULL;
for (int i = 0, n = htp_table_size(tx->response_headers); i < n; i++) {
h = htp_table_get_index(tx->response_headers, i, NULL);
bstr_free(h->name);
bstr_free(h->value);
free(h);
}
htp_table_destroy(tx->response_headers);
}
// Tell the connection to remove this transaction from the list.
htp_conn_remove_tx(tx->conn, tx);
// Invalidate the pointer to this transactions held
// by the connection parser. This is to allow a transaction
// to be destroyed from within the final response callback.
if (tx->connp != NULL) {
if (tx->connp->out_tx == tx) {
tx->connp->out_tx = NULL;
}
}
bstr_free(tx->request_content_type);
bstr_free(tx->response_content_type);
// Parsers
htp_urlenp_destroy(tx->request_urlenp_query);
htp_urlenp_destroy(tx->request_urlenp_body);
htp_mpartp_destroy(tx->request_mpartp);
// Request parameters
htp_param_t *param = NULL;
for (int i = 0, n = htp_table_size(tx->request_params); i < n; i++) {
param = htp_table_get_index(tx->request_params, i, NULL);
free(param->name);
free(param->value);
free(param);
}
htp_table_destroy(tx->request_params);
// Request cookies
if (tx->request_cookies != NULL) {
bstr *b = NULL;
for (int i = 0, n = htp_table_size(tx->request_cookies); i < n; i++) {
b = htp_table_get_index(tx->request_cookies, i, NULL);
bstr_free(b);
}
htp_table_destroy(tx->request_cookies);
}
htp_hook_destroy(tx->hook_request_body_data);
// If we're using a private configuration, destroy it.
if (tx->is_config_shared == HTP_CONFIG_PRIVATE) {
htp_config_destroy(tx->cfg);
}
free(tx);
}
size_t htp_table_size(const htp_table_t *table) {
if (table == NULL) return 0;
return htp_list_size(table->list) / 2;
}