void *htp_table_get_index(const htp_table_t *table, size_t idx, bstr **key) {
if (table == NULL) return NULL;
if (idx >= htp_list_size(table->list)) return NULL;
if (key != NULL) {
*key = htp_list_get(table->list, idx * 2);
}
return htp_list_get(table->list, (idx * 2) + 1);
}
void *htp_table_get_mem(const htp_table_t *table, const void *key, size_t key_len) {
if ((table == NULL)||(key == NULL)) return NULL;
// Iterate through the list, comparing
// keys with the parameter, return data if found.
for (size_t i = 0, n = htp_list_size(table->list); i < n; i += 2) {
bstr *key_candidate = htp_list_get(table->list, i);
void *element = htp_list_get(table->list, i + 1);
if (bstr_cmp_mem_nocase(key_candidate, key, key_len) == 0) {
return element;
}
}
return NULL;
}
/**
* The response idle state will initialize response processing, as well as
* finalize each transactions after we are done with it.
*
* @param[in] connp
* @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed.
*/
htp_status_t htp_connp_RES_IDLE(htp_connp_t *connp) {
// We want to start parsing the next response (and change
// the state from IDLE) only if there's at least one
// byte of data available. Otherwise we could be creating
// new structures even if there's no more data on the
// connection.
OUT_TEST_NEXT_BYTE_OR_RETURN(connp);
// Parsing a new response
// Find the next outgoing transaction
connp->out_tx = htp_list_get(connp->conn->transactions, connp->out_next_tx_index);
if (connp->out_tx == NULL) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Unable to match response to request");
return HTP_ERROR;
}
// We've used one transaction
connp->out_next_tx_index++;
// TODO Detect state mismatch
connp->out_content_length = -1;
connp->out_body_data_left = -1;
int rc = htp_tx_state_response_start(connp->out_tx);
if (rc != HTP_OK) return rc;
return HTP_OK;
}
/**
* Free stream data.
*
* @param[in] sd
*/
void free_stream_data(stream_data *sd) {
if (sd == NULL) return;
// Free stream chunks, if any
if (sd->chunks != NULL) {
for (int i = 0, n = htp_list_size(sd->chunks); i < n; i++) {
chunk_t *chunk = htp_list_get(sd->chunks, i);
free(chunk->data);
free(chunk);
}
htp_list_destroy(sd->chunks);
sd->chunked = NULL;
}
// Free inbound chunks, if any
if (sd->inbound_chunks != NULL) {
for (int i = 0, n = htp_list_size(sd->inbound_chunks); i < n; i++) {
chunk_t *chunk = htp_list_get(sd->inbound_chunkds, i);
free(chunk->data);
free(chunk);
}
htp_list_destroy(sd->inbound_chunks);
sd->inbound_chunks = NULL;
}
// Free outbound chunks, if any
if (sd->outbound_chunks != NULL) {
for (int i = 0, n = htp_list_size(sd->outbound_chunks); i < n; i++) {
chunk_t *chunk = htp_list_get(sd->outbound_chunkds, i);
free(chunk->data);
free(chunk);
}
htp_list_destroy(sd->outbound_chunks);
sd->outbound_chunks = NULL;
}
// Close the stream file, if we have it open
if (sd->fd != -1) {
close(sd->fd);
}
free(sd);
}
/**
* Process as much buffered inbound and outbound data as possible
* (in that order)
*
* @param[in] sd
*/
void process_stored_stream_data(stream_data *sd) {
int loop = 0;
do {
// Reset loop
loop = 0;
// Send as much inbound data as possible
while((sd->connp->in_status == HTP_STREAM_DATA)&&(htp_list_size(sd->inbound_chunks) > 0)) {
chunk_t *chunk = (chunk_t *)htp_list_get(sd->inbound_chunks, 0);
int rc = htp_connp_req_data(sd->connp, 0, chunk->data + chunk->consumed, chunk->len - chunk->consumed);
if (rc == HTP_STREAM_DATA) {
// The entire chunk was consumed
htp_list_shift(sd->inbound_chunks);
free(chunk->data);
free(chunk);
} else {
// Partial consumption
chunk->consumed = htp_connp_req_data_consumed(sd->connp);
}
}
// Send as much outbound data as possible
while((sd->connp->out_status == HTP_STREAM_DATA)&&(htp_list_size(sd->outbound_chunks) > 0)) {
chunk_t *chunk = (chunk_t *)htp_list_get(sd->outbound_chunks, 0);
int rc = htp_connp_res_data(sd->connp, 0, chunk->data + chunk->consumed, chunk->len - chunk->consumed);
if (rc == HTP_STREAM_DATA) {
// The entire chunk was consumed
htp_list_shift(sd->outbound_chunks);
free(chunk->data);
free(chunk);
} else {
// Partial consumption
chunk->consumed = htp_connp_res_data_consumed(sd->connp);
}
// Whenever we send some outbound data to the parser,
// we need to go back and try to send inbound data
// if we have it.
loop = 1;
}
} while(loop);
}
void bstr_builder_clear(bstr_builder_t *bb) {
// Do nothing if the list is empty
if (htp_list_size(bb->pieces) == 0) return;
for (size_t i = 0, n = htp_list_size(bb->pieces); i < n; i++) {
bstr *b = htp_list_get(bb->pieces, i);
bstr_free(b);
}
htp_list_clear(bb->pieces);
}
void htp_hook_destroy(htp_hook_t *hook) {
if (hook == NULL) return;
for (size_t i = 0, n = htp_list_size(hook->callbacks); i < n; i++) {
free((htp_callback_t *) htp_list_get(hook->callbacks, i));
}
htp_list_array_destroy(hook->callbacks);
free(hook);
}
void htp_conn_destroy(htp_conn_t *conn) {
if (conn == NULL) return;
if (conn->transactions != NULL) {
// Destroy individual transactions. Do note that iterating
// using the iterator does not work here because some of the
// list element may be NULL (and with the iterator it is impossible
// to distinguish a NULL element from the end of the list).
for (size_t i = 0, n = htp_list_size(conn->transactions); i < n; i++) {
htp_tx_t *tx = htp_list_get(conn->transactions, i);
if (tx != NULL) {
htp_tx_destroy_incomplete(tx);
}
}
htp_list_destroy(conn->transactions);
conn->transactions = NULL;
}
if (conn->messages != NULL) {
// Destroy individual messages.
for (size_t i = 0, n = htp_list_size(conn->messages); i < n; i++) {
htp_log_t *l = htp_list_get(conn->messages, i);
free((void *) l->msg);
free(l);
}
htp_list_destroy(conn->messages);
conn->messages = NULL;
}
if (conn->server_addr != NULL) {
free(conn->server_addr);
}
if (conn->client_addr != NULL) {
free(conn->client_addr);
}
free(conn);
}
bstr *bstr_builder_to_str(const bstr_builder_t *bb) {
size_t len = 0;
// Determine the size of the string
for (size_t i = 0, n = htp_list_size(bb->pieces); i < n; i++) {
bstr *b = htp_list_get(bb->pieces, i);
len += bstr_len(b);
}
// Allocate string
bstr *bnew = bstr_alloc(len);
if (bnew == NULL) return NULL;
// Determine the size of the string
for (size_t i = 0, n = htp_list_size(bb->pieces); i < n; i++) {
bstr *b = htp_list_get(bb->pieces, i);
bstr_add_noex(bnew, b);
}
return bnew;
}
void bstr_builder_destroy(bstr_builder_t *bb) {
if (bb == NULL) return;
// Destroy any pieces we might have
for (size_t i = 0, n = htp_list_size(bb->pieces); i < n; i++) {
bstr *b = htp_list_get(bb->pieces, i);
bstr_free(b);
}
htp_list_destroy(bb->pieces);
free(bb);
}
static PyObject *htpy_connp_get_response_status(PyObject *self, PyObject *args) {
PyObject *ret;
htp_tx_t *tx = NULL;
tx = htp_list_get(((htpy_connp *) self)->connp->conn->transactions, htp_list_size(((htpy_connp *) self)->connp->conn->transactions) - 1);
if (!tx) {
PyErr_SetString(htpy_error, "Missing transaction.");
return NULL;
}
ret = Py_BuildValue("i", tx->response_status_number);
return ret;
}
/*
* XXX: Not sure I like mucking around in the transaction to get the method,
* but I'm not sure of a better way.
*/
static PyObject *htpy_connp_get_method(PyObject *self, PyObject *args) {
PyObject *ret;
htp_tx_t *tx = NULL;
tx = htp_list_get(((htpy_connp *) self)->connp->conn->transactions, htp_list_size(((htpy_connp *) self)->connp->conn->transactions) - 1);
if (!tx || !tx->request_method) {
PyErr_SetString(htpy_error, "Missing transaction or request method.");
return NULL;
}
ret = Py_BuildValue("s#", bstr_ptr(tx->request_method), bstr_len(tx->request_method));
return ret;
}
htp_status_t htp_conn_remove_tx(htp_conn_t *conn, const htp_tx_t *tx) {
if ((tx == NULL) || (conn == NULL)) return HTP_ERROR;
if (conn->transactions == NULL) return HTP_ERROR;
for (size_t i = 0, n = htp_list_size(conn->transactions); i < n; i++) {
htp_tx_t *candidate_tx = htp_list_get(conn->transactions, i);
if (tx == candidate_tx) {
htp_list_replace(conn->transactions, i, NULL);
return HTP_OK;
}
}
return HTP_DECLINED;
}
void htp_table_clear(htp_table_t *table) {
if (table == NULL) return;
// Free the table keys, but only if we're managing them.
if ((table->alloc_type == HTP_TABLE_KEYS_COPIED)||(table->alloc_type == HTP_TABLE_KEYS_ADOPTED)) {
bstr *key = NULL;
for (size_t i = 0, n = htp_list_size(table->list); i < n; i += 2) {
key = htp_list_get(table->list, i);
bstr_free(key);
}
}
htp_list_clear(table->list);
}
/**
* Invoked every time LibHTP wants to log.
*
* @param[in] log
*/
int callback_log(htp_log_t *log) {
stream_data *sd = (stream_data *)htp_connp_get_user_data(log->connp);
if ((sd->log_level == -1)||(sd->log_level > log->level)) {
sd->log_level = log->level;
}
if (log->code != 0) {
fprintf(stderr, "[#%d/%d][%d][code %d][file %s][line %d] %s\n", sd->id, sd->req_count,
log->level, log->code, log->file, log->line, log->msg);
} else {
fprintf(stderr, "[#%d/%d][%d][file %s][line %d] %s\n", sd->id, sd->req_count,
log->level, log->file, log->line, log->msg);
}
// If this is the first time a log message was generated for this connection,
// start writing the entire thing to a file on disk.
if (sd->fd == -1) {
char filename[256];
// TODO Use IP addresses and ports in filename
snprintf(filename, 255, "conn-%d.t", sd->id);
sd->fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR);
if (sd->fd == -1) {
fprintf(stderr, "Failed to create file %s: %s\n", filename, strerror(errno));
exit(1);
}
// Write to disk the data we have in memory
for (int i = 0, n = htp_list_size(sd->chunks); i < n; i++) {
chunk_t *chunk = htp_list_get(sd->chunks, i);
if (sd->chunk_counter != 0) {
write(sd->fd, "\r\n", 2);
}
if (sd->direction == chunk->direction) {
write(sd->fd, ">>>\r\n", 5);
} else {
write(sd->fd, "<<<\r\n", 5);
}
write(sd->fd, chunk->data, chunk->len);
sd->chunk_counter++;
}
}
}
htp_hook_t *htp_hook_copy(const htp_hook_t *hook) {
if (hook == NULL) return NULL;
htp_hook_t *copy = htp_hook_create();
if (copy == NULL) return NULL;
for (size_t i = 0, n = htp_list_size(hook->callbacks); i < n; i++) {
htp_callback_t *callback = htp_list_get(hook->callbacks, i);
if (htp_hook_register(©, callback->fn) != HTP_OK) {
htp_hook_destroy(copy);
return NULL;
}
}
return copy;
}
htp_status_t htp_hook_run_all(htp_hook_t *hook, void *user_data) {
if (hook == NULL) return HTP_OK;
// Loop through the registered callbacks, giving each a chance to run.
for (size_t i = 0, n = htp_list_size(hook->callbacks); i < n; i++) {
htp_callback_t *callback = htp_list_get(hook->callbacks, i);
htp_status_t rc = callback->fn(user_data);
// A hook can return HTP_OK to say that it did some work,
// or HTP_DECLINED to say that it did no work. Anything else
// is treated as an error.
if ((rc != HTP_OK) && (rc != HTP_DECLINED)) {
return rc;
}
}
return HTP_OK;
}
/**
* Finalize Multipart processing.
*
* @param[in] d
* @return HTP_OK on success, HTP_ERROR on failure.
*/
htp_status_t htp_ch_multipart_callback_request_body_data(htp_tx_data_t *d) {
htp_tx_t *tx = d->tx;
// Check that we were not invoked again after the finalization.
if (tx->request_mpartp->gave_up_data == 1) return HTP_ERROR;
if (d->data != NULL) {
// Process one chunk of data.
htp_mpartp_parse(tx->request_mpartp, d->data, d->len);
} else {
// Finalize parsing.
htp_mpartp_finalize(tx->request_mpartp);
htp_multipart_t *body = htp_mpartp_get_multipart(tx->request_mpartp);
for (size_t i = 0, n = htp_list_size(body->parts); i < n; i++) {
htp_multipart_part_t *part = htp_list_get(body->parts, i);
// Use text parameters.
if (part->type == MULTIPART_PART_TEXT) {
htp_param_t *param = calloc(1, sizeof (htp_param_t));
if (param == NULL) return HTP_ERROR;
param->name = part->name;
param->value = part->value;
param->source = HTP_SOURCE_BODY;
param->parser_id = HTP_PARSER_MULTIPART;
param->parser_data = part;
if (htp_tx_req_add_param(tx, param) != HTP_OK) {
free(param);
return HTP_ERROR;
}
}
}
// Tell the parser that it no longer owns names
// and values of MULTIPART_PART_TEXT parts.
tx->request_mpartp->gave_up_data = 1;
}
return HTP_OK;
}
htp_status_t htp_hook_run_one(htp_hook_t *hook, void *user_data) {
if (hook == NULL) return HTP_DECLINED;
for (size_t i = 0, n = htp_list_size(hook->callbacks); i < n; i++) {
htp_callback_t *callback = htp_list_get(hook->callbacks, i);
htp_status_t rc = callback->fn(user_data);
// A hook can return HTP_DECLINED to say that it did no work,
// and we'll ignore that. If we see HTP_OK or anything else,
// we stop processing (because it was either a successful
// handling or an error).
if (rc != HTP_DECLINED) {
// Return HTP_OK or an error.
return rc;
}
}
// No hook wanted to process the callback.
return HTP_DECLINED;
}
/**
* Extract one request header. A header can span multiple lines, in
* which case they will be folded into one before parsing is attempted.
*
* @param[in] connp
* @return HTP_OK or HTP_ERROR
*/
int htp_process_request_header_generic(htp_connp_t *connp) {
bstr *tempstr = NULL;
unsigned char *data = NULL;
size_t len = 0;
// Create new header structure
htp_header_t *h = calloc(1, sizeof (htp_header_t));
if (h == NULL) {
return HTP_ERROR;
}
// Ensure we have the necessary header data in a single buffer
if (connp->in_header_line_index + 1 == connp->in_header_line_counter) {
// Single line
htp_header_line_t *hl = htp_list_get(connp->in_tx->request_header_lines,
connp->in_header_line_index);
if (hl == NULL) {
// Internal error
free(h);
return HTP_ERROR;
}
data = (unsigned char *)bstr_ptr(hl->line);
len = bstr_len(hl->line);
hl->header = h;
} else {
// Multiple lines (folded)
int i = 0;
for (i = connp->in_header_line_index; i < connp->in_header_line_counter; i++) {
htp_header_line_t *hl = htp_list_get(connp->in_tx->request_header_lines, i);
len += bstr_len(hl->line);
}
tempstr = bstr_alloc(len);
if (tempstr == NULL) {
free(h);
return HTP_ERROR;
}
for (i = connp->in_header_line_index; i < connp->in_header_line_counter; i++) {
htp_header_line_t *hl = htp_list_get(connp->in_tx->request_header_lines, i);
unsigned char *line = bstr_ptr(hl->line);
size_t llen = bstr_len(hl->line);
htp_chomp((unsigned char *)line, &llen);
bstr_add_mem_noex(tempstr, line, llen);
hl->header = h;
if (i != connp->in_header_line_index) {
hl->flags |= HTP_FIELD_FOLDED;
}
}
h->flags |= HTP_FIELD_FOLDED;
data = (unsigned char *)bstr_ptr(tempstr);
len = bstr_len(tempstr);
}
// Now try to parse the header
if (htp_parse_request_header_generic(connp, h, data, len) != HTP_OK) {
bstr_free(tempstr);
free(h);
return HTP_ERROR;
}
// Do we already have a header with the same name?
htp_header_t *h_existing = htp_table_get(connp->in_tx->request_headers, h->name);
if (h_existing != NULL) {
// repeated header
int i = 0;
// TODO Do we want to keep a list of the headers that are
// allowed to be combined in this way?
// Add to existing header
bstr *new_value = bstr_expand(h_existing->value, bstr_len(h_existing->value)
+ 2 + bstr_len(h->value));
if (new_value == NULL) {
bstr_free(h->name);
bstr_free(h->value);
free(h);
bstr_free(tempstr);
return HTP_ERROR;
}
h_existing->value = new_value;
bstr_add_mem_noex(h_existing->value, (unsigned char *)", ", 2);
bstr_add_noex(h_existing->value, h->value);
// replace the header references in all lines
for (i = connp->in_header_line_index; i < connp->in_header_line_counter; i++) {
htp_header_line_t *hl = htp_list_get(connp->in_tx->request_header_lines, i);
hl->header = h_existing;
}
// The header fields are no longer needed
bstr_free(h->name);
bstr_free(h->value);
free(h);
// Keep track of same-name headers
h_existing->flags |= HTP_FIELD_REPEATED;
} else {
// Add as a new header
htp_table_add(connp->in_tx->request_headers, h->name, h);
}
bstr_free(tempstr);
return HTP_OK;
}
void htp_tx_destroy(htp_tx_t *tx) {
bstr_free(tx->request_line);
bstr_free(tx->request_line_raw);
bstr_free(tx->request_method);
bstr_free(tx->request_uri);
bstr_free(tx->request_uri_normalized);
bstr_free(tx->request_protocol);
bstr_free(tx->request_headers_sep);
if (tx->parsed_uri != NULL) {
bstr_free(tx->parsed_uri->scheme);
bstr_free(tx->parsed_uri->username);
bstr_free(tx->parsed_uri->password);
bstr_free(tx->parsed_uri->hostname);
bstr_free(tx->parsed_uri->port);
bstr_free(tx->parsed_uri->path);
bstr_free(tx->parsed_uri->query);
bstr_free(tx->parsed_uri->fragment);
free(tx->parsed_uri);
}
if (tx->parsed_uri_incomplete != NULL) {
bstr_free(tx->parsed_uri_incomplete->scheme);
bstr_free(tx->parsed_uri_incomplete->username);
bstr_free(tx->parsed_uri_incomplete->password);
bstr_free(tx->parsed_uri_incomplete->hostname);
bstr_free(tx->parsed_uri_incomplete->port);
bstr_free(tx->parsed_uri_incomplete->path);
bstr_free(tx->parsed_uri_incomplete->query);
bstr_free(tx->parsed_uri_incomplete->fragment);
free(tx->parsed_uri_incomplete);
}
// Destroy request_header_lines.
if (tx->request_header_lines != NULL) {
for (int i = 0, n = htp_list_size(tx->request_header_lines); i < n; i++) {
htp_header_line_t *hl = htp_list_get(tx->request_header_lines, i);
bstr_free(hl->line);
// No need to destroy hl->header because
// htp_header_line_t does not own it.
free(hl);
}
htp_list_destroy(tx->request_header_lines);
tx->request_header_lines = NULL;
}
// Destroy request_headers.
if (tx->request_headers != NULL) {
htp_header_t *h = NULL;
for (int i = 0, n = htp_table_size(tx->request_headers); i < n; i++) {
h = htp_table_get_index(tx->request_headers, i, NULL);
bstr_free(h->name);
bstr_free(h->value);
free(h);
}
htp_table_destroy(tx->request_headers);
}
if (tx->request_headers_raw != NULL) {
bstr_free(tx->request_headers_raw);
}
if (tx->response_headers_raw != NULL) {
bstr_free(tx->response_headers_raw);
}
bstr_free(tx->response_line);
bstr_free(tx->response_line_raw);
bstr_free(tx->response_protocol);
bstr_free(tx->response_status);
bstr_free(tx->response_message);
bstr_free(tx->response_headers_sep);
// Destroy response_header_lines.
if (tx->response_header_lines != NULL) {
for (int i = 0, n = htp_list_size(tx->response_header_lines); i < n; i++) {
htp_header_line_t *hl = htp_list_get(tx->response_header_lines, i);
bstr_free(hl->line);
// No need to destroy hl->header because
// htp_header_line_t does not own it.
free(hl);
}
htp_list_destroy(tx->response_header_lines);
tx->response_header_lines = NULL;
}
// Destroy response headers.
if (tx->response_headers != NULL) {
htp_header_t *h = NULL;
for (int i = 0, n = htp_table_size(tx->response_headers); i < n; i++) {
h = htp_table_get_index(tx->response_headers, i, NULL);
bstr_free(h->name);
bstr_free(h->value);
free(h);
}
htp_table_destroy(tx->response_headers);
}
// Tell the connection to remove this transaction from the list.
htp_conn_remove_tx(tx->conn, tx);
// Invalidate the pointer to this transactions held
// by the connection parser. This is to allow a transaction
// to be destroyed from within the final response callback.
if (tx->connp != NULL) {
if (tx->connp->out_tx == tx) {
tx->connp->out_tx = NULL;
}
}
bstr_free(tx->request_content_type);
bstr_free(tx->response_content_type);
// Parsers
htp_urlenp_destroy(tx->request_urlenp_query);
htp_urlenp_destroy(tx->request_urlenp_body);
htp_mpartp_destroy(tx->request_mpartp);
// Request parameters
htp_param_t *param = NULL;
for (int i = 0, n = htp_table_size(tx->request_params); i < n; i++) {
param = htp_table_get_index(tx->request_params, i, NULL);
free(param->name);
free(param->value);
free(param);
}
htp_table_destroy(tx->request_params);
// Request cookies
if (tx->request_cookies != NULL) {
bstr *b = NULL;
for (int i = 0, n = htp_table_size(tx->request_cookies); i < n; i++) {
b = htp_table_get_index(tx->request_cookies, i, NULL);
bstr_free(b);
}
htp_table_destroy(tx->request_cookies);
}
htp_hook_destroy(tx->hook_request_body_data);
// If we're using a private configuration, destroy it.
if (tx->is_config_shared == HTP_CONFIG_PRIVATE) {
htp_config_destroy(tx->cfg);
}
free(tx);
}
