static void LogHttpLogExtended(LogHttpLogThread *aft, htp_tx_t *tx)
{
MemBufferWriteString(aft->buffer, " [**] ");
/* referer */
htp_header_t *h_referer = NULL;
if (tx->request_headers != NULL) {
h_referer = htp_table_get_c(tx->request_headers, "referer");
}
if (h_referer != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,
(uint8_t *)bstr_ptr(h_referer->value),
bstr_len(h_referer->value));
} else {
MemBufferWriteString(aft->buffer, "<no referer>");
}
MemBufferWriteString(aft->buffer, " [**] ");
/* method */
if (tx->request_method != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,
(uint8_t *)bstr_ptr(tx->request_method),
bstr_len(tx->request_method));
}
MemBufferWriteString(aft->buffer, " [**] ");
/* protocol */
if (tx->request_protocol != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,
(uint8_t *)bstr_ptr(tx->request_protocol),
bstr_len(tx->request_protocol));
} else {
MemBufferWriteString(aft->buffer, "<no protocol>");
}
MemBufferWriteString(aft->buffer, " [**] ");
/* response status */
if (tx->response_status != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,
(uint8_t *)bstr_ptr(tx->response_status),
bstr_len(tx->response_status));
/* Redirect? */
if ((tx->response_status_number > 300) && ((tx->response_status_number) < 303)) {
htp_header_t *h_location = htp_table_get_c(tx->response_headers, "location");
if (h_location != NULL) {
MemBufferWriteString(aft->buffer, " => ");
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,
(uint8_t *)bstr_ptr(h_location->value),
bstr_len(h_location->value));
}
}
} else {
MemBufferWriteString(aft->buffer, "<no status>");
}
/* length */
MemBufferWriteString(aft->buffer, " [**] %"PRIuMAX" bytes", (uintmax_t)tx->response_message_len);
}
/**
* Parses Cookie request header in v0 format.
*
* @param[in] connp
* @return HTP_OK on success, HTP_ERROR on error
*/
htp_status_t htp_parse_cookies_v0(htp_connp_t *connp) {
htp_header_t *cookie_header = htp_table_get_c(connp->in_tx->request_headers, "cookie");
if (cookie_header == NULL) return HTP_OK;
// Create a new table to store cookies
connp->in_tx->request_cookies = htp_table_create(4);
if (connp->in_tx->request_cookies == NULL) return HTP_ERROR;
unsigned char *data = bstr_ptr(cookie_header->value);
size_t len = bstr_len(cookie_header->value);
size_t pos = 0;
while (pos < len) {
// Ignore whitespace at the beginning
while ((pos < len) && (isspace((int)data[pos]))) pos++;
if (pos == len) return HTP_OK;
size_t start = pos;
// Find the end of the cookie
while ((pos < len) && (data[pos] != ';')) pos++;
if (htp_parse_single_cookie_v0(connp, data + start, pos - start) != HTP_OK) {
return HTP_ERROR;
}
// Go over the semicolon
if (pos != len) pos++;
}
return HTP_OK;
}
static int HttpGetHeader(lua_State *luastate, int dir)
{
if (!(LuaStateNeedProto(luastate, ALPROTO_HTTP)))
return LuaCallbackError(luastate, "error: protocol not http");
htp_tx_t *tx = LuaStateGetTX(luastate);
if (tx == NULL)
return LuaCallbackError(luastate, "internal error: no tx");
const char *name = LuaGetStringArgument(luastate, 1);
if (name == NULL)
return LuaCallbackError(luastate, "1st argument missing, empty or wrong type");
htp_table_t *headers = tx->request_headers;
if (dir == 1)
headers = tx->response_headers;
if (headers == NULL)
return LuaCallbackError(luastate, "tx has no headers");
htp_header_t *h = (htp_header_t *)htp_table_get_c(headers, name);
if (h == NULL || bstr_len(h->value) == 0)
return LuaCallbackError(luastate, "header not found");
return LuaPushStringBuffer(luastate,
bstr_ptr(h->value), bstr_len(h->value));
}
htp_status_t htp_tx_state_response_headers(htp_tx_t *tx) {
if (tx == NULL) return HTP_ERROR;
// Check for compression.
// Determine content encoding.
tx->response_content_encoding = HTP_COMPRESSION_NONE;
htp_header_t *ce = htp_table_get_c(tx->response_headers, "content-encoding");
if (ce != NULL) {
if ((bstr_cmp_c(ce->value, "gzip") == 0) || (bstr_cmp_c(ce->value, "x-gzip") == 0)) {
tx->response_content_encoding = HTP_COMPRESSION_GZIP;
} else if ((bstr_cmp_c(ce->value, "deflate") == 0) || (bstr_cmp_c(ce->value, "x-deflate") == 0)) {
tx->response_content_encoding = HTP_COMPRESSION_DEFLATE;
}
}
// Configure decompression, if enabled in the configuration.
if (tx->connp->cfg->response_decompression_enabled) {
tx->response_content_encoding_processing = tx->response_content_encoding;
} else {
tx->response_content_encoding_processing = HTP_COMPRESSION_NONE;
}
// Finalize sending raw header data.
htp_status_t rc = htp_connp_res_receiver_finalize_clear(tx->connp);
if (rc != HTP_OK) return rc;
// Run hook RESPONSE_HEADERS.
rc = htp_hook_run_all(tx->connp->cfg->hook_response_headers, tx);
if (rc != HTP_OK) return rc;
// Initialize the decompression engine as necessary. We can deal with three
// scenarios:
//
// 1. Decompression is enabled, compression indicated in headers, and we decompress.
//
// 2. As above, but the user disables decompression by setting response_content_encoding
// to COMPRESSION_NONE.
//
// 3. Decompression is disabled and we do not attempt to enable it, but the user
// forces decompression by setting response_content_encoding to one of the
// supported algorithms.
if ((tx->response_content_encoding_processing == HTP_COMPRESSION_GZIP) || (tx->response_content_encoding_processing == HTP_COMPRESSION_DEFLATE)) {
if (tx->connp->out_decompressor != NULL) {
tx->connp->out_decompressor->destroy(tx->connp->out_decompressor);
tx->connp->out_decompressor = NULL;
}
tx->connp->out_decompressor = (htp_decompressor_t *) htp_gzip_decompressor_create(tx->connp,
tx->response_content_encoding_processing);
if (tx->connp->out_decompressor == NULL) return HTP_ERROR;
tx->connp->out_decompressor->callback = htp_tx_res_process_body_data_decompressor_callback;
} else if (tx->response_content_encoding_processing != HTP_COMPRESSION_NONE) {
return HTP_ERROR;
}
return HTP_OK;
}
/**
* Inspect request headers and register the Multipart request data hook
* if it contains a multipart/form-data body.
*
* @param[in] connp
* @return HTP_OK if a new parser has been setup, HTP_DECLINED if the MIME type
* is not appropriate for this parser, and HTP_ERROR on failure.
*/
htp_status_t htp_ch_multipart_callback_request_headers(htp_tx_t *tx) {
#ifdef HTP_DEBUG
fprintf(stderr, "htp_ch_multipart_callback_request_headers: Need to determine if multipart body is present\n");
#endif
// The field tx->request_content_type does not contain the entire C-T
// value and so we cannot use it to look for a boundary, but we can
// use it for a quick check to determine if the C-T header exists.
if (tx->request_content_type == NULL) {
#ifdef HTP_DEBUG
fprintf(stderr, "htp_ch_multipart_callback_request_headers: Not multipart body (no C-T header)\n");
#endif
return HTP_DECLINED;
}
// Look for a boundary.
htp_header_t *ct = htp_table_get_c(tx->request_headers, "content-type");
if (ct == NULL) return HTP_ERROR;
bstr *boundary = NULL;
uint64_t flags = 0;
htp_status_t rc = htp_mpartp_find_boundary(ct->value, &boundary, &flags);
if (rc != HTP_OK) {
#ifdef HTP_DEBUG
if (rc == HTP_DECLINED) {
fprintf(stderr, "htp_ch_multipart_callback_request_headers: Not multipart body\n");
}
#endif
// No boundary (HTP_DECLINED) or error (HTP_ERROR).
return rc;
}
if (boundary == NULL) return HTP_ERROR;
// Create a Multipart parser instance.
tx->request_mpartp = htp_mpartp_create(tx->connp->cfg, boundary, flags);
if (tx->request_mpartp == NULL) {
bstr_free(boundary);
return HTP_ERROR;
}
// Configure file extraction.
if (tx->cfg->extract_request_files) {
tx->request_mpartp->extract_files = 1;
tx->request_mpartp->extract_dir = tx->connp->cfg->tmpdir;
}
// Register a request body data callback.
htp_tx_register_request_body_data(tx, htp_ch_multipart_callback_request_body_data);
return HTP_OK;
}
static void LogFileMetaGetUserAgent(FILE *fp, Packet *p, File *ff) {
HtpState *htp_state = (HtpState *)p->flow->alstate;
if (htp_state != NULL) {
htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
if (tx != NULL) {
htp_header_t *h = NULL;
h = (htp_header_t *)htp_table_get_c(tx->request_headers,
"User-Agent");
if (h != NULL) {
PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
bstr_len(h->value));
return;
}
}
}
fprintf(fp, "<unknown>");
}
static json_t *LogFileMetaGetUserAgent(const Packet *p, const File *ff) {
HtpState *htp_state = (HtpState *)p->flow->alstate;
json_t *js = NULL;
if (htp_state != NULL) {
htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
if (tx != NULL) {
htp_header_t *h = NULL;
h = (htp_header_t *)htp_table_get_c(tx->request_headers,
"User-Agent");
if (h != NULL) {
char *s = bstr_util_strdup_to_c(h->value);
if (s != NULL) {
js = json_string(s);
SCFree(s);
}
return js;
}
}
}
return json_string("<unknown>");
}
/**
* Parses Authorization request header.
*
* @param[in] connp
*/
int htp_parse_authorization(htp_connp_t *connp) {
htp_header_t *auth_header = htp_table_get_c(connp->in_tx->request_headers, "authorization");
if (auth_header == NULL) {
connp->in_tx->request_auth_type = HTP_AUTH_NONE;
return HTP_OK;
}
if (bstr_begins_with_c_nocase(auth_header->value, "basic")) {
// Basic authentication
connp->in_tx->request_auth_type = HTP_AUTH_BASIC;
return htp_parse_authorization_basic(connp, auth_header);
} else if (bstr_begins_with_c_nocase(auth_header->value, "digest")) {
// Digest authentication
connp->in_tx->request_auth_type = HTP_AUTH_DIGEST;
return htp_parse_authorization_digest(connp, auth_header);
} else {
// Unrecognized authentication method
connp->in_tx->request_auth_type = HTP_AUTH_UNRECOGNIZED;
}
return HTP_OK;
}
/**
* Determines presence (and encoding) of a response body.
*
* @param[in] connp
* @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed.
*/
htp_status_t htp_connp_RES_BODY_DETERMINE(htp_connp_t *connp) {
// If the request uses the CONNECT method, then not only are we
// to assume there's no body, but we need to ignore all
// subsequent data in the stream.
if (connp->out_tx->request_method_number == HTP_M_CONNECT) {
if ((connp->out_tx->response_status_number >= 200)
&& (connp->out_tx->response_status_number <= 299)) {
// This is a successful CONNECT stream, which means
// we need to switch into tunnelling mode.
connp->in_status = HTP_STREAM_TUNNEL;
connp->out_status = HTP_STREAM_TUNNEL;
connp->out_state = htp_connp_RES_FINALIZE;
return HTP_OK;
} else {
// This is a failed CONNECT stream, which means that
// we can unblock request parsing
connp->in_status = HTP_STREAM_DATA;
// We are going to continue processing this transaction,
// adding a note for ourselves to stop at the end (because
// we don't want to see the beginning of a new transaction).
connp->out_data_other_at_tx_end = 1;
}
}
// Check for an interim "100 Continue" response. Ignore it if found, and revert back to RES_FIRST_LINE.
if (connp->out_tx->response_status_number == 100) {
if (connp->out_tx->seen_100continue != 0) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Already seen 100-Continue.");
return HTP_ERROR;
}
// Ignore any response headers seen so far.
htp_header_t *h = NULL;
for (int i = 0, n = htp_table_size(connp->out_tx->response_headers); i < n; i++) {
h = htp_table_get_index(connp->out_tx->response_headers, i, NULL);
bstr_free(h->name);
bstr_free(h->value);
free(h);
}
htp_table_clear(connp->out_tx->response_headers);
// Expecting to see another response line next.
connp->out_state = htp_connp_RES_LINE;
connp->out_tx->progress = HTP_RESPONSE_LINE;
connp->out_tx->seen_100continue++;
return HTP_OK;
}
// 1. Any response message which MUST NOT include a message-body
// (such as the 1xx, 204, and 304 responses and any response to a HEAD
// request) is always terminated by the first empty line after the
// header fields, regardless of the entity-header fields present in the
// message.
if (((connp->out_tx->response_status_number >= 100) && (connp->out_tx->response_status_number <= 199))
|| (connp->out_tx->response_status_number == 204) || (connp->out_tx->response_status_number == 304)
|| (connp->out_tx->request_method_number == HTP_M_HEAD)) {
// There's no response body
connp->out_tx->response_transfer_coding = HTP_CODING_NO_BODY;
connp->out_state = htp_connp_RES_FINALIZE;
} else {
// We have a response body
htp_header_t *ct = htp_table_get_c(connp->out_tx->response_headers, "content-type");
htp_header_t *cl = htp_table_get_c(connp->out_tx->response_headers, "content-length");
htp_header_t *te = htp_table_get_c(connp->out_tx->response_headers, "transfer-encoding");
if (ct != NULL) {
connp->out_tx->response_content_type = bstr_dup_lower(ct->value);
if (connp->out_tx->response_content_type == NULL) return HTP_ERROR;
// Ignore parameters
unsigned char *data = bstr_ptr(connp->out_tx->response_content_type);
size_t len = bstr_len(ct->value);
size_t newlen = 0;
while (newlen < len) {
// TODO Some platforms may do things differently here.
if (htp_is_space(data[newlen]) || (data[newlen] == ';')) {
bstr_adjust_len(connp->out_tx->response_content_type, newlen);
break;
}
newlen++;
}
}
// 2. If a Transfer-Encoding header field (section 14.40) is present and
// indicates that the "chunked" transfer coding has been applied, then
// the length is defined by the chunked encoding (section 3.6).
if ((te != NULL) && (bstr_cmp_c(te->value, "chunked") == 0)) {
// If the T-E header is present we are going to use it.
connp->out_tx->response_transfer_coding = HTP_CODING_CHUNKED;
// We are still going to check for the presence of C-L
if (cl != NULL) {
// This is a violation of the RFC
connp->out_tx->flags |= HTP_REQUEST_SMUGGLING;
}
connp->out_state = htp_connp_RES_BODY_CHUNKED_LENGTH;
connp->out_tx->progress = HTP_RESPONSE_BODY;
}// 3. If a Content-Length header field (section 14.14) is present, its
// value in bytes represents the length of the message-body.
else if (cl != NULL) {
// We know the exact length
connp->out_tx->response_transfer_coding = HTP_CODING_IDENTITY;
// Check for multiple C-L headers
if (cl->flags & HTP_FIELD_REPEATED) {
connp->out_tx->flags |= HTP_REQUEST_SMUGGLING;
}
// Get body length
connp->out_tx->response_content_length = htp_parse_content_length(cl->value);
if (connp->out_tx->response_content_length < 0) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Invalid C-L field in response: %d",
connp->out_tx->response_content_length);
return HTP_ERROR;
} else {
connp->out_content_length = connp->out_tx->response_content_length;
connp->out_body_data_left = connp->out_content_length;
if (connp->out_content_length != 0) {
connp->out_state = htp_connp_RES_BODY_IDENTITY_CL_KNOWN;
connp->out_tx->progress = HTP_RESPONSE_BODY;
} else {
connp->out_state = htp_connp_RES_FINALIZE;
}
}
} else {
// 4. If the message uses the media type "multipart/byteranges", which is
// self-delimiting, then that defines the length. This media type MUST
// NOT be used unless the sender knows that the recipient can parse it;
// the presence in a request of a Range header with multiple byte-range
// specifiers implies that the client can parse multipart/byteranges
// responses.
if (ct != NULL) {
// TODO Handle multipart/byteranges
if (bstr_index_of_c_nocase(ct->value, "multipart/byteranges") != -1) {
htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
"C-T multipart/byteranges in responses not supported");
return HTP_ERROR;
}
}
// 5. By the server closing the connection. (Closing the connection
// cannot be used to indicate the end of a request body, since that
// would leave no possibility for the server to send back a response.)
connp->out_state = htp_connp_RES_BODY_IDENTITY_STREAM_CLOSE;
connp->out_tx->response_transfer_coding = HTP_CODING_IDENTITY;
connp->out_tx->progress = HTP_RESPONSE_BODY;
connp->out_body_data_left = -1;
}
}
// NOTE We do not need to check for short-style HTTP/0.9 requests here because
// that is done earlier, before response line parsing begins
int rc = htp_tx_state_response_headers(connp->out_tx);
if (rc != HTP_OK) return rc;
return HTP_OK;
}
/* JSON format logging */
static void JsonHttpLogJSON(JsonHttpLogThread *aft, json_t *js, htp_tx_t *tx)
{
LogHttpFileCtx *http_ctx = aft->httplog_ctx;
json_t *hjs = json_object();
if (hjs == NULL) {
return;
}
char *c;
/* hostname */
if (tx->request_hostname != NULL)
{
c = bstr_util_strdup_to_c(tx->request_hostname);
if (c != NULL) {
json_object_set_new(hjs, "hostname", json_string(c));
SCFree(c);
}
} else {
json_object_set_new(hjs, "hostname", json_string("<unknown>"));
}
/* uri */
if (tx->request_uri != NULL)
{
c = bstr_util_strdup_to_c(tx->request_uri);
if (c != NULL) {
json_object_set_new(hjs, "url", json_string(c));
SCFree(c);
}
}
/* user agent */
htp_header_t *h_user_agent = NULL;
if (tx->request_headers != NULL) {
h_user_agent = htp_table_get_c(tx->request_headers, "user-agent");
}
if (h_user_agent != NULL) {
c = bstr_util_strdup_to_c(h_user_agent->value);
if (c != NULL) {
json_object_set_new(hjs, "http_user_agent", json_string(c));
SCFree(c);
}
} else {
json_object_set_new(hjs, "http_user_agent", json_string("unknown>"));
}
/* x-forwarded-for */
htp_header_t *h_x_forwarded_for = NULL;
if (tx->request_headers != NULL) {
h_x_forwarded_for = htp_table_get_c(tx->request_headers, "x-forwarded-for");
}
if (h_x_forwarded_for != NULL) {
c = bstr_util_strdup_to_c(h_x_forwarded_for->value);
if (c != NULL) {
json_object_set_new(hjs, "xff", json_string(c));
SCFree(c);
}
}
/* content-type */
htp_header_t *h_content_type = NULL;
if (tx->response_headers != NULL) {
h_content_type = htp_table_get_c(tx->response_headers, "content-type");
}
if (h_content_type != NULL) {
char *p;
c = bstr_util_strdup_to_c(h_content_type->value);
if (c != NULL) {
p = strchr(c, ';');
if (p != NULL)
*p = '\0';
json_object_set_new(hjs, "http_content_type", json_string(c));
SCFree(c);
}
}
if (http_ctx->flags & LOG_HTTP_EXTENDED) {
/* referer */
htp_header_t *h_referer = NULL;
if (tx->request_headers != NULL) {
h_referer = htp_table_get_c(tx->request_headers, "referer");
}
if (h_referer != NULL) {
c = bstr_util_strdup_to_c(h_referer->value);
if (c != NULL) {
json_object_set_new(hjs, "http_refer", json_string(c));
SCFree(c);
}
}
/* method */
if (tx->request_method != NULL) {
c = bstr_util_strdup_to_c(tx->request_method);
if (c != NULL) {
json_object_set_new(hjs, "http_method", json_string(c));
SCFree(c);
}
}
/* protocol */
if (tx->request_protocol != NULL) {
c = bstr_util_strdup_to_c(tx->request_protocol);
if (c != NULL) {
json_object_set_new(hjs, "protocol", json_string(c));
SCFree(c);
}
}
/* response status */
if (tx->response_status != NULL) {
c = bstr_util_strdup_to_c(tx->response_status);
if (c != NULL) {
json_object_set_new(hjs, "status", json_string(c));
SCFree(c);
}
htp_header_t *h_location = htp_table_get_c(tx->response_headers, "location");
if (h_location != NULL) {
c = bstr_util_strdup_to_c(h_location->value);
if (c != NULL) {
json_object_set_new(hjs, "redirect", json_string(c));
SCFree(c);
}
}
}
/* length */
json_object_set_new(hjs, "length", json_integer(tx->response_message_len));
}
json_object_set_new(js, "http", hjs);
}
static TmEcode LogHttpLogIPWrapper(ThreadVars *tv, void *data, const Packet *p, Flow *f, HtpState *htp_state, htp_tx_t *tx, uint64_t tx_id, int ipproto)
{
SCEnter();
LogHttpLogThread *aft = (LogHttpLogThread *)data;
LogHttpFileCtx *hlog = aft->httplog_ctx;
char timebuf[64];
/* check if we have HTTP state or not */
CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
char srcip[46], dstip[46];
Port sp, dp;
if ((PKT_IS_TOSERVER(p))) {
switch (ipproto) {
case AF_INET:
PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip));
PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip));
break;
case AF_INET6:
PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip));
PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip));
break;
default:
goto end;
}
sp = p->sp;
dp = p->dp;
} else {
switch (ipproto) {
case AF_INET:
PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), srcip, sizeof(srcip));
PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), dstip, sizeof(dstip));
break;
case AF_INET6:
PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), srcip, sizeof(srcip));
PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), dstip, sizeof(dstip));
break;
default:
goto end;
}
sp = p->dp;
dp = p->sp;
}
SCLogDebug("got a HTTP request and now logging !!");
/* reset */
MemBufferReset(aft->buffer);
if (hlog->flags & LOG_HTTP_CUSTOM) {
LogHttpLogCustom(aft, tx, &p->ts, srcip, sp, dstip, dp);
} else {
/* time */
MemBufferWriteString(aft->buffer, "%s ", timebuf);
/* hostname */
if (tx->request_hostname != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,
(uint8_t *)bstr_ptr(tx->request_hostname),
bstr_len(tx->request_hostname));
} else {
MemBufferWriteString(aft->buffer, "<hostname unknown>");
}
MemBufferWriteString(aft->buffer, " [**] ");
/* uri */
if (tx->request_uri != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,
(uint8_t *)bstr_ptr(tx->request_uri),
bstr_len(tx->request_uri));
}
MemBufferWriteString(aft->buffer, " [**] ");
/* user agent */
htp_header_t *h_user_agent = NULL;
if (tx->request_headers != NULL) {
h_user_agent = htp_table_get_c(tx->request_headers, "user-agent");
}
if (h_user_agent != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,
(uint8_t *)bstr_ptr(h_user_agent->value),
bstr_len(h_user_agent->value));
} else {
MemBufferWriteString(aft->buffer, "<useragent unknown>");
}
if (hlog->flags & LOG_HTTP_EXTENDED) {
LogHttpLogExtended(aft, tx);
}
/* ip/tcp header info */
MemBufferWriteString(aft->buffer,
" [**] %s:%" PRIu16 " -> %s:%" PRIu16 "\n",
srcip, sp, dstip, dp);
}
aft->uri_cnt ++;
SCMutexLock(&hlog->file_ctx->fp_mutex);
hlog->file_ctx->Write((const char *)MEMBUFFER_BUFFER(aft->buffer),
MEMBUFFER_OFFSET(aft->buffer), hlog->file_ctx);
SCMutexUnlock(&hlog->file_ctx->fp_mutex);
end:
SCReturnInt(0);
}
/* Custom format logging */
static void LogHttpLogCustom(LogHttpLogThread *aft, htp_tx_t *tx, const struct timeval *ts,
char *srcip, Port sp, char *dstip, Port dp)
{
LogHttpFileCtx *httplog_ctx = aft->httplog_ctx;
uint32_t i;
uint32_t datalen;
char buf[128];
uint8_t *cvalue = NULL;
uint32_t cvalue_len = 0;
htp_header_t *h_request_hdr;
htp_header_t *h_response_hdr;
time_t time = ts->tv_sec;
struct tm local_tm;
struct tm *timestamp = SCLocalTime(time, &local_tm);
for (i = 0; i < httplog_ctx->cf_n; i++) {
h_request_hdr = NULL;
h_response_hdr = NULL;
switch (httplog_ctx->cf_nodes[i]->type){
case LOG_HTTP_CF_LITERAL:
/* LITERAL */
MemBufferWriteString(aft->buffer, "%s", httplog_ctx->cf_nodes[i]->data);
break;
case LOG_HTTP_CF_TIMESTAMP:
/* TIMESTAMP */
if (httplog_ctx->cf_nodes[i]->data[0] == '\0') {
strftime(buf, 62, TIMESTAMP_DEFAULT_FORMAT, timestamp);
} else {
strftime(buf, 62, httplog_ctx->cf_nodes[i]->data, timestamp);
}
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)buf,strlen(buf));
break;
case LOG_HTTP_CF_TIMESTAMP_U:
/* TIMESTAMP USECONDS */
snprintf(buf, 62, "%06u", (unsigned int) ts->tv_usec);
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)buf,strlen(buf));
break;
case LOG_HTTP_CF_CLIENT_IP:
/* CLIENT IP ADDRESS */
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)srcip,strlen(srcip));
break;
case LOG_HTTP_CF_SERVER_IP:
/* SERVER IP ADDRESS */
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)dstip,strlen(dstip));
break;
case LOG_HTTP_CF_CLIENT_PORT:
/* CLIENT PORT */
MemBufferWriteString(aft->buffer, "%" PRIu16 "", sp);
break;
case LOG_HTTP_CF_SERVER_PORT:
/* SERVER PORT */
MemBufferWriteString(aft->buffer, "%" PRIu16 "", dp);
break;
case LOG_HTTP_CF_REQUEST_METHOD:
/* METHOD */
if (tx->request_method != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)bstr_ptr(tx->request_method),
bstr_len(tx->request_method));
} else {
MemBufferWriteString(aft->buffer, LOG_HTTP_CF_NONE);
}
break;
case LOG_HTTP_CF_REQUEST_URI:
/* URI */
if (tx->request_uri != NULL) {
datalen = httplog_ctx->cf_nodes[i]->maxlen;
if (datalen == 0 || datalen > bstr_len(tx->request_uri)) {
datalen = bstr_len(tx->request_uri);
}
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)bstr_ptr(tx->request_uri),
datalen);
} else {
MemBufferWriteString(aft->buffer, LOG_HTTP_CF_NONE);
}
break;
case LOG_HTTP_CF_REQUEST_HOST:
/* HOSTNAME */
if (tx->request_hostname != NULL)
{
datalen = httplog_ctx->cf_nodes[i]->maxlen;
if (datalen == 0 || datalen > bstr_len(tx->request_hostname)) {
datalen = bstr_len(tx->request_hostname);
}
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)bstr_ptr(tx->request_hostname),
datalen);
} else {
MemBufferWriteString(aft->buffer, LOG_HTTP_CF_NONE);
}
break;
case LOG_HTTP_CF_REQUEST_PROTOCOL:
/* PROTOCOL */
if (tx->request_protocol != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)bstr_ptr(tx->request_protocol),
bstr_len(tx->request_protocol));
} else {
MemBufferWriteString(aft->buffer, LOG_HTTP_CF_NONE);
}
break;
case LOG_HTTP_CF_REQUEST_HEADER:
/* REQUEST HEADER */
if (tx->request_headers != NULL) {
h_request_hdr = htp_table_get_c(tx->request_headers, httplog_ctx->cf_nodes[i]->data);
}
if (h_request_hdr != NULL) {
datalen = httplog_ctx->cf_nodes[i]->maxlen;
if (datalen == 0 || datalen > bstr_len(h_request_hdr->value)) {
datalen = bstr_len(h_request_hdr->value);
}
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)bstr_ptr(h_request_hdr->value),
datalen);
} else {
MemBufferWriteString(aft->buffer, LOG_HTTP_CF_NONE);
}
break;
case LOG_HTTP_CF_REQUEST_COOKIE:
/* REQUEST COOKIE */
if (tx->request_headers != NULL) {
h_request_hdr = htp_table_get_c(tx->request_headers, "Cookie");
if (h_request_hdr != NULL) {
cvalue_len = GetCookieValue((uint8_t *) bstr_ptr(h_request_hdr->value),
bstr_len(h_request_hdr->value), (char *) httplog_ctx->cf_nodes[i]->data,
&cvalue);
}
}
if (cvalue_len > 0 && cvalue != NULL) {
datalen = httplog_ctx->cf_nodes[i]->maxlen;
if (datalen == 0 || datalen > cvalue_len) {
datalen = cvalue_len;
}
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, cvalue, datalen);
} else {
MemBufferWriteString(aft->buffer, LOG_HTTP_CF_NONE);
}
break;
case LOG_HTTP_CF_REQUEST_LEN:
/* REQUEST LEN */
MemBufferWriteString(aft->buffer, "%"PRIuMAX"", (uintmax_t)tx->request_message_len);
break;
case LOG_HTTP_CF_RESPONSE_STATUS:
/* RESPONSE STATUS */
if (tx->response_status != NULL) {
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)bstr_ptr(tx->response_status),
bstr_len(tx->response_status));
} else {
MemBufferWriteString(aft->buffer, LOG_HTTP_CF_NONE);
}
break;
case LOG_HTTP_CF_RESPONSE_HEADER:
/* RESPONSE HEADER */
if (tx->response_headers != NULL) {
h_response_hdr = htp_table_get_c(tx->response_headers,
httplog_ctx->cf_nodes[i]->data);
}
if (h_response_hdr != NULL) {
datalen = httplog_ctx->cf_nodes[i]->maxlen;
if (datalen == 0 || datalen > bstr_len(h_response_hdr->value)) {
datalen = bstr_len(h_response_hdr->value);
}
PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
aft->buffer->size, (uint8_t *)bstr_ptr(h_response_hdr->value),
datalen);
} else {
MemBufferWriteString(aft->buffer, LOG_HTTP_CF_NONE);
}
break;
case LOG_HTTP_CF_RESPONSE_LEN:
/* RESPONSE LEN */
MemBufferWriteString(aft->buffer, "%"PRIuMAX"", (uintmax_t)tx->response_message_len);
break;
default:
/* NO MATCH */
MemBufferWriteString(aft->buffer, LOG_HTTP_CF_NONE);
SCLogDebug("No matching parameter %%%c for custom http log.", httplog_ctx->cf_nodes[i]->type);
break;
}
}
MemBufferWriteString(aft->buffer, "\n");
}
static htp_status_t htp_tx_process_request_headers(htp_tx_t *tx) {
if (tx == NULL) return HTP_ERROR;
// Determine if we have a request body, and how it is packaged.
htp_status_t rc = HTP_OK;
htp_header_t *cl = htp_table_get_c(tx->request_headers, "content-length");
htp_header_t *te = htp_table_get_c(tx->request_headers, "transfer-encoding");
// Check for the Transfer-Encoding header, which would indicate a chunked request body.
if (te != NULL) {
// Make sure it contains "chunked" only.
// TODO The HTTP/1.1 RFC also allows the T-E header to contain "identity", which
// presumably should have the same effect as T-E header absence. However, Apache
// (2.2.22 on Ubuntu 12.04 LTS) instead errors out with "Unknown Transfer-Encoding: identity".
// And it behaves strangely, too, sending a 501 and proceeding to process the request
// (e.g., PHP is run), but without the body. It then closes the connection.
if (bstr_cmp_c(te->value, "chunked") != 0) {
// Invalid T-E header value.
tx->request_transfer_coding = HTP_CODING_INVALID;
tx->flags |= HTP_REQUEST_INVALID_T_E;
tx->flags |= HTP_REQUEST_INVALID;
} else {
// Chunked encoding is a HTTP/1.1 feature, so check that an earlier protocol
// version is not used. The flag will also be set if the protocol could not be parsed.
//
// TODO IIS 7.0, for example, would ignore the T-E header when it
// it is used with a protocol below HTTP 1.1. This should be a
// personality trait.
if (tx->request_protocol_number < HTP_PROTOCOL_1_1) {
tx->flags |= HTP_REQUEST_INVALID_T_E;
tx->flags |= HTP_REQUEST_SMUGGLING;
}
// If the T-E header is present we are going to use it.
tx->request_transfer_coding = HTP_CODING_CHUNKED;
// We are still going to check for the presence of C-L.
if (cl != NULL) {
// According to the HTTP/1.1 RFC (section 4.4):
//
// "The Content-Length header field MUST NOT be sent
// if these two lengths are different (i.e., if a Transfer-Encoding
// header field is present). If a message is received with both a
// Transfer-Encoding header field and a Content-Length header field,
// the latter MUST be ignored."
//
tx->flags |= HTP_REQUEST_SMUGGLING;
}
}
} else if (cl != NULL) {
// Check for a folded C-L header.
if (cl->flags & HTP_FIELD_FOLDED) {
tx->flags |= HTP_REQUEST_SMUGGLING;
}
// Check for multiple C-L headers.
if (cl->flags & HTP_FIELD_REPEATED) {
tx->flags |= HTP_REQUEST_SMUGGLING;
// TODO Personality trait to determine which C-L header to parse.
// At the moment we're parsing the combination of all instances,
// which is bound to fail (because it will contain commas).
}
// Get the body length.
tx->request_content_length = htp_parse_content_length(cl->value);
if (tx->request_content_length < 0) {
tx->request_transfer_coding = HTP_CODING_INVALID;
tx->flags |= HTP_REQUEST_INVALID_C_L;
tx->flags |= HTP_REQUEST_INVALID;
} else {
// We have a request body of known length.
tx->request_transfer_coding = HTP_CODING_IDENTITY;
}
} else {
// No body.
tx->request_transfer_coding = HTP_CODING_NO_BODY;
}
// If we could not determine the correct body handling,
// consider the request invalid.
if (tx->request_transfer_coding == HTP_CODING_UNKNOWN) {
tx->request_transfer_coding = HTP_CODING_INVALID;
tx->flags |= HTP_REQUEST_INVALID;
}
// Check for PUT requests, which we need to treat as file uploads.
if (tx->request_method_number == HTP_M_PUT) {
if (htp_tx_req_has_body(tx)) {
// Prepare to treat PUT request body as a file.
tx->connp->put_file = calloc(1, sizeof (htp_file_t));
if (tx->connp->put_file == NULL) return HTP_ERROR;
tx->connp->put_file->source = HTP_FILE_PUT;
} else {
// TODO Warn about PUT request without a body.
}
return HTP_OK;
}
// Determine hostname.
// Use the hostname from the URI, when available.
if (tx->parsed_uri->hostname != NULL) {
tx->request_hostname = bstr_dup(tx->parsed_uri->hostname);
if (tx->request_hostname == NULL) return HTP_ERROR;
}
tx->request_port_number = tx->parsed_uri->port_number;
// Examine the Host header.
htp_header_t *h = htp_table_get_c(tx->request_headers, "host");
if (h == NULL) {
// No host information in the headers.
// HTTP/1.1 requires host information in the headers.
if (tx->request_protocol_number >= HTP_PROTOCOL_1_1) {
tx->flags |= HTP_HOST_MISSING;
}
} else {
// Host information available in the headers.
bstr *hostname;
int port;
rc = htp_parse_header_hostport(h->value, &hostname, &port, &(tx->flags));
if (rc != HTP_OK) return rc;
// Is there host information in the URI?
if (tx->request_hostname == NULL) {
// There is no host information in the URI. Place the
// hostname from the headers into the parsed_uri structure.
tx->request_hostname = hostname;
tx->request_port_number = port;
} else {
// The host information appears in the URI and in the headers. It's
// OK if both have the same thing, but we want to check for differences.
if ((bstr_cmp_nocase(hostname, tx->request_hostname) != 0) || (port != tx->request_port_number)) {
// The host information is different in the headers and the URI. The
// HTTP RFC states that we should ignore the header copy.
tx->flags |= HTP_HOST_AMBIGUOUS;
}
bstr_free(hostname);
}
}
// Determine Content-Type.
htp_header_t *ct = htp_table_get_c(tx->request_headers, "content-type");
if (ct != NULL) {
rc = htp_parse_ct_header(ct->value, &tx->request_content_type);
if (rc != HTP_OK) return rc;
}
// Parse cookies.
if (tx->connp->cfg->parse_request_cookies) {
rc = htp_parse_cookies_v0(tx->connp);
if (rc != HTP_OK) return rc;
}
// Parse authentication information.
if (tx->connp->cfg->parse_request_auth) {
rc = htp_parse_authorization(tx->connp);
if (rc == HTP_DECLINED) {
// Don't fail the stream if an authorization header is invalid, just set a flag.
tx->flags |= HTP_AUTH_INVALID;
} else {
if (rc != HTP_OK) return rc;
}
}
// Finalize sending raw header data.
rc = htp_connp_req_receiver_finalize_clear(tx->connp);
if (rc != HTP_OK) return rc;
// Run hook REQUEST_HEADERS.
rc = htp_hook_run_all(tx->connp->cfg->hook_request_headers, tx);
if (rc != HTP_OK) return rc;
// We cannot proceed if the request is invalid.
if (tx->flags & HTP_REQUEST_INVALID) {
return HTP_ERROR;
}
return HTP_OK;
}
static htp_status_t htp_tx_process_request_headers(htp_tx_t *tx) {
// Remember how many header lines there were before trailers.
tx->request_header_lines_no_trailers = htp_list_size(tx->request_header_lines);
// Determine if we have a request body, and how it is packaged.
htp_header_t *cl = htp_table_get_c(tx->request_headers, "content-length");
htp_header_t *te = htp_table_get_c(tx->request_headers, "transfer-encoding");
// Check for the Transfer-Encoding header, which would indicate a chunked request body.
if (te != NULL) {
// Make sure it contains "chunked" only.
if (bstr_cmp_c(te->value, "chunked") != 0) {
// Invalid T-E header value.
tx->flags |= HTP_INVALID_CHUNKING;
htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0,
"Invalid T-E value in request");
}
// Chunked encoding is a HTTP/1.1 feature. Check that some other protocol is not
// used. The flag will also be set if the protocol could not be parsed.
//
// TODO IIS 7.0, for example, would ignore the T-E header when it
// it is used with a protocol below HTTP 1.1.
if (tx->request_protocol_number < HTP_PROTOCOL_1_1) {
tx->flags |= HTP_INVALID_CHUNKING;
}
// If the T-E header is present we are going to use it.
tx->request_transfer_coding = HTP_CODING_CHUNKED;
// We are still going to check for the presence of C-L.
if (cl != NULL) {
// This is a violation of the RFC.
tx->flags |= HTP_REQUEST_SMUGGLING;
}
} else if (cl != NULL) {
// We have a request body of known length.
tx->request_transfer_coding = HTP_CODING_IDENTITY;
// Check for a folded C-L header.
if (cl->flags & HTP_FIELD_FOLDED) {
tx->flags |= HTP_REQUEST_SMUGGLING;
}
// Check for multiple C-L headers.
if (cl->flags & HTP_FIELD_REPEATED) {
tx->flags |= HTP_REQUEST_SMUGGLING;
}
// Get body length.
tx->request_content_length = htp_parse_content_length(cl->value);
if (tx->request_content_length < 0) {
htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Invalid C-L field in request");
return HTP_ERROR;
}
} else {
// No body.
tx->request_transfer_coding = HTP_CODING_NO_BODY;
}
// Check for PUT requests, which we need to treat as file uploads.
if (tx->request_method_number == HTP_M_PUT) {
if (htp_tx_req_has_body(tx)) {
// Prepare to treat PUT request body as a file.
tx->connp->put_file = calloc(1, sizeof (htp_file_t));
if (tx->connp->put_file == NULL) return HTP_ERROR;
tx->connp->put_file->source = HTP_FILE_PUT;
} else {
// TODO Warn about PUT request without a body.
}
return HTP_OK;
}
// Host resolution
htp_header_t *h = htp_table_get_c(tx->request_headers, "host");
if (h == NULL) {
// No host information in the headers.
// HTTP/1.1 requires host information in the headers.
if (tx->request_protocol_number >= HTP_PROTOCOL_1_1) {
tx->flags |= HTP_HOST_MISSING;
htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0,
"Host information in request headers required by HTTP/1.1");
}
} else {
// Host information available in the headers.
bstr *hostname;
int port;
if (htp_parse_hostport(h->value, &hostname, &port, &(tx->flags)) != HTP_OK) return HTP_ERROR;
// Is there host information in the URI?
if (tx->parsed_uri->hostname == NULL) {
// There is no host information in the URI. Place the
// hostname from the headers into the parsed_uri structure.
tx->parsed_uri->hostname = hostname;
tx->parsed_uri->port_number = port;
} else {
if ((bstr_cmp_nocase(hostname, tx->parsed_uri->hostname) != 0) || (port != tx->parsed_uri->port_number)) {
// The host information is different in the
// headers and the URI. The HTTP RFC states that
// we should ignore the header copy.
tx->flags |= HTP_HOST_AMBIGUOUS;
htp_log(tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Host information ambiguous");
}
bstr_free(hostname);
}
}
// Parse the Content-Type header.
htp_header_t *ct = htp_table_get_c(tx->request_headers, "content-type");
if (ct != NULL) {
if (htp_parse_ct_header(ct->value, &tx->request_content_type) != HTP_OK) return HTP_ERROR;
}
// Parse cookies.
if (tx->connp->cfg->parse_request_cookies) {
htp_parse_cookies_v0(tx->connp);
}
// Parse authentication information.
if (tx->connp->cfg->parse_request_http_authentication) {
htp_parse_authorization(tx->connp);
}
// Run hook REQUEST_HEADERS.
int rc = htp_hook_run_all(tx->connp->cfg->hook_request_headers, tx->connp);
if (rc != HTP_OK) return rc;
return HTP_OK;
}
