static void LogFileMetaGetUserAgent(FILE *fp, const Packet *p, const File *ff)
{
HtpState *htp_state = (HtpState *)p->flow->alstate;
if (htp_state != NULL) {
htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
if (tx != NULL) {
htp_header_t *h = NULL;
h = (htp_header_t *)htp_table_get_c(tx->request_headers,
"User-Agent");
if (h != NULL) {
PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
bstr_len(h->value));
return;
}
}
}
fprintf(fp, "<unknown>");
}
static void LogFileMetaGetUri(FILE *fp, const Packet *p, const File *ff)
{
HtpState *htp_state = (HtpState *)p->flow->alstate;
if (htp_state != NULL) {
htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
if (tx != NULL) {
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
if (tx_ud != NULL) {
if (tx_ud->request_uri_normalized != NULL) {
PrintRawJsonFp(fp,
bstr_ptr(tx_ud->request_uri_normalized),
bstr_len(tx_ud->request_uri_normalized));
return;
}
}
}
}
fprintf(fp, "<unknown>");
}
static json_t *LogFileMetaGetUri(const Packet *p, const File *ff) {
HtpState *htp_state = (HtpState *)p->flow->alstate;
json_t *js = NULL;
if (htp_state != NULL) {
htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
if (tx != NULL) {
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
if (tx_ud->request_uri_normalized != NULL) {
char *s = bstr_util_strdup_to_c(tx_ud->request_uri_normalized);
if (s != NULL) {
js = json_string(s);
SCFree(s);
}
}
return js;
}
}
return json_string("<unknown>");
}
/** Handle the case where no JSON support is compiled in.
*
*/
static void AlertJsonHttp(const Flow *f, json_t *js)
{
HtpState *htp_state = (HtpState *)FlowGetAppState(f);
if (htp_state) {
uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
if (tx) {
json_t *hjs = json_object();
if (unlikely(hjs == NULL))
return;
JsonHttpLogJSONBasic(hjs, tx);
JsonHttpLogJSONExtended(hjs, tx);
json_object_set_new(js, "http", hjs);
}
}
return;
}
static json_t *LogFileMetaGetUserAgent(const Packet *p, const File *ff) {
HtpState *htp_state = (HtpState *)p->flow->alstate;
json_t *js = NULL;
if (htp_state != NULL) {
htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
if (tx != NULL) {
htp_header_t *h = NULL;
h = (htp_header_t *)htp_table_get_c(tx->request_headers,
"User-Agent");
if (h != NULL) {
char *s = bstr_util_strdup_to_c(h->value);
if (s != NULL) {
js = json_string(s);
SCFree(s);
}
return js;
}
}
}
return json_string("<unknown>");
}
static void LogFilestoreMetaGetSmtp(FILE *fp, const Packet *p, const File *ff)
{
SMTPState *state = (SMTPState *) p->flow->alstate;
if (state != NULL) {
SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, state, ff->txid);
if (tx == NULL || tx->msg_tail == NULL)
return;
/* Message Id */
if (tx->msg_tail->msg_id != NULL) {
fprintf(fp, "MESSAGE-ID: ");
PrintRawUriFp(fp, (uint8_t *) tx->msg_tail->msg_id, tx->msg_tail->msg_id_len);
fprintf(fp, "\n");
}
/* Sender */
MimeDecField *field = MimeDecFindField(tx->msg_tail, "from");
if (field != NULL) {
fprintf(fp, "SENDER: ");
PrintRawUriFp(fp, (uint8_t *) field->value, field->value_len);
fprintf(fp, "\n");
}
}
}
static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
{
BUG_ON(thread_data == NULL);
if (list == NULL) {
/* No child loggers registered. */
return TM_ECODE_OK;
}
OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
OutputTxLogger *logger = list;
OutputLoggerThreadStore *store = op_thread_data->store;
BUG_ON(logger == NULL && store != NULL);
BUG_ON(logger != NULL && store == NULL);
BUG_ON(logger == NULL && store == NULL);
if (p->flow == NULL)
return TM_ECODE_OK;
Flow * const f = p->flow;
AppProto alproto = f->alproto;
if (AppLayerParserProtocolIsTxAware(p->proto, alproto) == 0)
goto end;
if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0)
goto end;
void *alstate = f->alstate;
if (alstate == NULL) {
SCLogDebug("no alstate");
goto end;
}
uint64_t total_txs = AppLayerParserGetTxCnt(p->proto, alproto, alstate);
uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
for (; tx_id < total_txs; tx_id++)
{
int logger_not_logged = 0;
void *tx = AppLayerParserGetTx(p->proto, alproto, alstate, tx_id);
if (tx == NULL) {
SCLogDebug("tx is NULL not logging");
continue;
}
int tx_progress_ts = AppLayerParserGetStateProgress(p->proto, alproto,
tx, FlowGetDisruptionFlags(f, STREAM_TOSERVER));
int tx_progress_tc = AppLayerParserGetStateProgress(p->proto, alproto,
tx, FlowGetDisruptionFlags(f, STREAM_TOCLIENT));
SCLogDebug("tx_progress_ts %d tx_progress_tc %d",
tx_progress_ts, tx_progress_tc);
// call each logger here (pseudo code)
logger = list;
store = op_thread_data->store;
while (logger && store) {
BUG_ON(logger->LogFunc == NULL);
SCLogDebug("logger %p, LogCondition %p, ts_log_progress %d "
"tc_log_progress %d", logger, logger->LogCondition,
logger->ts_log_progress, logger->tc_log_progress);
if (logger->alproto == alproto) {
SCLogDebug("alproto match, logging tx_id %"PRIu64, tx_id);
if (AppLayerParserGetTxLogged(p->proto, alproto, alstate, tx,
logger->id)) {
SCLogDebug("logger has already logged this transaction");
goto next;
}
if (!(AppLayerParserStateIssetFlag(f->alparser,
APP_LAYER_PARSER_EOF))) {
if (logger->LogCondition) {
int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
if (r == FALSE) {
SCLogDebug("conditions not met, not logging");
logger_not_logged = 1;
goto next;
}
} else {
if (tx_progress_tc < logger->tc_log_progress) {
SCLogDebug("progress not far enough, not logging");
logger_not_logged = 1;
goto next;
}
if (tx_progress_ts < logger->ts_log_progress) {
SCLogDebug("progress not far enough, not logging");
logger_not_logged = 1;
goto next;
}
}
}
PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
AppLayerParserSetTxLogged(p->proto, alproto, alstate, tx,
logger->id);
}
next:
logger = logger->next;
store = store->next;
BUG_ON(logger == NULL && store != NULL);
BUG_ON(logger != NULL && store == NULL);
}
if (!logger_not_logged) {
SCLogDebug("updating log tx_id %"PRIu64, tx_id);
AppLayerParserSetTransactionLogId(f->alparser);
}
}
end:
return TM_ECODE_OK;
}
int HttpBodyIterator(Flow *f, int close, void *cbdata, uint8_t iflags)
{
SCLogDebug("called with %p, %d, %p, %02x", f, close, cbdata, iflags);
HtpState *s = f->alstate;
if (s != NULL && s->conn != NULL) {
int tx_progress_done_value_ts =
AppLayerParserGetStateProgressCompletionStatus(IPPROTO_TCP,
ALPROTO_HTTP, STREAM_TOSERVER);
int tx_progress_done_value_tc =
AppLayerParserGetStateProgressCompletionStatus(IPPROTO_TCP,
ALPROTO_HTTP, STREAM_TOCLIENT);
// for each tx
uint64_t tx_id = 0;
uint64_t total_txs = AppLayerParserGetTxCnt(f->proto, f->alproto, f->alstate);
SCLogDebug("s->conn %p", s->conn);
for (tx_id = 0; tx_id < total_txs; tx_id++) { // TODO optimization store log tx
htp_tx_t *tx = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, tx_id);
if (tx != NULL) {
int tx_done = 0;
int tx_logged = 0;
int tx_progress_ts = AppLayerParserGetStateProgress(
IPPROTO_TCP, ALPROTO_HTTP, tx, FlowGetDisruptionFlags(f, STREAM_TOSERVER));
if (tx_progress_ts >= tx_progress_done_value_ts) {
int tx_progress_tc = AppLayerParserGetStateProgress(
IPPROTO_TCP, ALPROTO_HTTP, tx, FlowGetDisruptionFlags(f, STREAM_TOCLIENT));
if (tx_progress_tc >= tx_progress_done_value_tc) {
tx_done = 1;
}
}
SCLogDebug("tx %p", tx);
HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
if (htud != NULL) {
SCLogDebug("htud %p", htud);
HtpBody *body = NULL;
if (iflags & OUTPUT_STREAMING_FLAG_TOCLIENT)
body = &htud->request_body;
else if (iflags & OUTPUT_STREAMING_FLAG_TOSERVER)
body = &htud->response_body;
if (body == NULL) {
SCLogDebug("no body");
goto next;
}
if (body->first == NULL) {
SCLogDebug("no body chunks");
goto next;
}
if (body->last->logged == 1) {
SCLogDebug("all logged already");
goto next;
}
// for each chunk
HtpBodyChunk *chunk = body->first;
for ( ; chunk != NULL; chunk = chunk->next) {
if (chunk->logged) {
SCLogDebug("logged %d", chunk->logged);
continue;
}
uint8_t flags = iflags | OUTPUT_STREAMING_FLAG_TRANSACTION;
if (chunk->stream_offset == 0)
flags |= OUTPUT_STREAMING_FLAG_OPEN;
/* if we need to close and we're at the last segment in the list
* we add the 'close' flag so the logger can close up. */
if ((tx_done || close) && chunk->next == NULL) {
flags |= OUTPUT_STREAMING_FLAG_CLOSE;
}
// invoke Streamer
Streamer(cbdata, f, chunk->data, (uint32_t)chunk->len, tx_id, flags);
//PrintRawDataFp(stdout, chunk->data, chunk->len);
chunk->logged = 1;
tx_logged = 1;
}
next:
/* if we need to close we need to invoke the Streamer for sure. If we
* logged no chunks, we call the Streamer with NULL data so it can
* close up. */
if (tx_logged == 0 && (close||tx_done)) {
Streamer(cbdata, f, NULL, 0, tx_id,
OUTPUT_STREAMING_FLAG_CLOSE|OUTPUT_STREAMING_FLAG_TRANSACTION);
}
}
}
}
}
return 0;
}
static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQueue *pq, PacketQueue *postpq)
{
BUG_ON(thread_data == NULL);
BUG_ON(list == NULL);
OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
OutputTxLogger *logger = list;
OutputLoggerThreadStore *store = op_thread_data->store;
BUG_ON(logger == NULL && store != NULL);
BUG_ON(logger != NULL && store == NULL);
BUG_ON(logger == NULL && store == NULL);
if (p->flow == NULL)
return TM_ECODE_OK;
Flow * const f = p->flow;
FLOWLOCK_WRLOCK(f); /* WRITE lock before we updated flow logged id */
AppProto alproto = f->alproto;
if (AppLayerParserProtocolIsTxAware(p->proto, alproto) == 0)
goto end;
if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0)
goto end;
void *alstate = f->alstate;
if (alstate == NULL) {
SCLogDebug("no alstate");
goto end;
}
uint64_t total_txs = AppLayerParserGetTxCnt(p->proto, alproto, alstate);
uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
int tx_progress_done_value_ts =
AppLayerParserGetStateProgressCompletionStatus(p->proto, alproto,
STREAM_TOSERVER);
int tx_progress_done_value_tc =
AppLayerParserGetStateProgressCompletionStatus(p->proto, alproto,
STREAM_TOCLIENT);
for (; tx_id < total_txs; tx_id++)
{
int proto_logged = 0;
void *tx = AppLayerParserGetTx(p->proto, alproto, alstate, tx_id);
if (tx == NULL) {
SCLogDebug("tx is NULL not logging");
continue;
}
if (!(AppLayerParserStateIssetFlag(f->alparser, APP_LAYER_PARSER_EOF)))
{
int tx_progress = AppLayerParserGetStateProgress(p->proto, alproto,
tx, FlowGetDisruptionFlags(f, STREAM_TOSERVER));
if (tx_progress < tx_progress_done_value_ts) {
SCLogDebug("progress not far enough, not logging");
break;
}
tx_progress = AppLayerParserGetStateProgress(p->proto, alproto,
tx, FlowGetDisruptionFlags(f, STREAM_TOCLIENT));
if (tx_progress < tx_progress_done_value_tc) {
SCLogDebug("progress not far enough, not logging");
break;
}
}
// call each logger here (pseudo code)
logger = list;
store = op_thread_data->store;
while (logger && store) {
BUG_ON(logger->LogFunc == NULL);
SCLogDebug("logger %p", logger);
if (logger->alproto == alproto) {
SCLogDebug("alproto match, logging tx_id %ju", tx_id);
PACKET_PROFILING_TMM_START(p, logger->module_id);
logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
PACKET_PROFILING_TMM_END(p, logger->module_id);
proto_logged = 1;
}
logger = logger->next;
store = store->next;
BUG_ON(logger == NULL && store != NULL);
BUG_ON(logger != NULL && store == NULL);
}
if (proto_logged) {
SCLogDebug("updating log tx_id %ju", tx_id);
AppLayerParserSetTransactionLogId(f->alparser);
}
}
end:
FLOWLOCK_UNLOCK(f);
return TM_ECODE_OK;
}
static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
{
BUG_ON(thread_data == NULL);
if (list == NULL) {
/* No child loggers registered. */
return TM_ECODE_OK;
}
OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
if (p->flow == NULL)
return TM_ECODE_OK;
Flow * const f = p->flow;
const AppProto alproto = f->alproto;
if (AppLayerParserProtocolIsTxAware(p->proto, alproto) == 0)
goto end;
if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0)
goto end;
void *alstate = f->alstate;
if (alstate == NULL) {
SCLogDebug("no alstate");
goto end;
}
const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER);
const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT);
const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
uint64_t max_id = tx_id;
int logged = 0;
int gap = 0;
for (; tx_id < total_txs; tx_id++)
{
/* Track the number of loggers, of the eligible loggers that
* actually logged this transaction. They all must have logged
* before the transaction is considered logged. */
int number_of_loggers = 0;
int loggers_that_logged = 0;
void *tx = AppLayerParserGetTx(p->proto, alproto, alstate, tx_id);
if (tx == NULL) {
SCLogDebug("tx is NULL not logging");
continue;
}
int tx_progress_ts = AppLayerParserGetStateProgress(p->proto, alproto,
tx, ts_disrupt_flags);
int tx_progress_tc = AppLayerParserGetStateProgress(p->proto, alproto,
tx, tc_disrupt_flags);
SCLogDebug("tx_progress_ts %d tx_progress_tc %d",
tx_progress_ts, tx_progress_tc);
const OutputTxLogger *logger = list;
const OutputLoggerThreadStore *store = op_thread_data->store;
#ifdef DEBUG_VALIDATION
BUG_ON(logger == NULL && store != NULL);
BUG_ON(logger != NULL && store == NULL);
BUG_ON(logger == NULL && store == NULL);
#endif
while (logger && store) {
BUG_ON(logger->LogFunc == NULL);
SCLogDebug("logger %p, LogCondition %p, ts_log_progress %d "
"tc_log_progress %d", logger, logger->LogCondition,
logger->ts_log_progress, logger->tc_log_progress);
if (logger->alproto == alproto) {
SCLogDebug("alproto match, logging tx_id %"PRIu64, tx_id);
number_of_loggers++;
if (AppLayerParserGetTxLogged(f, alstate, tx, logger->id)) {
SCLogDebug("logger has already logged this transaction");
loggers_that_logged++;
goto next;
}
if (!(AppLayerParserStateIssetFlag(f->alparser,
APP_LAYER_PARSER_EOF))) {
if (logger->LogCondition) {
int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
if (r == FALSE) {
SCLogDebug("conditions not met, not logging");
goto next;
}
} else {
if (tx_progress_tc < logger->tc_log_progress) {
SCLogDebug("progress not far enough, not logging");
goto next;
}
if (tx_progress_ts < logger->ts_log_progress) {
SCLogDebug("progress not far enough, not logging");
goto next;
}
}
}
SCLogDebug("Logging tx_id %"PRIu64" to logger %d", tx_id,
logger->logger_id);
PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
AppLayerParserSetTxLogged(p->proto, alproto, alstate, tx,
logger->id);
loggers_that_logged++;
}
next:
logger = logger->next;
store = store->next;
#ifdef DEBUG_VALIDATION
BUG_ON(logger == NULL && store != NULL);
BUG_ON(logger != NULL && store == NULL);
#endif
}
/* If all loggers logged set a flag and update the last tx_id
* that was logged.
*
* If not all loggers were logged we flag that there was a gap
* so any subsequent transactions in this loop don't increase
* the maximum ID that was logged. */
if (!gap && loggers_that_logged == number_of_loggers) {
logged = 1;
max_id = tx_id;
} else {
gap = 1;
}
}
/* Update the the last ID that has been logged with all
* transactions before it. */
if (logged) {
SCLogDebug("updating log tx_id %"PRIu64, max_id);
AppLayerParserSetTransactionLogId(f->alparser, max_id + 1);
}
end:
return TM_ECODE_OK;
}
