void JsonAddFlow(Flow *f, json_t *js, json_t *hjs)
{
json_object_set_new(js, "app_proto",
json_string(AppProtoToString(f->alproto)));
if (f->alproto_ts != f->alproto) {
json_object_set_new(js, "app_proto_ts",
json_string(AppProtoToString(f->alproto_ts)));
}
if (f->alproto_tc != f->alproto) {
json_object_set_new(js, "app_proto_tc",
json_string(AppProtoToString(f->alproto_tc)));
}
if (f->alproto_orig != f->alproto && f->alproto_orig != ALPROTO_UNKNOWN) {
json_object_set_new(js, "app_proto_orig",
json_string(AppProtoToString(f->alproto_orig)));
}
if (f->alproto_expect != f->alproto && f->alproto_expect != ALPROTO_UNKNOWN) {
json_object_set_new(js, "app_proto_expected",
json_string(AppProtoToString(f->alproto_expect)));
}
json_object_set_new(hjs, "pkts_toserver",
json_integer(f->todstpktcnt));
json_object_set_new(hjs, "pkts_toclient",
json_integer(f->tosrcpktcnt));
json_object_set_new(hjs, "bytes_toserver",
json_integer(f->todstbytecnt));
json_object_set_new(hjs, "bytes_toclient",
json_integer(f->tosrcbytecnt));
char timebuf1[64];
CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
json_object_set_new(hjs, "start", json_string(timebuf1));
}
/** \internal
* \brief fill lua stack with AppLayerProto
* \param luastate the lua state
* \param f flow, locked
* \retval cnt number of data items placed on the stack
*
* Places: alproto as string (string)
*/
static int LuaCallbackAppLayerProtoPushToStackFromFlow(lua_State *luastate, const Flow *f)
{
const char *string = AppProtoToString(f->alproto);
if (string == NULL)
string = "unknown";
lua_pushstring(luastate, string);
return 1;
}
static void JsonNetFlowLogJSONToClient(JsonNetFlowLogThread *aft, json_t *js, Flow *f)
{
json_t *hjs = json_object();
if (hjs == NULL) {
return;
}
json_object_set_new(js, "app_proto",
json_string(AppProtoToString(f->alproto_tc ? f->alproto_tc : f->alproto)));
json_object_set_new(hjs, "pkts",
json_integer(f->tosrcpktcnt));
json_object_set_new(hjs, "bytes",
json_integer(f->tosrcbytecnt));
char timebuf1[64], timebuf2[64];
CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
json_object_set_new(hjs, "start", json_string(timebuf1));
json_object_set_new(hjs, "end", json_string(timebuf2));
int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
json_object_set_new(hjs, "age",
json_integer(age));
/* To client is zero if we did not see any packet */
if (f->tosrcpktcnt) {
json_object_set_new(hjs, "min_ttl", json_integer(f->min_ttl_toclient));
json_object_set_new(hjs, "max_ttl", json_integer(f->max_ttl_toclient));
}
json_object_set_new(js, "netflow", hjs);
/* TCP */
if (f->proto == IPPROTO_TCP) {
json_t *tjs = json_object();
if (tjs == NULL) {
return;
}
TcpSession *ssn = f->protoctx;
char hexflags[3];
snprintf(hexflags, sizeof(hexflags), "%02x",
ssn ? ssn->server.tcp_flags : 0);
json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
JsonTcpFlags(ssn ? ssn->server.tcp_flags : 0, tjs);
json_object_set_new(js, "tcp", tjs);
}
}
void SCProfilingDumpPacketStats(void)
{
int i;
FILE *fp;
char totalstr[256];
uint64_t total;
if (profiling_packets_enabled == 0)
return;
if (profiling_packets_output_to_file == 1) {
fp = fopen(profiling_packets_file_name, profiling_packets_file_mode);
if (fp == NULL) {
SCLogError(SC_ERR_FOPEN, "failed to open %s: %s",
profiling_packets_file_name, strerror(errno));
return;
}
} else {
fp = stdout;
}
fprintf(fp, "\n\nPacket profile dump:\n");
fprintf(fp, "\n%-6s %-5s %-12s %-12s %-12s %-12s %-12s %-3s\n",
"IP ver", "Proto", "cnt", "min", "max", "avg", "tot", "%%");
fprintf(fp, "%-6s %-5s %-12s %-12s %-12s %-12s %-12s %-3s\n",
"------", "-----", "----------", "------------", "------------", "-----------", "-----------", "---");
total = 0;
for (i = 0; i < 257; i++) {
SCProfilePacketData *pd = &packet_profile_data4[i];
total += pd->tot;
pd = &packet_profile_data6[i];
total += pd->tot;
}
for (i = 0; i < 257; i++) {
SCProfilePacketData *pd = &packet_profile_data4[i];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, " IPv4 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %6.2f\n", i, pd->cnt,
pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
for (i = 0; i < 257; i++) {
SCProfilePacketData *pd = &packet_profile_data6[i];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, " IPv6 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %6.2f\n", i, pd->cnt,
pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
fprintf(fp, "Note: Protocol 256 tracks pseudo/tunnel packets.\n");
fprintf(fp, "\nPer Thread module stats:\n");
fprintf(fp, "\n%-24s %-6s %-5s %-12s %-12s %-12s %-12s %-12s %-3s",
"Thread Module", "IP ver", "Proto", "cnt", "min", "max", "avg", "tot", "%%");
#ifdef PROFILE_LOCKING
fprintf(fp, " %-10s %-10s %-12s %-12s %-10s %-10s %-12s %-12s\n",
"locks", "ticks", "cont.", "cont.avg", "slocks", "sticks", "scont.", "scont.avg");
#else
fprintf(fp, "\n");
#endif
fprintf(fp, "%-24s %-6s %-5s %-12s %-12s %-12s %-12s %-12s %-3s",
"------------------------", "------", "-----", "----------", "------------", "------------", "-----------", "-----------", "---");
#ifdef PROFILE_LOCKING
fprintf(fp, " %-10s %-10s %-12s %-12s %-10s %-10s %-12s %-12s\n",
"--------", "--------", "----------", "-----------", "--------", "--------", "------------", "-----------");
#else
fprintf(fp, "\n");
#endif
int m;
total = 0;
for (m = 0; m < TMM_SIZE; m++) {
if (tmm_modules[m].flags & TM_FLAG_LOGAPI_TM)
continue;
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_tmm_data4[m][p];
total += pd->tot;
pd = &packet_profile_tmm_data6[m][p];
total += pd->tot;
}
}
for (m = 0; m < TMM_SIZE; m++) {
if (tmm_modules[m].flags & TM_FLAG_LOGAPI_TM)
continue;
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_tmm_data4[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-24s IPv4 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %6.2f",
TmModuleTmmIdToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
#ifdef PROFILE_LOCKING
fprintf(fp, " %10.2f %12"PRIu64" %12"PRIu64" %10.2f %10.2f %12"PRIu64" %12"PRIu64" %10.2f\n",
(float)pd->lock/pd->cnt, (uint64_t)pd->ticks/pd->cnt, pd->contention, (float)pd->contention/pd->cnt, (float)pd->slock/pd->cnt, (uint64_t)pd->sticks/pd->cnt, pd->scontention, (float)pd->scontention/pd->cnt);
#else
fprintf(fp, "\n");
#endif
}
}
for (m = 0; m < TMM_SIZE; m++) {
if (tmm_modules[m].flags & TM_FLAG_LOGAPI_TM)
continue;
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_tmm_data6[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-24s IPv6 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %6.2f\n",
TmModuleTmmIdToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
}
DumpFlowWorker(fp);
fprintf(fp, "\nPer App layer parser stats:\n");
fprintf(fp, "\n%-20s %-6s %-5s %-12s %-12s %-12s %-12s\n",
"App Layer", "IP ver", "Proto", "cnt", "min", "max", "avg");
fprintf(fp, "%-20s %-6s %-5s %-12s %-12s %-12s %-12s\n",
"--------------------", "------", "-----", "----------", "------------", "------------", "-----------");
total = 0;
for (m = 0; m < ALPROTO_MAX; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_app_data4[m][p];
total += pd->tot;
pd = &packet_profile_app_data6[m][p];
total += pd->tot;
}
}
for (m = 0; m < ALPROTO_MAX; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_app_data4[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-20s IPv4 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %-6.2f\n",
AppProtoToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
}
for (m = 0; m < ALPROTO_MAX; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_app_data6[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-20s IPv6 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %-6.2f\n",
AppProtoToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
}
/* proto detect output */
{
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_app_pd_data4[p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
fprintf(fp, "%-20s IPv4 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s\n",
"Proto detect", p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr);
}
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_app_pd_data6[p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
fprintf(fp, "%-20s IPv6 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s\n",
"Proto detect", p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr);
}
}
total = 0;
for (m = 0; m < PROF_DETECT_SIZE; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_detect_data4[m][p];
total += pd->tot;
pd = &packet_profile_detect_data6[m][p];
total += pd->tot;
}
}
fprintf(fp, "\n%-24s %-6s %-5s %-12s %-12s %-12s %-12s %-12s %-3s",
"Log Thread Module", "IP ver", "Proto", "cnt", "min", "max", "avg", "tot", "%%");
#ifdef PROFILE_LOCKING
fprintf(fp, " %-10s %-10s %-12s %-12s %-10s %-10s %-12s %-12s\n",
"locks", "ticks", "cont.", "cont.avg", "slocks", "sticks", "scont.", "scont.avg");
#else
fprintf(fp, "\n");
#endif
fprintf(fp, "%-24s %-6s %-5s %-12s %-12s %-12s %-12s %-12s %-3s",
"------------------------", "------", "-----", "----------", "------------", "------------", "-----------", "-----------", "---");
#ifdef PROFILE_LOCKING
fprintf(fp, " %-10s %-10s %-12s %-12s %-10s %-10s %-12s %-12s\n",
"--------", "--------", "----------", "-----------", "--------", "--------", "------------", "-----------");
#else
fprintf(fp, "\n");
#endif
total = 0;
for (m = 0; m < TMM_SIZE; m++) {
if (!(tmm_modules[m].flags & TM_FLAG_LOGAPI_TM))
continue;
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_tmm_data4[m][p];
total += pd->tot;
pd = &packet_profile_tmm_data6[m][p];
total += pd->tot;
}
}
for (m = 0; m < TMM_SIZE; m++) {
if (!(tmm_modules[m].flags & TM_FLAG_LOGAPI_TM))
continue;
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_tmm_data4[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-24s IPv4 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %6.2f",
TmModuleTmmIdToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
#ifdef PROFILE_LOCKING
fprintf(fp, " %10.2f %12"PRIu64" %12"PRIu64" %10.2f %10.2f %12"PRIu64" %12"PRIu64" %10.2f\n",
(float)pd->lock/pd->cnt, (uint64_t)pd->ticks/pd->cnt, pd->contention, (float)pd->contention/pd->cnt, (float)pd->slock/pd->cnt, (uint64_t)pd->sticks/pd->cnt, pd->scontention, (float)pd->scontention/pd->cnt);
#else
fprintf(fp, "\n");
#endif
}
}
for (m = 0; m < TMM_SIZE; m++) {
if (!(tmm_modules[m].flags & TM_FLAG_LOGAPI_TM))
continue;
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_tmm_data6[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-24s IPv6 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %6.2f\n",
TmModuleTmmIdToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
}
fprintf(fp, "\nLogger/output stats:\n");
total = 0;
for (m = 0; m < LOGGER_SIZE; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_log_data4[m][p];
total += pd->tot;
pd = &packet_profile_log_data6[m][p];
total += pd->tot;
}
}
fprintf(fp, "\n%-24s %-6s %-5s %-12s %-12s %-12s %-12s %-12s\n",
"Logger", "IP ver", "Proto", "cnt", "min", "max", "avg", "tot");
fprintf(fp, "%-24s %-6s %-5s %-12s %-12s %-12s %-12s %-12s\n",
"------------------------", "------", "-----", "----------", "------------", "------------", "-----------", "-----------");
for (m = 0; m < LOGGER_SIZE; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_log_data4[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-24s IPv4 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %-6.2f\n",
PacketProfileLoggertIdToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
}
for (m = 0; m < LOGGER_SIZE; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_log_data6[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-24s IPv6 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %-6.2f\n",
PacketProfileLoggertIdToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
}
fprintf(fp, "\nGeneral detection engine stats:\n");
total = 0;
for (m = 0; m < PROF_DETECT_SIZE; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_detect_data4[m][p];
total += pd->tot;
pd = &packet_profile_detect_data6[m][p];
total += pd->tot;
}
}
fprintf(fp, "\n%-24s %-6s %-5s %-12s %-12s %-12s %-12s %-12s\n",
"Detection phase", "IP ver", "Proto", "cnt", "min", "max", "avg", "tot");
fprintf(fp, "%-24s %-6s %-5s %-12s %-12s %-12s %-12s %-12s\n",
"------------------------", "------", "-----", "----------", "------------", "------------", "-----------", "-----------");
for (m = 0; m < PROF_DETECT_SIZE; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_detect_data4[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-24s IPv4 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %-6.2f\n",
PacketProfileDetectIdToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
}
for (m = 0; m < PROF_DETECT_SIZE; m++) {
int p;
for (p = 0; p < 257; p++) {
SCProfilePacketData *pd = &packet_profile_detect_data6[m][p];
if (pd->cnt == 0) {
continue;
}
FormatNumber(pd->tot, totalstr, sizeof(totalstr));
double percent = (long double)pd->tot /
(long double)total * 100;
fprintf(fp, "%-24s IPv6 %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %12s %-6.2f\n",
PacketProfileDetectIdToString(m), p, pd->cnt, pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
}
}
fclose(fp);
}
/**
* \brief Initialize profiling.
*/
void
SCProfilingInit(void)
{
ConfNode *conf;
SC_ATOMIC_INIT(samples);
intmax_t rate_v = 0;
(void)ConfGetInt("profiling.sample-rate", &rate_v);
if (rate_v > 0 && rate_v < INT_MAX) {
rate = (int)rate_v;
if (rate != 1)
SCLogInfo("profiling runs for every %dth packet", rate);
else
SCLogInfo("profiling runs for every packet");
}
conf = ConfGetNode("profiling.packets");
if (conf != NULL) {
if (ConfNodeChildValueIsTrue(conf, "enabled")) {
profiling_packets_enabled = 1;
if (pthread_mutex_init(&packet_profile_lock, NULL) != 0) {
SCLogError(SC_ERR_MUTEX,
"Failed to initialize packet profiling mutex.");
exit(EXIT_FAILURE);
}
memset(&packet_profile_data4, 0, sizeof(packet_profile_data4));
memset(&packet_profile_data6, 0, sizeof(packet_profile_data6));
memset(&packet_profile_tmm_data4, 0, sizeof(packet_profile_tmm_data4));
memset(&packet_profile_tmm_data6, 0, sizeof(packet_profile_tmm_data6));
memset(&packet_profile_app_data4, 0, sizeof(packet_profile_app_data4));
memset(&packet_profile_app_data6, 0, sizeof(packet_profile_app_data6));
memset(&packet_profile_app_pd_data4, 0, sizeof(packet_profile_app_pd_data4));
memset(&packet_profile_app_pd_data6, 0, sizeof(packet_profile_app_pd_data6));
memset(&packet_profile_detect_data4, 0, sizeof(packet_profile_detect_data4));
memset(&packet_profile_detect_data6, 0, sizeof(packet_profile_detect_data6));
memset(&packet_profile_log_data4, 0, sizeof(packet_profile_log_data4));
memset(&packet_profile_log_data6, 0, sizeof(packet_profile_log_data6));
memset(&packet_profile_flowworker_data, 0, sizeof(packet_profile_flowworker_data));
const char *filename = ConfNodeLookupChildValue(conf, "filename");
if (filename != NULL) {
char *log_dir;
log_dir = ConfigGetLogDirectory();
profiling_packets_file_name = SCMalloc(PATH_MAX);
if (unlikely(profiling_packets_file_name == NULL)) {
SCLogError(SC_ERR_MEM_ALLOC, "can't duplicate file name");
exit(EXIT_FAILURE);
}
snprintf(profiling_packets_file_name, PATH_MAX, "%s/%s", log_dir, filename);
const char *v = ConfNodeLookupChildValue(conf, "append");
if (v == NULL || ConfValIsTrue(v)) {
profiling_packets_file_mode = "a";
} else {
profiling_packets_file_mode = "w";
}
profiling_packets_output_to_file = 1;
}
}
conf = ConfGetNode("profiling.packets.csv");
if (conf != NULL) {
if (ConfNodeChildValueIsTrue(conf, "enabled")) {
const char *filename = ConfNodeLookupChildValue(conf, "filename");
if (filename == NULL) {
filename = "packet_profile.csv";
}
char *log_dir;
log_dir = ConfigGetLogDirectory();
profiling_csv_file_name = SCMalloc(PATH_MAX);
if (unlikely(profiling_csv_file_name == NULL)) {
SCLogError(SC_ERR_MEM_ALLOC, "out of memory");
exit(EXIT_FAILURE);
}
snprintf(profiling_csv_file_name, PATH_MAX, "%s/%s", log_dir, filename);
packet_profile_csv_fp = fopen(profiling_csv_file_name, "w");
if (packet_profile_csv_fp == NULL) {
return;
}
fprintf(packet_profile_csv_fp, "pcap_cnt,ipver,ipproto,total,");
int i;
for (i = 0; i < TMM_SIZE; i++) {
fprintf(packet_profile_csv_fp, "%s,", TmModuleTmmIdToString(i));
}
fprintf(packet_profile_csv_fp, "threading,");
for (i = 0; i < ALPROTO_MAX; i++) {
fprintf(packet_profile_csv_fp, "%s,", AppProtoToString(i));
}
fprintf(packet_profile_csv_fp, "proto detect,");
for (i = 0; i < PROF_DETECT_SIZE; i++) {
fprintf(packet_profile_csv_fp, "%s,", PacketProfileDetectIdToString(i));
}
fprintf(packet_profile_csv_fp, "\n");
profiling_packets_csv_enabled = 1;
}
}
}
conf = ConfGetNode("profiling.locks");
if (conf != NULL) {
if (ConfNodeChildValueIsTrue(conf, "enabled")) {
#ifndef PROFILE_LOCKING
SCLogWarning(SC_WARN_PROFILE, "lock profiling not compiled in. Add --enable-profiling-locks to configure.");
#else
profiling_locks_enabled = 1;
LockRecordInitHash();
const char *filename = ConfNodeLookupChildValue(conf, "filename");
if (filename != NULL) {
char *log_dir;
log_dir = ConfigGetLogDirectory();
profiling_locks_file_name = SCMalloc(PATH_MAX);
if (unlikely(profiling_locks_file_name == NULL)) {
SCLogError(SC_ERR_MEM_ALLOC, "can't duplicate file name");
exit(EXIT_FAILURE);
}
snprintf(profiling_locks_file_name, PATH_MAX, "%s/%s", log_dir, filename);
const char *v = ConfNodeLookupChildValue(conf, "append");
if (v == NULL || ConfValIsTrue(v)) {
profiling_locks_file_mode = "a";
} else {
profiling_locks_file_mode = "w";
}
profiling_locks_output_to_file = 1;
}
#endif
}
}
}
/**
* \brief Prints analysis of loaded rules.
*
* Warns if potential rule issues are detected. For example,
* warns if a rule uses a construct that may perform poorly,
* e.g. pcre without content or with http_method content only;
* warns if a rule uses a construct that may not be consistent with intent,
* e.g. client side ports only, http and content without any http_* modifiers, etc.
*
* \param s Pointer to the signature.
*/
void EngineAnalysisRules(Signature *s, char *line)
{
uint32_t rule_bidirectional = 0;
uint32_t rule_pcre = 0;
uint32_t rule_pcre_http = 0;
uint32_t rule_content = 0;
uint32_t rule_flow = 0;
uint32_t rule_flags = 0;
uint32_t rule_flow_toserver = 0;
uint32_t rule_flow_toclient = 0;
uint32_t rule_flow_nostream = 0;
uint32_t rule_ipv4_only = 0;
uint32_t rule_ipv6_only = 0;
uint32_t rule_flowbits = 0;
uint32_t rule_flowint = 0;
//uint32_t rule_flowvar = 0;
uint32_t rule_content_http = 0;
uint32_t rule_content_offset_depth = 0;
uint32_t list_id = 0;
uint32_t rule_warning = 0;
uint32_t raw_http_buf = 0;
uint32_t norm_http_buf = 0;
uint32_t stream_buf = 0;
uint32_t packet_buf = 0;
uint32_t http_header_buf = 0;
uint32_t http_uri_buf = 0;
uint32_t http_method_buf = 0;
uint32_t http_cookie_buf = 0;
uint32_t http_client_body_buf = 0;
uint32_t http_server_body_buf = 0;
uint32_t http_stat_code_buf = 0;
uint32_t http_stat_msg_buf = 0;
uint32_t http_raw_header_buf = 0;
uint32_t http_raw_uri_buf = 0;
uint32_t http_ua_buf = 0;
uint32_t warn_pcre_no_content = 0;
uint32_t warn_pcre_http_content = 0;
uint32_t warn_pcre_http = 0;
uint32_t warn_content_http_content = 0;
uint32_t warn_content_http = 0;
uint32_t warn_tcp_no_flow = 0;
uint32_t warn_client_ports = 0;
uint32_t warn_direction = 0;
uint32_t warn_method_toclient = 0;
uint32_t warn_method_serverbody = 0;
uint32_t warn_pcre_method = 0;
uint32_t warn_encoding_norm_http_buf = 0;
uint32_t warn_offset_depth_pkt_stream = 0;
uint32_t warn_offset_depth_alproto = 0;
uint32_t warn_non_alproto_fp_for_alproto_sig = 0;
if (s->init_flags & SIG_FLAG_INIT_BIDIREC) {
rule_bidirectional = 1;
}
if (s->flags & SIG_FLAG_REQUIRE_PACKET) {
packet_buf += 1;
}
if (s->flags & SIG_FLAG_REQUIRE_STREAM) {
stream_buf += 1;
}
if (s->proto.flags & DETECT_PROTO_IPV4) {
rule_ipv4_only += 1;
}
if (s->proto.flags & DETECT_PROTO_IPV6) {
rule_ipv6_only += 1;
}
for (list_id = 0; list_id < DETECT_SM_LIST_MAX; list_id++) {
SigMatch *sm = NULL;
for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_PCRE) {
if (list_id == DETECT_SM_LIST_HCBDMATCH) {
rule_pcre_http += 1;
http_client_body_buf += 1;
raw_http_buf += 1;
}
else if (list_id == DETECT_SM_LIST_UMATCH) {
rule_pcre_http += 1;
norm_http_buf += 1;
http_uri_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HHDMATCH) {
rule_pcre_http += 1;
norm_http_buf += 1;
http_header_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HCDMATCH) {
rule_pcre_http += 1;
norm_http_buf += 1;
http_cookie_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HSBDMATCH) {
rule_pcre_http += 1;
http_server_body_buf += 1;
raw_http_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HRHDMATCH) {
rule_pcre_http += 1;
raw_http_buf += 1;
http_raw_header_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HMDMATCH) {
rule_pcre_http += 1;
raw_http_buf += 1;
http_method_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HRUDMATCH) {
rule_pcre_http += 1;
raw_http_buf += 1;
http_raw_uri_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HSMDMATCH) {
rule_pcre_http += 1;
raw_http_buf += 1;
http_stat_msg_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HSCDMATCH) {
rule_pcre_http += 1;
raw_http_buf += 1;
http_stat_code_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HUADMATCH) {
rule_pcre_http += 1;
norm_http_buf += 1;
http_ua_buf += 1;
}
else {
rule_pcre += 1;
}
}
else if (sm->type == DETECT_CONTENT) {
if (list_id == DETECT_SM_LIST_UMATCH
|| list_id == DETECT_SM_LIST_HHDMATCH
|| list_id == DETECT_SM_LIST_HCDMATCH) {
rule_content_http += 1;
norm_http_buf += 1;
DetectContentData *cd = (DetectContentData *)sm->ctx;
if (cd != NULL && PerCentEncodingMatch(cd->content, cd->content_len) > 0) {
warn_encoding_norm_http_buf += 1;
rule_warning += 1;
}
if (list_id == DETECT_SM_LIST_UMATCH) {
http_uri_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HHDMATCH) {
http_header_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HCDMATCH) {
http_cookie_buf += 1;
}
}
else if (list_id == DETECT_SM_LIST_HCBDMATCH) {
rule_content_http += 1;
raw_http_buf += 1;
http_client_body_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HSBDMATCH) {
rule_content_http += 1;
raw_http_buf += 1;
http_server_body_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HRHDMATCH) {
rule_content_http += 1;
raw_http_buf += 1;
http_raw_header_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HRUDMATCH) {
rule_content_http += 1;
raw_http_buf += 1;
http_raw_uri_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HSMDMATCH) {
rule_content_http += 1;
raw_http_buf += 1;
http_stat_msg_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HSCDMATCH) {
rule_content_http += 1;
raw_http_buf += 1;
http_stat_code_buf += 1;
}
else if (list_id == DETECT_SM_LIST_HMDMATCH) {
rule_content_http += 1;
raw_http_buf += 1;
http_method_buf += 1;
}
else if (list_id == DETECT_SM_LIST_PMATCH) {
rule_content += 1;
DetectContentData *cd = (DetectContentData *)sm->ctx;
if (cd->flags &
(DETECT_CONTENT_OFFSET | DETECT_CONTENT_DEPTH)) {
rule_content_offset_depth++;
}
}
}
else if (sm->type == DETECT_FLOW) {
rule_flow += 1;
if ((s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_TOCLIENT)) {
rule_flow_toserver = 1;
}
else if ((s->flags & SIG_FLAG_TOCLIENT) && !(s->flags & SIG_FLAG_TOSERVER)) {
rule_flow_toclient = 1;
}
DetectFlowData *fd = (DetectFlowData *)sm->ctx;
if (fd != NULL) {
if (fd->flags & FLOW_PKT_NOSTREAM) rule_flow_nostream = 1;
}
}
else if (sm->type == DETECT_FLOWBITS) {
if (list_id == DETECT_SM_LIST_MATCH) {
rule_flowbits += 1;
}
}
else if (sm->type == DETECT_FLOWINT) {
if (list_id == DETECT_SM_LIST_MATCH) {
rule_flowint += 1;
}
}
else if (sm->type == DETECT_FLAGS) {
DetectFlagsData *fd = (DetectFlagsData *)sm->ctx;
if (fd != NULL) {
rule_flags = 1;
}
}
} /* for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) */
} /* for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) */
if (rule_pcre > 0 && rule_content == 0 && rule_content_http == 0) {
rule_warning += 1;
warn_pcre_no_content = 1;
}
if (rule_content_http > 0 && rule_pcre > 0 && rule_pcre_http == 0) {
rule_warning += 1;
warn_pcre_http_content = 1;
}
else if (s->alproto == ALPROTO_HTTP && rule_pcre > 0 && rule_pcre_http == 0) {
rule_warning += 1;
warn_pcre_http = 1;
}
if (rule_content > 0 && rule_content_http > 0) {
rule_warning += 1;
warn_content_http_content = 1;
}
if (s->alproto == ALPROTO_HTTP && rule_content > 0 && rule_content_http == 0) {
rule_warning += 1;
warn_content_http = 1;
}
if (rule_content == 1) {
//todo: warning if content is weak, separate warning for pcre + weak content
}
if (rule_flow == 0 && rule_flags == 0
&& !(s->proto.flags & DETECT_PROTO_ANY) && DetectProtoContainsProto(&s->proto, IPPROTO_TCP)
&& (rule_content || rule_content_http || rule_pcre || rule_pcre_http || rule_flowbits)) {
rule_warning += 1;
warn_tcp_no_flow = 1;
}
if (rule_flow && !rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)
&& !((s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY))) {
if (((s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY))
|| ((s->flags & SIG_FLAG_TOCLIENT) && !(s->flags & SIG_FLAG_DP_ANY) && (s->flags & SIG_FLAG_SP_ANY))) {
rule_warning += 1;
warn_client_ports = 1;
}
}
if (rule_flow && rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)) {
rule_warning += 1;
warn_direction = 1;
}
if (http_method_buf) {
if (rule_flow && rule_flow_toclient) {
rule_warning += 1;
warn_method_toclient = 1;
}
if (http_server_body_buf) {
rule_warning += 1;
warn_method_serverbody = 1;
}
if (rule_content == 0 && rule_content_http == 0 && (rule_pcre > 0 || rule_pcre_http > 0)) {
rule_warning += 1;
warn_pcre_method = 1;
}
}
if (rule_content_offset_depth > 0 && stream_buf && packet_buf) {
rule_warning += 1;
warn_offset_depth_pkt_stream = 1;
}
if (rule_content_offset_depth > 0 && !stream_buf && packet_buf && s->alproto != ALPROTO_UNKNOWN) {
rule_warning += 1;
warn_offset_depth_alproto = 1;
}
if (s->mpm_sm != NULL && s->alproto == ALPROTO_HTTP &&
SigMatchListSMBelongsTo(s, s->mpm_sm) == DETECT_SM_LIST_PMATCH) {
rule_warning += 1;
warn_non_alproto_fp_for_alproto_sig = 1;
}
if (!rule_warnings_only || (rule_warnings_only && rule_warning > 0)) {
fprintf(rule_engine_analysis_FD, "== Sid: %u ==\n", s->id);
fprintf(rule_engine_analysis_FD, "%s\n", line);
if (s->flags & SIG_FLAG_IPONLY) fprintf(rule_engine_analysis_FD, " Rule is ip only.\n");
if (rule_ipv6_only) fprintf(rule_engine_analysis_FD, " Rule is IPv6 only.\n");
if (rule_ipv4_only) fprintf(rule_engine_analysis_FD, " Rule is IPv4 only.\n");
if (packet_buf) fprintf(rule_engine_analysis_FD, " Rule matches on packets.\n");
if (!rule_flow_nostream && stream_buf && (rule_flow || rule_flowbits || rule_content || rule_pcre)) {
fprintf(rule_engine_analysis_FD, " Rule matches on reassembled stream.\n");
}
if (http_uri_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http uri buffer.\n");
if (http_header_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http header buffer.\n");
if (http_cookie_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http cookie buffer.\n");
if (http_raw_uri_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http raw uri buffer.\n");
if (http_raw_header_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http raw header buffer.\n");
if (http_method_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http method buffer.\n");
if (http_server_body_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http server body buffer.\n");
if (http_client_body_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http client body buffer.\n");
if (http_stat_msg_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http stat msg buffer.\n");
if (http_stat_code_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http stat code buffer.\n");
if (http_ua_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http user agent buffer.\n");
if (s->alproto != ALPROTO_UNKNOWN) {
fprintf(rule_engine_analysis_FD, " App layer protocol is %s.\n", AppProtoToString(s->alproto));
}
if (rule_content || rule_content_http || rule_pcre || rule_pcre_http) {
fprintf(rule_engine_analysis_FD, " Rule contains %d content options, %d http content options, %d pcre options, and %d pcre options with http modifiers.\n", rule_content, rule_content_http, rule_pcre, rule_pcre_http);
}
/* print fast pattern info */
EngineAnalysisRulesPrintFP(s);
/* this is where the warnings start */
if (warn_pcre_no_content /*rule_pcre > 0 && rule_content == 0 && rule_content_http == 0*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule uses pcre without a content option present.\n"
" -Consider adding a content to improve performance of this rule.\n");
}
if (warn_pcre_http_content /*rule_content_http > 0 && rule_pcre > 0 && rule_pcre_http == 0*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule uses content options with http_* and pcre options without http modifiers.\n"
" -Consider adding http pcre modifier.\n");
}
else if (warn_pcre_http /*s->alproto == ALPROTO_HTTP && rule_pcre > 0 && rule_pcre_http == 0*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule app layer protocol is http, but pcre options do not have http modifiers.\n"
" -Consider adding http pcre modifiers.\n");
}
if (warn_content_http_content /*rule_content > 0 && rule_content_http > 0*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule contains content with http_* and content without http_*.\n"
" -Consider adding http content modifiers.\n");
}
if (warn_content_http /*s->alproto == ALPROTO_HTTP && rule_content > 0 && rule_content_http == 0*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule app layer protocol is http, but content options do not have http_* modifiers.\n"
" -Consider adding http content modifiers.\n");
}
if (rule_content == 1) {
//todo: warning if content is weak, separate warning for pcre + weak content
}
if (warn_encoding_norm_http_buf) {
fprintf(rule_engine_analysis_FD, " Warning: Rule may contain percent encoded content for a normalized http buffer match.\n");
}
if (warn_tcp_no_flow /*rule_flow == 0 && rule_flow == 0
&& !(s->proto.flags & DETECT_PROTO_ANY) && DetectProtoContainsProto(&s->proto, IPPROTO_TCP)*/) {
fprintf(rule_engine_analysis_FD, " Warning: TCP rule without a flow or flags option.\n"
" -Consider adding flow or flags to improve performance of this rule.\n");
}
if (warn_client_ports /*rule_flow && !rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)
&& !((s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY)))
if (((s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY))
|| ((s->flags & SIG_FLAG_TOCLIENT) && !(s->flags & SIG_FLAG_DP_ANY) && (s->flags & SIG_FLAG_SP_ANY))*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule contains ports or port variables only on the client side.\n"
" -Flow direction possibly inconsistent with rule.\n");
}
if (warn_direction /*rule_flow && rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule is bidirectional and has a flow option with a specific direction.\n");
}
if (warn_method_toclient /*http_method_buf && rule_flow && rule_flow_toclient*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule uses content or pcre for http_method with flow:to_client or from_server\n");
}
if (warn_method_serverbody /*http_method_buf && http_server_body_buf*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule uses content or pcre for http_method with content or pcre for http_server_body.\n");
}
if (warn_pcre_method /*http_method_buf && rule_content == 0 && rule_content_http == 0
&& (rule_pcre > 0 || rule_pcre_http > 0)*/) {
fprintf(rule_engine_analysis_FD, " Warning: Rule uses pcre with only a http_method content; possible performance issue.\n");
}
if (warn_offset_depth_pkt_stream) {
fprintf(rule_engine_analysis_FD, " Warning: Rule has depth"
"/offset with raw content keywords. Please note the "
"offset/depth will be checked against both packet "
"payloads and stream. If you meant to have the offset/"
"depth checked against just the payload, you can update "
"the signature as \"alert tcp-pkt...\"\n");
}
if (warn_offset_depth_alproto) {
fprintf(rule_engine_analysis_FD, " Warning: Rule has "
"offset/depth set along with a match on a specific "
"app layer protocol - %d. This can lead to FNs if we "
"have a offset/depth content match on a packet payload "
"before we can detect the app layer protocol for the "
"flow.\n", s->alproto);
}
if (warn_non_alproto_fp_for_alproto_sig) {
fprintf(rule_engine_analysis_FD, " Warning: Rule app layer "
"protocol is http, but the fast_pattern is set on the raw "
"stream. Consider adding fast_pattern over a http "
"buffer for increased performance.");
}
if (rule_warning == 0) {
fprintf(rule_engine_analysis_FD, " No warnings for this rule.\n");
}
fprintf(rule_engine_analysis_FD, "\n");
}
return;
}
int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto,
TxLogger LogFunc,
OutputCtx *output_ctx, int tc_log_progress,
int ts_log_progress, TxLoggerCondition LogCondition,
ThreadInitFunc ThreadInit,
ThreadDeinitFunc ThreadDeinit,
void (*ThreadExitPrintStats)(ThreadVars *, void *))
{
if (!(AppLayerParserIsTxAware(alproto))) {
SCLogNotice("%s logger not enabled: protocol %s is disabled",
name, AppProtoToString(alproto));
return -1;
}
OutputTxLogger *op = SCMalloc(sizeof(*op));
if (op == NULL)
return -1;
memset(op, 0x00, sizeof(*op));
op->alproto = alproto;
op->LogFunc = LogFunc;
op->LogCondition = LogCondition;
op->output_ctx = output_ctx;
op->name = name;
op->logger_id = id;
op->ThreadInit = ThreadInit;
op->ThreadDeinit = ThreadDeinit;
op->ThreadExitPrintStats = ThreadExitPrintStats;
if (tc_log_progress < 0) {
op->tc_log_progress =
AppLayerParserGetStateProgressCompletionStatus(alproto,
STREAM_TOCLIENT);
} else {
op->tc_log_progress = tc_log_progress;
}
if (ts_log_progress < 0) {
op->ts_log_progress =
AppLayerParserGetStateProgressCompletionStatus(alproto,
STREAM_TOSERVER);
} else {
op->ts_log_progress = ts_log_progress;
}
if (list == NULL) {
op->id = 1;
list = op;
} else {
OutputTxLogger *t = list;
while (t->next)
t = t->next;
if (t->id * 2 > UINT32_MAX) {
SCLogError(SC_ERR_FATAL, "Too many loggers registered.");
exit(EXIT_FAILURE);
}
op->id = t->id * 2;
t->next = op;
}
SCLogDebug("OutputRegisterTxLogger happy");
return 0;
}
/**
* \internal
* \brief Write meta data on a single line json record
*/
static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff)
{
json_t *js = CreateJSONHeader((Packet *)p, 0, "fileinfo"); //TODO const
json_t *hjs = NULL;
if (unlikely(js == NULL))
return;
/* reset */
MemBufferReset(aft->buffer);
switch (p->flow->alproto) {
case ALPROTO_HTTP:
hjs = JsonHttpAddMetadata(p->flow, ff->txid);
if (hjs)
json_object_set_new(js, "http", hjs);
break;
case ALPROTO_SMTP:
hjs = JsonSMTPAddMetadata(p->flow, ff->txid);
if (hjs)
json_object_set_new(js, "smtp", hjs);
hjs = JsonEmailAddMetadata(p->flow, ff->txid);
if (hjs)
json_object_set_new(js, "email", hjs);
break;
}
json_object_set_new(js, "app_proto",
json_string(AppProtoToString(p->flow->alproto)));
json_t *fjs = json_object();
if (unlikely(fjs == NULL)) {
json_decref(js);
return;
}
char *s = BytesToString(ff->name, ff->name_len);
json_object_set_new(fjs, "filename", json_string(s));
if (s != NULL)
SCFree(s);
if (ff->magic)
json_object_set_new(fjs, "magic", json_string((char *)ff->magic));
switch (ff->state) {
case FILE_STATE_CLOSED:
json_object_set_new(fjs, "state", json_string("CLOSED"));
#ifdef HAVE_NSS
if (ff->flags & FILE_MD5) {
size_t x;
int i;
char s[256];
for (i = 0, x = 0; x < sizeof(ff->md5); x++) {
i += snprintf(&s[i], 255-i, "%02x", ff->md5[x]);
}
json_object_set_new(fjs, "md5", json_string(s));
}
#endif
break;
case FILE_STATE_TRUNCATED:
json_object_set_new(fjs, "state", json_string("TRUNCATED"));
break;
case FILE_STATE_ERROR:
json_object_set_new(fjs, "state", json_string("ERROR"));
break;
default:
json_object_set_new(fjs, "state", json_string("UNKNOWN"));
break;
}
json_object_set_new(fjs, "stored",
(ff->flags & FILE_STORED) ? json_true() : json_false());
if (ff->flags & FILE_STORED) {
json_object_set_new(fjs, "file_id", json_integer(ff->file_id));
}
json_object_set_new(fjs, "size", json_integer(ff->size));
json_object_set_new(fjs, "tx_id", json_integer(ff->txid));
/* originally just 'file', but due to bug 1127 naming it fileinfo */
json_object_set_new(js, "fileinfo", fjs);
OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, &aft->buffer);
json_object_del(js, "fileinfo");
switch (p->flow->alproto) {
case ALPROTO_HTTP:
json_object_del(js, "http");
break;
case ALPROTO_SMTP:
json_object_del(js, "smtp");
json_object_del(js, "email");
break;
}
json_object_clear(js);
json_decref(js);
}
static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *filename, int ipver) {
char metafilename[PATH_MAX] = "";
snprintf(metafilename, sizeof(metafilename), "%s.meta", filename);
FILE *fp = fopen(metafilename, "w+");
if (fp != NULL) {
char timebuf[64];
CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
fprintf(fp, "TIME: %s\n", timebuf);
if (p->pcap_cnt > 0) {
fprintf(fp, "PCAP PKT NUM: %"PRIu64"\n", p->pcap_cnt);
}
char srcip[46], dstip[46];
Port sp, dp;
switch (ipver) {
case AF_INET:
PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip));
PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip));
break;
case AF_INET6:
PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip));
PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip));
break;
default:
strlcpy(srcip, "<unknown>", sizeof(srcip));
strlcpy(dstip, "<unknown>", sizeof(dstip));
break;
}
sp = p->sp;
dp = p->dp;
fprintf(fp, "SRC IP: %s\n", srcip);
fprintf(fp, "DST IP: %s\n", dstip);
fprintf(fp, "PROTO: %" PRIu32 "\n", p->proto);
if (PKT_IS_TCP(p) || PKT_IS_UDP(p)) {
fprintf(fp, "SRC PORT: %" PRIu16 "\n", sp);
fprintf(fp, "DST PORT: %" PRIu16 "\n", dp);
}
fprintf(fp, "APP PROTO: %s\n",
AppProtoToString(p->flow->alproto));
/* Only applicable to HTTP traffic */
if (p->flow->alproto == ALPROTO_HTTP) {
fprintf(fp, "HTTP URI: ");
LogFilestoreMetaGetUri(fp, p, ff);
fprintf(fp, "\n");
fprintf(fp, "HTTP HOST: ");
LogFilestoreMetaGetHost(fp, p, ff);
fprintf(fp, "\n");
fprintf(fp, "HTTP REFERER: ");
LogFilestoreMetaGetReferer(fp, p, ff);
fprintf(fp, "\n");
fprintf(fp, "HTTP USER AGENT: ");
LogFilestoreMetaGetUserAgent(fp, p, ff);
fprintf(fp, "\n");
} else if (p->flow->alproto == ALPROTO_SMTP) {
/* Only applicable to SMTP */
LogFilestoreMetaGetSmtp(fp, p, ff);
}
fprintf(fp, "FILENAME: ");
PrintRawUriFp(fp, ff->name, ff->name_len);
fprintf(fp, "\n");
fclose(fp);
}
}
/* JSON format logging */
static void JsonFlowLogJSON(JsonFlowLogThread *aft, json_t *js, Flow *f)
{
#if 0
LogJsonFileCtx *flow_ctx = aft->flowlog_ctx;
#endif
json_t *hjs = json_object();
if (hjs == NULL) {
return;
}
json_object_set_new(js, "app_proto", json_string(AppProtoToString(f->alproto)));
json_object_set_new(hjs, "pkts_toserver",
json_integer(f->todstpktcnt));
json_object_set_new(hjs, "pkts_toclient",
json_integer(f->tosrcpktcnt));
json_object_set_new(hjs, "bytes_toserver",
json_integer(f->todstbytecnt));
json_object_set_new(hjs, "bytes_toclient",
json_integer(f->tosrcbytecnt));
char timebuf1[64], timebuf2[64];
CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
json_object_set_new(hjs, "start", json_string(timebuf1));
json_object_set_new(hjs, "end", json_string(timebuf2));
int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
json_object_set_new(hjs, "age",
json_integer(age));
if (f->flow_end_flags & FLOW_END_FLAG_EMERGENCY)
json_object_set_new(hjs, "emergency", json_true());
const char *state = NULL;
if (f->flow_end_flags & FLOW_END_FLAG_STATE_NEW)
state = "new";
else if (f->flow_end_flags & FLOW_END_FLAG_STATE_ESTABLISHED)
state = "established";
else if (f->flow_end_flags & FLOW_END_FLAG_STATE_CLOSED)
state = "closed";
json_object_set_new(hjs, "state",
json_string(state));
const char *reason = NULL;
if (f->flow_end_flags & FLOW_END_FLAG_TIMEOUT)
reason = "timeout";
else if (f->flow_end_flags & FLOW_END_FLAG_FORCED)
reason = "forced";
else if (f->flow_end_flags & FLOW_END_FLAG_SHUTDOWN)
reason = "shutdown";
json_object_set_new(hjs, "reason",
json_string(reason));
json_object_set_new(js, "flow", hjs);
/* TCP */
if (f->proto == IPPROTO_TCP) {
json_t *tjs = json_object();
if (tjs == NULL) {
return;
}
TcpSession *ssn = f->protoctx;
char hexflags[3] = "";
snprintf(hexflags, sizeof(hexflags), "%02x",
ssn ? ssn->tcp_packet_flags : 0);
json_object_set_new(tjs, "tcp_flags", json_string(hexflags));
snprintf(hexflags, sizeof(hexflags), "%02x",
ssn ? ssn->client.tcp_flags : 0);
json_object_set_new(tjs, "tcp_flags_ts", json_string(hexflags));
snprintf(hexflags, sizeof(hexflags), "%02x",
ssn ? ssn->server.tcp_flags : 0);
json_object_set_new(tjs, "tcp_flags_tc", json_string(hexflags));
JsonTcpFlags(ssn ? ssn->tcp_packet_flags : 0, tjs);
if (ssn) {
char *state = NULL;
switch (ssn->state) {
case TCP_NONE:
state = "none";
break;
case TCP_LISTEN:
state = "listen";
break;
case TCP_SYN_SENT:
state = "syn_sent";
break;
case TCP_SYN_RECV:
state = "syn_recv";
break;
case TCP_ESTABLISHED:
state = "established";
break;
case TCP_FIN_WAIT1:
state = "fin_wait1";
break;
case TCP_FIN_WAIT2:
state = "fin_wait2";
break;
case TCP_TIME_WAIT:
state = "time_wait";
break;
case TCP_LAST_ACK:
state = "last_ack";
break;
case TCP_CLOSE_WAIT:
state = "close_wait";
break;
case TCP_CLOSING:
state = "closing";
break;
case TCP_CLOSED:
state = "closed";
break;
}
json_object_set_new(tjs, "state", json_string(state));
}
json_object_set_new(js, "tcp", tjs);
}
}