/**
* \brief Close the file in the flow
*
* \param s http state
* \param data data chunk if any
* \param data_len length of the data portion
* \param flags flags to indicate events
* \param direction flow direction
*
* Currently on the FLOW_FILE_TRUNCATED flag is implemented, indicating
* that the file isn't complete but we're stopping storing it.
*
* \retval 0 ok
* \retval -1 error
* \retval -2 not storing files on this flow/tx
*/
int HTPFileClose(HtpState *s, uint8_t *data, uint32_t data_len,
uint8_t flags, uint8_t direction)
{
SCEnter();
int retval = 0;
int result = 0;
FileContainer *files = NULL;
if (s == NULL) {
SCReturnInt(-1);
}
if (direction & STREAM_TOCLIENT) {
files = s->files_tc;
} else {
files = s->files_ts;
}
if (files == NULL) {
retval = -1;
goto end;
}
result = FileCloseFile(files, data, data_len, flags);
if (result == -1) {
retval = -1;
} else if (result == -2) {
retval = -2;
}
FilePrune(files);
end:
SCReturnInt(retval);
}
static TmEcode LogFileLogWrap(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq, int ipver)
{
SCEnter();
LogFileLogThread *aft = (LogFileLogThread *)data;
uint8_t flags = 0;
/* no flow, no htp state */
if (p->flow == NULL) {
SCReturnInt(TM_ECODE_OK);
}
if (p->flowflags & FLOW_PKT_TOCLIENT)
flags |= STREAM_TOCLIENT;
else
flags |= STREAM_TOSERVER;
int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0;
int file_trunc = 0;
FLOWLOCK_WRLOCK(p->flow);
file_trunc = StreamTcpReassembleDepthReached(p);
FileContainer *ffc = AppLayerParserGetFiles(IPPROTO_TCP, p->flow->alproto,
p->flow->alstate, flags);
SCLogDebug("ffc %p", ffc);
if (ffc != NULL) {
File *ff;
for (ff = ffc->head; ff != NULL; ff = ff->next) {
if (ff->flags & FILE_LOGGED)
continue;
if (FileForceMagic() && ff->magic == NULL) {
FilemagicGlobalLookup(ff);
}
SCLogDebug("ff %p", ff);
if (file_trunc && ff->state < FILE_STATE_CLOSED)
ff->state = FILE_STATE_TRUNCATED;
if (ff->state == FILE_STATE_CLOSED ||
ff->state == FILE_STATE_TRUNCATED || ff->state == FILE_STATE_ERROR ||
(file_close == 1 && ff->state < FILE_STATE_CLOSED))
{
LogFileWriteJsonRecord(aft, p, ff, ipver);
ff->flags |= FILE_LOGGED;
aft->file_cnt++;
}
}
FilePrune(ffc);
}
FLOWLOCK_UNLOCK(p->flow);
SCReturnInt(TM_ECODE_OK);
}
/**
* \brief Store a chunk of data in the flow
*
* \param s http state
* \param data data chunk (if any)
* \param data_len length of the data portion
* \param direction flow direction
*
* \retval 0 ok
* \retval -1 error
* \retval -2 file doesn't need storing
*/
int HTPFileStoreChunk(HtpState *s, uint8_t *data, uint32_t data_len,
uint8_t direction)
{
SCEnter();
int retval = 0;
int result = 0;
FileContainer *files = NULL;
if (s == NULL) {
SCReturnInt(-1);
}
if (direction & STREAM_TOCLIENT) {
files = s->files_tc;
} else {
files = s->files_ts;
}
if (files == NULL) {
SCLogDebug("no files in state");
retval = -1;
goto end;
}
result = FileAppendData(files, data, data_len);
if (result == -1) {
SCLogDebug("appending data failed");
retval = -1;
} else if (result == -2) {
retval = -2;
}
FilePrune(files);
end:
SCReturnInt(retval);
}
static TmEcode LogFilestoreLogWrap(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq, int ipver)
{
SCEnter();
LogFilestoreLogThread *aft = (LogFilestoreLogThread *)data;
uint8_t flags = 0;
/* no flow, no htp state */
if (p->flow == NULL) {
SCReturnInt(TM_ECODE_OK);
}
if (p->flowflags & FLOW_PKT_TOCLIENT)
flags |= STREAM_TOCLIENT;
else
flags |= STREAM_TOSERVER;
int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0;
int file_trunc = 0;
FLOWLOCK_WRLOCK(p->flow);
file_trunc = StreamTcpReassembleDepthReached(p);
FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow, flags);
SCLogDebug("ffc %p", ffc);
if (ffc != NULL) {
File *ff;
for (ff = ffc->head; ff != NULL; ff = ff->next) {
int file_fd = -1;
if (FileForceMagic() && ff->magic == NULL) {
FilemagicGlobalLookup(ff);
}
SCLogDebug("ff %p", ff);
if (ff->flags & FILE_STORED) {
SCLogDebug("stored flag set");
continue;
}
if (!(ff->flags & FILE_STORE)) {
SCLogDebug("ff FILE_STORE not set");
continue;
}
FileData *ffd;
for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) {
SCLogDebug("ffd %p", ffd);
if (ffd->stored == 1) {
if (file_close == 1 && ffd->next == NULL) {
LogFilestoreLogCloseMetaFile(ff);
ff->flags |= FILE_STORED;
}
continue;
}
/* store */
SCLogDebug("trying to open file");
char filename[PATH_MAX] = "";
if (ff->file_id == 0) {
ff->file_id = SC_ATOMIC_ADD(file_id, 1);
snprintf(filename, sizeof(filename), "%s/file.%u",
g_logfile_base_dir, ff->file_id);
file_fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644);
if (file_fd == -1) {
SCLogDebug("failed to open file");
continue;
}
/* create a .meta file that contains time, src/dst/sp/dp/proto */
LogFilestoreLogCreateMetaFile(p, ff, filename, ipver);
aft->file_cnt++;
} else {
snprintf(filename, sizeof(filename), "%s/file.%u",
g_logfile_base_dir, ff->file_id);
file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY);
if (file_fd == -1) {
SCLogDebug("failed to open file %s: %s", filename, strerror(errno));
continue;
}
}
ssize_t r = write(file_fd, (const void *)ffd->data, (size_t)ffd->len);
if (r == -1) {
SCLogDebug("write failed: %s", strerror(errno));
close(file_fd);
continue;
}
close(file_fd);
if (file_trunc && ff->state < FILE_STATE_CLOSED)
ff->state = FILE_STATE_TRUNCATED;
if (ff->state == FILE_STATE_CLOSED ||
ff->state == FILE_STATE_TRUNCATED ||
ff->state == FILE_STATE_ERROR ||
(file_close == 1 && ff->state < FILE_STATE_CLOSED))
{
if (ffd->next == NULL) {
LogFilestoreLogCloseMetaFile(ff);
ff->flags |= FILE_STORED;
}
}
ffd->stored = 1;
}
}
FilePrune(ffc);
}
FLOWLOCK_UNLOCK(p->flow);
SCReturnInt(TM_ECODE_OK);
}
/**
* \brief Open the file with "filename" and pass the first chunk
* of data if any.
*
* \param s http state
* \param filename name of the file
* \param filename_len length of the name
* \param data data chunk (if any)
* \param data_len length of the data portion
* \param direction flow direction
*
* \retval 0 ok
* \retval -1 error
* \retval -2 not handling files on this flow
*/
int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len,
uint8_t *data, uint32_t data_len, uint16_t txid, uint8_t direction)
{
int retval = 0;
uint8_t flags = 0;
FileContainer *files = NULL;
FileContainer *files_opposite = NULL;
SCLogDebug("data %p data_len %"PRIu32, data, data_len);
if (s == NULL) {
SCReturnInt(-1);
}
if (direction & STREAM_TOCLIENT) {
if (s->files_tc == NULL) {
s->files_tc = FileContainerAlloc();
if (s->files_tc == NULL) {
retval = -1;
goto end;
}
}
files = s->files_tc;
files_opposite = s->files_ts;
if (s->flags & HTP_FLAG_STORE_FILES_TS ||
(s->flags & HTP_FLAG_STORE_FILES_TX_TS && txid == s->store_tx_id)) {
flags |= FILE_STORE;
}
if (s->f->flags & FLOW_FILE_NO_MAGIC_TC) {
SCLogDebug("no magic for this flow in toclient direction, so none for this file");
flags |= FILE_NOMAGIC;
}
if (s->f->flags & FLOW_FILE_NO_MD5_TC) {
SCLogDebug("no md5 for this flow in toclient direction, so none for this file");
flags |= FILE_NOMD5;
}
if (!(flags & FILE_STORE) && s->f->flags & FLOW_FILE_NO_STORE_TC) {
flags |= FILE_NOSTORE;
}
} else {
if (s->files_ts == NULL) {
s->files_ts = FileContainerAlloc();
if (s->files_ts == NULL) {
retval = -1;
goto end;
}
}
files = s->files_ts;
files_opposite = s->files_tc;
if (s->flags & HTP_FLAG_STORE_FILES_TC ||
(s->flags & HTP_FLAG_STORE_FILES_TX_TC && txid == s->store_tx_id)) {
flags |= FILE_STORE;
}
if (s->f->flags & FLOW_FILE_NO_MAGIC_TS) {
SCLogDebug("no magic for this flow in toserver direction, so none for this file");
flags |= FILE_NOMAGIC;
}
if (s->f->flags & FLOW_FILE_NO_MD5_TS) {
SCLogDebug("no md5 for this flow in toserver direction, so none for this file");
flags |= FILE_NOMD5;
}
if (!(flags & FILE_STORE) && s->f->flags & FLOW_FILE_NO_STORE_TS) {
flags |= FILE_NOSTORE;
}
}
/* if the previous file is in the same txid, we reset the file part of the
* stateful detection engine. We cannot do that here directly, because of
* locking order. Flow is locked at this point and we can't lock flow
* before de_state */
if (files != NULL && files->tail != NULL && files->tail->txid == txid) {
SCLogDebug("new file in same tx, flagging http state for de_state reset");
if (direction & STREAM_TOCLIENT) {
s->flags |= HTP_FLAG_NEW_FILE_TX_TC;
} else {
s->flags |= HTP_FLAG_NEW_FILE_TX_TS;
}
}
if (files_opposite != NULL && files_opposite->tail != NULL && files_opposite->tail->txid == txid) {
SCLogDebug("new file in same tx, flagging http state for de_state reset");
if (direction & STREAM_TOCLIENT) {
SCLogDebug("flagging TC");
s->flags |= HTP_FLAG_NEW_FILE_TX_TC;
} else {
SCLogDebug("flagging TS");
s->flags |= HTP_FLAG_NEW_FILE_TX_TS;
}
}
if (FileOpenFile(files, filename, filename_len,
data, data_len, flags) == NULL)
{
retval = -1;
}
FileSetTx(files->tail, txid);
FilePrune(files);
end:
SCReturnInt(retval);
}
static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQueue *pq, PacketQueue *postpq)
{
BUG_ON(thread_data == NULL);
BUG_ON(list == NULL);
OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
OutputFileLogger *logger = list;
OutputLoggerThreadStore *store = op_thread_data->store;
BUG_ON(logger == NULL && store != NULL);
BUG_ON(logger != NULL && store == NULL);
BUG_ON(logger == NULL && store == NULL);
uint8_t flags = 0;
Flow * const f = p->flow;
/* no flow, no files */
if (f == NULL) {
SCReturnInt(TM_ECODE_OK);
}
if (p->flowflags & FLOW_PKT_TOCLIENT)
flags |= STREAM_TOCLIENT;
else
flags |= STREAM_TOSERVER;
int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0;
int file_trunc = 0;
FLOWLOCK_WRLOCK(f); // < need write lock for FilePrune below
file_trunc = StreamTcpReassembleDepthReached(p);
FileContainer *ffc = AppLayerParserGetFiles(p->proto, f->alproto,
f->alstate, flags);
SCLogDebug("ffc %p", ffc);
if (ffc != NULL) {
File *ff;
for (ff = ffc->head; ff != NULL; ff = ff->next) {
if (ff->flags & FILE_LOGGED)
continue;
SCLogDebug("ff %p", ff);
if (file_trunc && ff->state < FILE_STATE_CLOSED)
ff->state = FILE_STATE_TRUNCATED;
if (file_close && ff->state < FILE_STATE_CLOSED)
ff->state = FILE_STATE_TRUNCATED;
if (ff->state == FILE_STATE_CLOSED ||
ff->state == FILE_STATE_TRUNCATED ||
ff->state == FILE_STATE_ERROR)
{
int file_logged = 0;
if (FileForceMagic() && ff->magic == NULL) {
FilemagicGlobalLookup(ff);
}
logger = list;
store = op_thread_data->store;
while (logger && store) {
BUG_ON(logger->LogFunc == NULL);
SCLogDebug("logger %p", logger);
PACKET_PROFILING_TMM_START(p, logger->module_id);
logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff);
PACKET_PROFILING_TMM_END(p, logger->module_id);
file_logged = 1;
logger = logger->next;
store = store->next;
BUG_ON(logger == NULL && store != NULL);
BUG_ON(logger != NULL && store == NULL);
}
if (file_logged) {
ff->flags |= FILE_LOGGED;
}
}
}
FilePrune(ffc);
}
FLOWLOCK_UNLOCK(f);
return TM_ECODE_OK;
}
