int main(int argc, char **argv)
{
skstream_t *in_stream;
int in_rv = SKSTREAM_OK;
int rv = SKSTREAM_OK;
appSetup(argc, argv); /* never returns on error */
#error "Loop over files on command line or read from stdin."
#error "Process each file, preferably in a separate function."
/* For each input, process each record */
while (NULL != (in_stream = appNextInput(argc, argv))) {
while ((in_rv = skStreamReadRecord(in_stream, &rwrec))==SKSTREAM_OK) {
/* process record */
rv = skStreamWriteRecord(out_stream, &rwrec);
if (SKSTREAM_OK != rv) {
skStreamPrintLastErr(out_stream, rv, &skAppPrintErr);
skStreamDestroy(&in_stream);
goto END;
}
}
if (SKSTREAM_ERR_EOF != in_rv) {
skStreamPrintLastErr(in_stream, in_rv, &skAppPrintErr);
}
skStreamDestroy(&in_stream);
}
rv = skStreamClose(out_stream);
if (SKSTREAM_OK != rv) {
skStreamPrintLastErr(out_stream, rv, &skAppPrintErr);
}
END:
return ((SKSTREAM_OK == rv) ? EXIT_SUCCESS : EXIT_FAILURE);
}
/*
* status = tailFile(stream);
*
* Read SiLK flow records from the file at 'stream' and store the
* most recent 'tail_recs' number of records in the 'tail_buf'
* buffer.
*
* Return -1 on read error, or 0 otherwise.
*/
static int
tailFile(
skstream_t *stream)
{
int rv = SKSTREAM_OK;
while ((rv = skStreamReadRecord(stream, tail_buf_cur)) == SKSTREAM_OK) {
++tail_buf_cur;
if (tail_buf_cur == &tail_buf[tail_recs]) {
tail_buf_cur = tail_buf;
tail_buf_full = 1;
}
}
if (SKSTREAM_ERR_EOF != rv) {
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
return -1;
}
return 0;
}
/*
* status = readerGetRecord(&out_rwrec, &out_probe, flow_processor);
*
* Invoked by input_mode_type->get_record_fn();
*/
static fp_get_record_result_t
readerGetRecord(
rwRec *out_rwrec,
const skpc_probe_t **out_probe,
flow_proc_t *fproc)
{
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
skstream_t *fcfile;
const char *filename;
fp_get_record_result_t retVal = FP_GET_ERROR;
int rv;
pthread_mutex_lock(&mutex);
/* If we don't have a source, get a file from the directory poller
* and start processing it. */
if (fproc->flow_src == NULL) {
switch (readerGetNextValidFile(fproc)) {
case 0:
/* Success */
break;
case -1:
/* Error getting file name (maybe in shutdown?) */
goto END;
case -2: /* Error opening file */
default: /* Unexpected value */
retVal = FP_FATAL_ERROR;
goto END;
}
}
fcfile = (skstream_t*)fproc->flow_src;
/* Assume we can get a record from the probe. */
retVal = FP_RECORD;
*out_probe = fproc->probe;
/* Try to get a record */
rv = skStreamReadRecord(fcfile, out_rwrec);
if (rv) {
/* get failed: either at EOF or got an error. */
if (rv != SKSTREAM_ERR_EOF) {
skStreamPrintLastErr(fcfile, rv, &WARNINGMSG);
}
retVal = FP_FILE_BREAK;
*out_probe = NULL;
/* Print results for the file we just processed. */
filename = skStreamGetPathname(fcfile);
INFOMSG("Processed file %s, %" PRIu64 " records.",
filename, skStreamGetRecordCount(fcfile));
skStreamClose(fcfile);
/* Either archive the file or remove it */
archiveDirectoryInsertOrRemove(filename, NULL);
/* All done with the flow source */
skStreamDestroy(&fcfile);
fproc->flow_src = NULL;
fproc->probe = NULL;
}
END:
pthread_mutex_unlock(&mutex);
return retVal;
}
/*
* sortRandom();
*
* Don't make any assumptions about the input. Store the input
* records in a large buffer, and sort those in-core records once
* all records are processed or the buffer is full. If the buffer
* fills up, store the sorted records into temporary files. Once
* all records are read, use mergeFiles() above to merge-sort the
* temporary files.
*
* Exits the application if an error occurs.
*/
static void
sortRandom(
void)
{
int temp_file_idx = -1;
skstream_t *input_rwios = NULL; /* input stream */
uint8_t *record_buffer = NULL; /* Region of memory for records */
uint8_t *cur_node = NULL; /* Ptr into record_buffer */
uint8_t *next_node = NULL; /* Ptr into record_buffer */
uint32_t buffer_max_recs; /* max buffer size (in number of recs) */
uint32_t buffer_recs; /* current buffer size (# records) */
uint32_t buffer_chunk_recs; /* how to grow from current to max buf */
uint32_t num_chunks; /* how quickly to grow buffer */
uint32_t record_count = 0; /* Number of records read */
int rv;
/* Determine the maximum number of records that will fit into the
* buffer if it grows the maximum size */
buffer_max_recs = buffer_size / NODE_SIZE;
TRACEMSG((("buffer_size = %" PRIu64
"\nnode_size = %" PRIu32
"\nbuffer_max_recs = %" PRIu32),
buffer_size, NODE_SIZE, buffer_max_recs));
/* We will grow to the maximum size in chunks */
num_chunks = NUM_CHUNKS;
if (num_chunks <= 0) {
num_chunks = 1;
}
/* Attempt to allocate the initial chunk. If we fail, increment
* the number of chunks---which will decrease the amount we
* attempt to allocate at once---and try again. */
for (;;) {
buffer_chunk_recs = buffer_max_recs / num_chunks;
TRACEMSG((("num_chunks = %" PRIu32
"\nbuffer_chunk_recs = %" PRIu32),
num_chunks, buffer_chunk_recs));
record_buffer = (uint8_t*)malloc(NODE_SIZE * buffer_chunk_recs);
if (record_buffer) {
/* malloc was successful */
break;
} else if (buffer_chunk_recs < MIN_IN_CORE_RECORDS) {
/* give up at this point */
skAppPrintErr("Error allocating space for %d records",
MIN_IN_CORE_RECORDS);
appExit(EXIT_FAILURE);
} else {
/* reduce the amount we allocate at once by increasing the
* number of chunks and try again */
TRACEMSG(("malloc() failed"));
++num_chunks;
}
}
buffer_recs = buffer_chunk_recs;
TRACEMSG((("buffer_recs = %" PRIu32), buffer_recs));
/* open first file */
rv = appNextInput(&input_rwios);
if (rv < 0) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
record_count = 0;
cur_node = record_buffer;
while (input_rwios != NULL) {
/* read record */
if ((rv = skStreamReadRecord(input_rwios, (rwRec*)cur_node))
!= SKSTREAM_OK)
{
if (rv != SKSTREAM_ERR_EOF) {
skStreamPrintLastErr(input_rwios, rv, &skAppPrintErr);
}
/* end of file: close current and open next */
skStreamDestroy(&input_rwios);
rv = appNextInput(&input_rwios);
if (rv < 0) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
continue;
}
++record_count;
cur_node += NODE_SIZE;
if (record_count == buffer_recs) {
/* Filled the current buffer */
/* If buffer not at max size, see if we can grow it */
if (buffer_recs < buffer_max_recs) {
uint8_t *old_buf = record_buffer;
/* add a chunk of records. if we are near the max,
* set the size to the max */
buffer_recs += buffer_chunk_recs;
if (buffer_recs + buffer_chunk_recs > buffer_max_recs) {
buffer_recs = buffer_max_recs;
}
TRACEMSG((("Buffer full---attempt to grow to %" PRIu32
" records, %" PRIu32 " bytes"),
buffer_recs, NODE_SIZE * buffer_recs));
/* attempt to grow */
record_buffer = (uint8_t*)realloc(record_buffer,
NODE_SIZE * buffer_recs);
if (record_buffer) {
/* Success, make certain cur_node points into the
* new buffer */
cur_node = (record_buffer + (record_count * NODE_SIZE));
} else {
/* Unable to grow it */
TRACEMSG(("realloc() failed"));
record_buffer = old_buf;
buffer_max_recs = buffer_recs = record_count;
}
}
/* Either buffer at maximum size or attempt to grow it
* failed. */
if (record_count == buffer_max_recs) {
/* Sort */
skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare);
/* Write to temp file */
if (skTempFileWriteBufferStream(
tmpctx, &temp_file_idx,
record_buffer, NODE_SIZE, record_count))
{
skAppPrintSyserror(
"Error writing sorted buffer to temporary file");
free(record_buffer);
appExit(EXIT_FAILURE);
}
/* Reset record buffer to 'empty' */
record_count = 0;
cur_node = record_buffer;
}
}
}
/* Sort (and maybe store) last batch of records */
if (record_count > 0) {
skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare);
if (temp_file_idx >= 0) {
/* Write last batch to temp file */
if (skTempFileWriteBufferStream(
tmpctx, &temp_file_idx,
record_buffer, NODE_SIZE, record_count))
{
skAppPrintSyserror(
"Error writing sorted buffer to temporary file");
free(record_buffer);
appExit(EXIT_FAILURE);
}
}
}
/* Generate the output */
if (record_count == 0 && temp_file_idx == -1) {
/* No records were read at all; write the header to the output
* file */
rv = skStreamWriteSilkHeader(out_rwios);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
}
} else if (temp_file_idx == -1) {
/* No temp files written, just output batch of records */
uint32_t c;
TRACEMSG((("Writing %" PRIu32 " records to '%s'"),
record_count, skStreamGetPathname(out_rwios)));
/* get first two records from the sorted buffer */
cur_node = record_buffer;
next_node = record_buffer + NODE_SIZE;
for (c = 1; c < record_count; ++c, next_node += NODE_SIZE) {
if (0 != rwrecCompare(cur_node, next_node)) {
/* records differ. print earlier record */
rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
if (SKSTREAM_ERROR_IS_FATAL(rv)) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
}
cur_node = next_node;
}
/* else records are duplicates: ignore latter record */
}
/* print remaining record */
rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
if (SKSTREAM_ERROR_IS_FATAL(rv)) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
}
} else {
/* no longer have a need for the record buffer */
free(record_buffer);
record_buffer = NULL;
/* now merge all the temp files */
mergeFiles(temp_file_idx);
}
if (record_buffer) {
free(record_buffer);
}
}
/*
* status = cutFile(stream);
*
* Read SiLK flow records from the file at 'stream' and maybe print
* them according the values in 'skip_recs' and 'num_recs'.
*
* Return -1 on error. Return 1 if all requested records have been
* printed and processing should stop. Return 0 if processing
* should continue.
*/
static int
cutFile(
skstream_t *stream)
{
static int copy_input_only = 0;
rwRec rwrec;
int rv = SKSTREAM_OK;
size_t num_skipped;
int ret_val = 0;
/* handle case where all requested records have been printed, but
* we need to write all records to the --copy-input stream. */
if (copy_input_only) {
while ((rv = skStreamSkipRecords(stream, CUT_SKIP_COUNT, NULL))
== SKSTREAM_OK)
; /* empty */
if (SKSTREAM_ERR_EOF != rv) {
ret_val = -1;
}
goto END;
}
/* skip any leading records */
if (skip_recs) {
rv = skStreamSkipRecords(stream, skip_recs, &num_skipped);
switch (rv) {
case SKSTREAM_OK:
skip_recs -= num_skipped;
break;
case SKSTREAM_ERR_EOF:
skip_recs -= num_skipped;
goto END;
default:
ret_val = -1;
goto END;
}
}
if (0 == num_recs) {
/* print all records */
while ((rv = skStreamReadRecord(stream, &rwrec)) == SKSTREAM_OK) {
rwAsciiPrintRec(ascii_str, &rwrec);
}
if (SKSTREAM_ERR_EOF != rv) {
ret_val = -1;
}
} else {
while (num_recs
&& ((rv = skStreamReadRecord(stream, &rwrec)) == SKSTREAM_OK))
{
rwAsciiPrintRec(ascii_str, &rwrec);
--num_recs;
}
switch (rv) {
case SKSTREAM_OK:
case SKSTREAM_ERR_EOF:
break;
default:
ret_val = -1;
goto END;
}
if (0 == num_recs) {
if (0 == skOptionsCtxCopyStreamIsActive(optctx)) {
/* we're done */
ret_val = 1;
} else {
/* send all remaining records to copy-input */
copy_input_only = 1;
while ((rv = skStreamSkipRecords(stream, CUT_SKIP_COUNT, NULL))
== SKSTREAM_OK)
; /* empty */
if (SKSTREAM_ERR_EOF != rv) {
ret_val = -1;
}
}
}
}
END:
if (-1 == ret_val) {
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
}
return ret_val;
}
/*
* Process a single input stream (file) of SiLK Flow records: Copy
* the header entries from the input stream to the output stream,
* read the file, fill a Key and Counter for each flow record, and
* add the Key and Counter to the AggBag.
*/
static int
processFile(
skstream_t *stream)
{
sk_aggbag_field_t k_it;
sk_aggbag_field_t c_it;
sk_aggbag_aggregate_t key;
sk_aggbag_aggregate_t counter;
skipaddr_t ip;
rwRec rwrec;
ssize_t rv;
ssize_t err;
/* copy invocation and notes (annotations) from the SiLK Flow
* files to the output stream; these headers will not be written
* to the output if --invocation-strip or --notes-strip was
* specified. */
rv = skHeaderCopyEntries(skStreamGetSilkHeader(output),
skStreamGetSilkHeader(stream),
SK_HENTRY_INVOCATION_ID);
if (rv) {
skStreamPrintLastErr(output, rv, &skAppPrintErr);
}
rv = skHeaderCopyEntries(skStreamGetSilkHeader(output),
skStreamGetSilkHeader(stream),
SK_HENTRY_ANNOTATION_ID);
if (rv) {
skStreamPrintLastErr(output, rv, &skAppPrintErr);
}
err = SKAGGBAG_OK;
while (SKSTREAM_OK == (rv = skStreamReadRecord(stream, &rwrec))) {
skAggBagInitializeKey(ab, &key, &k_it);
do {
switch (skAggBagFieldIterGetType(&k_it)) {
case SKAGGBAG_FIELD_SIPv6:
case SKAGGBAG_FIELD_SIPv4:
rwRecMemGetSIP(&rwrec, &ip);
skAggBagAggregateSetIPAddress(&key, &k_it, &ip);
break;
case SKAGGBAG_FIELD_DIPv6:
case SKAGGBAG_FIELD_DIPv4:
rwRecMemGetDIP(&rwrec, &ip);
skAggBagAggregateSetIPAddress(&key, &k_it, &ip);
break;
case SKAGGBAG_FIELD_NHIPv6:
case SKAGGBAG_FIELD_NHIPv4:
rwRecMemGetNhIP(&rwrec, &ip);
skAggBagAggregateSetIPAddress(&key, &k_it, &ip);
break;
case SKAGGBAG_FIELD_SPORT:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetSPort(&rwrec));
break;
case SKAGGBAG_FIELD_DPORT:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetDPort(&rwrec));
break;
case SKAGGBAG_FIELD_ICMP_TYPE:
skAggBagAggregateSetUnsigned(
&key, &k_it,
(rwRecIsICMP(&rwrec) ? rwRecGetIcmpType(&rwrec) : 0));
break;
case SKAGGBAG_FIELD_ICMP_CODE:
skAggBagAggregateSetUnsigned(
&key, &k_it,
(rwRecIsICMP(&rwrec) ? rwRecGetIcmpCode(&rwrec) : 0));
break;
case SKAGGBAG_FIELD_PROTO:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetProto(&rwrec));
break;
case SKAGGBAG_FIELD_PACKETS:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetPkts(&rwrec));
break;
case SKAGGBAG_FIELD_BYTES:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetBytes(&rwrec));
break;
case SKAGGBAG_FIELD_FLAGS:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetFlags(&rwrec));
break;
case SKAGGBAG_FIELD_SID:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetSensor(&rwrec));
break;
case SKAGGBAG_FIELD_INPUT:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetInput(&rwrec));
break;
case SKAGGBAG_FIELD_OUTPUT:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetOutput(&rwrec));
break;
case SKAGGBAG_FIELD_INIT_FLAGS:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetInitFlags(&rwrec));
break;
case SKAGGBAG_FIELD_REST_FLAGS:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetRestFlags(&rwrec));
break;
case SKAGGBAG_FIELD_TCP_STATE:
skAggBagAggregateSetUnsigned(
&key, &k_it,
(rwRecGetTcpState(&rwrec) & SK_TCPSTATE_ATTRIBUTE_MASK));
break;
case SKAGGBAG_FIELD_APPLICATION:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetApplication(&rwrec));
break;
case SKAGGBAG_FIELD_FTYPE_CLASS:
skAggBagAggregateSetUnsigned(
&key, &k_it,
sksiteFlowtypeGetClassID(rwRecGetFlowType(&rwrec)));
break;
case SKAGGBAG_FIELD_FTYPE_TYPE:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetFlowType(&rwrec));
break;
case SKAGGBAG_FIELD_STARTTIME:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetStartSeconds(&rwrec));
break;
case SKAGGBAG_FIELD_ELAPSED:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetElapsedSeconds(&rwrec));
break;
case SKAGGBAG_FIELD_ENDTIME:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetEndSeconds(&rwrec));
break;
default:
break;
}
} while (skAggBagFieldIterNext(&k_it) == SK_ITERATOR_OK);
skAggBagInitializeCounter(ab, &counter, &c_it);
do {
switch (skAggBagFieldIterGetType(&c_it)) {
case SKAGGBAG_FIELD_RECORDS:
skAggBagAggregateSetUnsigned(&counter, &c_it, 1);
break;
case SKAGGBAG_FIELD_SUM_BYTES:
skAggBagAggregateSetUnsigned(
&counter, &c_it, rwRecGetBytes(&rwrec));
break;
case SKAGGBAG_FIELD_SUM_PACKETS:
skAggBagAggregateSetUnsigned(
&counter, &c_it, rwRecGetPkts(&rwrec));
break;
case SKAGGBAG_FIELD_SUM_ELAPSED:
skAggBagAggregateSetUnsigned(
&counter, &c_it, rwRecGetElapsedSeconds(&rwrec));
break;
default:
break;
}
} while (skAggBagFieldIterNext(&c_it) == SK_ITERATOR_OK);
err = skAggBagKeyCounterAdd(ab, &key, &counter, NULL);
if (err) {
skAppPrintErr("Unable to add to key: %s", skAggBagStrerror(err));
break;
}
}
if (rv != SKSTREAM_ERR_EOF) {
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
return -1;
}
return 0;
}
static int
compareFiles(
char **file)
{
skstream_t *stream[2] = {NULL, NULL};
rwRec rec[2];
int i;
int rv;
int status = 2;
uint64_t rec_count = 0;
int eof = -1;
memset(stream, 0, sizeof(stream));
memset(rec, 0, sizeof(rec));
for (i = 0; i < 2; ++i) {
if ((rv = skStreamCreate(&stream[i], SK_IO_READ, SK_CONTENT_SILK_FLOW))
|| (rv = skStreamBind(stream[i], file[i]))
|| (rv = skStreamOpen(stream[i]))
|| (rv = skStreamReadSilkHeader(stream[i], NULL)))
{
/* Give up if we can't read the beginning of the silk header */
if (rv != SKSTREAM_OK) {
if (!quiet) {
skStreamPrintLastErr(stream[i], rv, &skAppPrintErr);
}
goto END;
}
}
}
while ((rv = skStreamReadRecord(stream[0], &rec[0])) == SKSTREAM_OK) {
rv = skStreamReadRecord(stream[1], &rec[1]);
if (rv != SKSTREAM_OK) {
if (rv == SKSTREAM_ERR_EOF) {
/* file 0 longer than file 1 */
status = 1;
eof = 1;
} else {
if (!quiet) {
skStreamPrintLastErr(stream[1], rv, &skAppPrintErr);
}
status = -1;
}
goto END;
}
++rec_count;
if (0 != memcmp(&rec[0], &rec[1], sizeof(rwRec))) {
status = 1;
goto END;
}
}
if (rv != SKSTREAM_ERR_EOF) {
if (!quiet) {
skStreamPrintLastErr(stream[0], rv, &skAppPrintErr);
}
} else {
rv = skStreamReadRecord(stream[1], &rec[1]);
switch (rv) {
case SKSTREAM_OK:
/* file 1 longer than file 0 */
status = 1;
eof = 0;
break;
case SKSTREAM_ERR_EOF:
/* files identical */
status = 0;
break;
default:
if (!quiet) {
skStreamPrintLastErr(stream[1], rv, &skAppPrintErr);
}
break;
}
}
END:
for (i = 0; i < 2; ++i) {
skStreamDestroy(&stream[i]);
}
if (1 == status && !quiet) {
if (eof != -1) {
printf("%s %s differ: EOF %s\n",
file[0], file[1], file[eof]);
} else {
printf(("%s %s differ: record %" PRIu64 "\n"),
file[0], file[1], rec_count);
#ifdef RWCOMPARE_VERBOSE
printRecords(rec);
#endif
}
}
return status;
}
