int main(int argc, char **argv)
{
skstream_t *in_stream;
int in_rv = SKSTREAM_OK;
int rv = SKSTREAM_OK;
appSetup(argc, argv); /* never returns on error */
#error "Loop over files on command line or read from stdin."
#error "Process each file, preferably in a separate function."
/* For each input, process each record */
while (NULL != (in_stream = appNextInput(argc, argv))) {
while ((in_rv = skStreamReadRecord(in_stream, &rwrec))==SKSTREAM_OK) {
/* process record */
rv = skStreamWriteRecord(out_stream, &rwrec);
if (SKSTREAM_OK != rv) {
skStreamPrintLastErr(out_stream, rv, &skAppPrintErr);
skStreamDestroy(&in_stream);
goto END;
}
}
if (SKSTREAM_ERR_EOF != in_rv) {
skStreamPrintLastErr(in_stream, in_rv, &skAppPrintErr);
}
skStreamDestroy(&in_stream);
}
rv = skStreamClose(out_stream);
if (SKSTREAM_OK != rv) {
skStreamPrintLastErr(out_stream, rv, &skAppPrintErr);
}
END:
return ((SKSTREAM_OK == rv) ? EXIT_SUCCESS : EXIT_FAILURE);
}
/*
* stream = appNextInput(argc, argv);
*
* Open and return the next input file from the command line or the
* standard input if no files were given on the command line.
*/
static skstream_t *
appNextInput(
int argc,
char **argv)
{
static int initialized = 0;
skstream_t *stream = NULL;
const char *fname = NULL;
int rv;
if (arg_index < argc) {
/* get current file and prepare to get next */
fname = argv[arg_index];
++arg_index;
} else {
if (initialized) {
/* no more input */
return NULL;
}
/* input is from stdin */
fname = "stdin";
}
initialized = 1;
/* create stream and open file */
rv = skStreamOpenSilkFlow(&stream, fname, SK_IO_READ);
if (rv) {
skStreamPrintLastErr(stream, rv, NULL);
skStreamDestroy(&stream);
}
return stream;
}
/*
* appTeardown()
*
* Teardown all modules, close all files, and tidy up all
* application state.
*
* This function is idempotent.
*/
static void
appTeardown(
void)
{
static int teardownFlag = 0;
int rv;
if (teardownFlag) {
return;
}
teardownFlag = 1;
/* close SiLK flow output file */
if (silk_output) {
rv = skStreamClose(silk_output);
if (rv && rv != SKSTREAM_ERR_NOT_OPEN) {
skStreamPrintLastErr(silk_output, rv, &skAppPrintErr);
}
skStreamDestroy(&silk_output);
}
skpcTeardown();
/* set level to "warning" to avoid the "Stopped logging" message */
sklogSetLevel("warning");
sklogTeardown();
skOptionsNotesTeardown();
skOptionsCtxDestroy(&optctx);
skAppUnregister();
}
/*
* status = appOptionsHandler(cData, opt_index, opt_arg);
*
* Called by skOptionsParse(), this handles a user-specified switch
* that the application has registered, typically by setting global
* variables. Returns 1 if the switch processing failed or 0 if it
* succeeded. Returning a non-zero from from the handler causes
* skOptionsParse() to return a negative value.
*
* The clientData in 'cData' is typically ignored; 'opt_index' is
* the index number that was specified as the last value for each
* struct option in appOptions[]; 'opt_arg' is the user's argument
* to the switch for options that have a REQUIRED_ARG or an
* OPTIONAL_ARG.
*/
static int
appOptionsHandler(
clientData UNUSED(cData),
int opt_index,
char *opt_arg)
{
int rv;
switch ((appOptionsEnum)opt_index) {
#if 0
case OPT_HELP_FIELDS:
helpFields(USAGE_FH);
exit(EXIT_SUCCESS);
#endif /* #if 0 */
case OPT_KEYS:
if (keys_arg) {
skAppPrintErr("Invalid %s: Switch used multiple times",
appOptions[opt_index].name);
return 1;
}
keys_arg = opt_arg;
break;
case OPT_COUNTERS:
if (counters_arg) {
skAppPrintErr("Invalid %s: Switch used multiple times",
appOptions[opt_index].name);
return 1;
}
counters_arg = opt_arg;
break;
case OPT_OUTPUT_PATH:
if (output) {
skAppPrintErr("Invalid %s: Switch used multiple times",
appOptions[opt_index].name);
return 1;
}
if ((rv = skStreamCreate(&output, SK_IO_WRITE, SK_CONTENT_SILK))
|| (rv = skStreamBind(output, opt_arg)))
{
skStreamPrintLastErr(output, rv, &skAppPrintErr);
skStreamDestroy(&output);
return 1;
}
break;
}
return 0; /* OK */
}
int main(int argc, char **argv)
{
skstream_t *inputFile;
skPrefixMap_t *prefixMap;
skPrefixMapErr_t map_error = SKPREFIXMAP_OK;
char buf[DICTIONARY_ENTRY_BUFLEN];
int rv;
appSetup(argc, argv); /* never returns on error */
/* Okay. Now we should open the prefixmap file, read it in, and */
/* then look up our address! */
if ((rv = skStreamCreate(&inputFile, SK_IO_READ, SK_CONTENT_SILK))
|| (rv = skStreamBind(inputFile, prefixmap_test_opt.map_file))
|| (rv = skStreamOpen(inputFile)))
{
skStreamPrintLastErr(inputFile, rv, &skAppPrintErr);
skStreamDestroy(&inputFile);
exit(EXIT_FAILURE);
}
map_error = skPrefixMapRead(&prefixMap, inputFile);
skStreamDestroy(&inputFile);
if ( SKPREFIXMAP_OK != map_error ) {
skAppPrintErr("Failed to read map file: %s",
skPrefixMapStrerror(map_error));
exit(EXIT_FAILURE);
}
if ( prefixmap_test_opt.string ) {
int v = skPrefixMapFindString(prefixMap, &prefixmap_test_opt.address,
buf, sizeof(buf));
if ( v < 0 ) {
strncpy(buf, "(null)", sizeof(buf));
}
printf("%s\n", buf);
} else {
printf("%d\n",
skPrefixMapFindValue(prefixMap, &prefixmap_test_opt.address));
}
skPrefixMapDelete(prefixMap);
/* done */
appTeardown();
return 0;
}
int main(int argc, char **argv)
{
skstream_t *stream = NULL;
ssize_t rv;
/* Global setup */
appSetup(argc, argv);
/* process input */
while ((rv = skOptionsCtxNextSilkFile(optctx, &stream, &skAppPrintErr))
== 0)
{
skStreamSetIPv6Policy(stream, ipv6_policy);
if (0 != processFile(stream)) {
skAppPrintErr("Error processing input from %s",
skStreamGetPathname(stream));
skStreamDestroy(&stream);
return EXIT_FAILURE;
}
skStreamDestroy(&stream);
}
if (rv < 0) {
exit(EXIT_FAILURE);
}
rv = skAggBagWrite(ab, output);
if (rv) {
if (SKAGGBAG_E_WRITE == rv) {
skStreamPrintLastErr(output, skStreamGetLastReturnValue(output),
&skAppPrintErr);
} else {
skAppPrintErr("Error writing Aggregate Bag to '%s': %s",
skStreamGetPathname(output), skAggBagStrerror(rv));
}
exit(EXIT_FAILURE);
}
skAggBagDestroy(&ab);
/* Done, do cleanup */
appTeardown();
return 0;
}
/*
* status = tailFile(stream);
*
* Read SiLK flow records from the file at 'stream' and store the
* most recent 'tail_recs' number of records in the 'tail_buf'
* buffer.
*
* Return -1 on read error, or 0 otherwise.
*/
static int
tailFile(
skstream_t *stream)
{
int rv = SKSTREAM_OK;
while ((rv = skStreamReadRecord(stream, tail_buf_cur)) == SKSTREAM_OK) {
++tail_buf_cur;
if (tail_buf_cur == &tail_buf[tail_recs]) {
tail_buf_cur = tail_buf;
tail_buf_full = 1;
}
}
if (SKSTREAM_ERR_EOF != rv) {
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
return -1;
}
return 0;
}
int main(int argc, char **argv)
{
int rv;
appSetup(argc, argv); /* never returns on error */
sortRandom();
/* close the file */
if ((rv = skStreamClose(out_rwios))
|| (rv = skStreamDestroy(&out_rwios)))
{
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
appExit(EXIT_FAILURE);
}
out_rwios = NULL;
appExit(EXIT_SUCCESS);
return 0; /* NOTREACHED */
}
char *
get_ipa_config(
void)
{
skstream_t *conf_stream = NULL;
char filename[PATH_MAX];
char line[IPA_CONFIG_LINE_LENGTH];
char *ipa_url = NULL;
int rv;
/* Read in the data file */
if (NULL == skFindFile(IPA_CONFIG_FILE, filename, sizeof(filename), 1)) {
skAppPrintErr("Could not locate config file '%s'.",
IPA_CONFIG_FILE);
return NULL;
}
/* open input */
if ((rv = skStreamCreate(&conf_stream, SK_IO_READ, SK_CONTENT_TEXT))
|| (rv = skStreamBind(conf_stream, filename))
|| (rv = skStreamSetCommentStart(conf_stream, "#"))
|| (rv = skStreamOpen(conf_stream)))
{
skStreamPrintLastErr(conf_stream, rv, &skAppPrintErr);
skStreamDestroy(&conf_stream);
exit(EXIT_FAILURE);
}
while (skStreamGetLine(conf_stream, line, sizeof(line), NULL)
== SKSTREAM_OK)
{
/* FIXME: smarter config file reading, please */
if (strlen(line) > 0) {
ipa_url = strdup(line);
break;
}
}
skStreamDestroy(&conf_stream);
/* Should be free()d by the caller */
return ipa_url;
}
/*
* count = pdu2silk(filename);
*
* Read PDUs from 'filename' and write records to the global
* 'silk_output' file. Return number of records processed, or -1
* on error.
*/
static int64_t
pdu2silk(
const char *filename)
{
static unsigned int file_count = 0;
char probe_name[128];
skPDUSource_t *pdu_src;
skFlowSourceParams_t params;
int64_t count;
rwRec rwrec;
int rv;
++file_count;
snprintf(probe_name, sizeof(probe_name), "input%04u", file_count);
params.path_name = filename;
skpcProbeSetName(probe, probe_name);
pdu_src = skPDUSourceCreate(probe, ¶ms);
if (pdu_src == NULL) {
return -1;
}
count = 0;
while (-1 != skPDUSourceGetGeneric(pdu_src, &rwrec)) {
++count;
rv = skStreamWriteRecord(silk_output, &rwrec);
if (rv) {
skStreamPrintLastErr(silk_output, rv, &skAppPrintErr);
if (SKSTREAM_ERROR_IS_FATAL(rv)) {
exit(EXIT_FAILURE);
}
}
}
skPDUSourceLogStatsAndClear(pdu_src);
skPDUSourceDestroy(pdu_src);
return count;
}
int main(int argc, char **argv)
{
int rv;
appSetup(argc, argv);
/* read input file */
if (ip_ranges) {
if (buildIPSetRanges(in_stream)) {
return 1;
}
} else {
if (buildIPSetWildcards(in_stream)) {
return 1;
}
}
skIPSetClean(ipset);
/* write output to stream */
rv = skIPSetWrite(ipset, out_stream);
if (rv) {
if (SKIPSET_ERR_FILEIO == rv) {
skStreamPrintLastErr(out_stream,
skStreamGetLastReturnValue(out_stream),
&skAppPrintErr);
} else {
skAppPrintErr("Unable to write IPset to '%s': %s",
skStreamGetPathname(out_stream),skIPSetStrerror(rv));
}
return 1;
}
skStreamDestroy(&in_stream);
skStreamDestroy(&out_stream);
/* done */
return 0;
}
/*
* appSetup(argc, argv);
*
* Perform all the setup for this application include setting up
* required modules, parsing options, etc. This function should be
* passed the same arguments that were passed into main().
*
* Returns to the caller if all setup succeeds. If anything fails,
* this function will cause the application to exit with a FAILURE
* exit status.
*/
static void
appSetup(
int argc,
char **argv)
{
SILK_FEATURES_DEFINE_STRUCT(features);
unsigned int optctx_flags;
sk_file_header_t *silk_hdr;
int logmask;
int rv;
/* verify same number of options and help strings */
assert((sizeof(appHelp)/sizeof(char *)) ==
(sizeof(appOptions)/sizeof(struct option)));
/* register the application */
skAppRegister(argv[0]);
skAppVerifyFeatures(&features, NULL);
skOptionsSetUsageCallback(&appUsageLong);
optctx_flags = (SK_OPTIONS_CTX_INPUT_BINARY | SK_OPTIONS_CTX_XARGS);
/* register the options */
if (skOptionsCtxCreate(&optctx, optctx_flags)
|| skOptionsCtxOptionsRegister(optctx)
|| skOptionsRegister(appOptions, &appOptionsHandler, NULL)
|| skOptionsNotesRegister(NULL)
|| skCompMethodOptionsRegister(&comp_method))
{
skAppPrintErr("Unable to register options");
exit(EXIT_FAILURE);
}
/* enable the logger */
sklogSetup(0);
sklogSetDestination("stderr");
sklogSetStampFunction(&logprefix);
/* register the teardown handler */
if (atexit(appTeardown) < 0) {
skAppPrintErr("Unable to register appTeardown() with atexit()");
appTeardown();
exit(EXIT_FAILURE);
}
/* parse the options */
rv = skOptionsCtxOptionsParse(optctx, argc, argv);
if (rv < 0) {
skAppUsage(); /* never returns */
}
if ('\0' == log_destination[0]) {
strncpy(log_destination, LOG_DESTINATION_DEFAULT,
sizeof(log_destination));
} else {
sklogSetLevel("debug");
}
sklogSetDestination(log_destination);
/* default output is "stdout" */
if (!silk_output) {
if ((rv =skStreamCreate(&silk_output,SK_IO_WRITE,SK_CONTENT_SILK_FLOW))
|| (rv = skStreamBind(silk_output, "-")))
{
skStreamPrintLastErr(silk_output, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
}
/* get the header */
silk_hdr = skStreamGetSilkHeader(silk_output);
/* open the output */
if ((rv = skHeaderSetCompressionMethod(silk_hdr, comp_method))
|| (rv = skOptionsNotesAddToStream(silk_output))
|| (rv = skHeaderAddInvocation(silk_hdr, 1, argc, argv))
|| (rv = skStreamOpen(silk_output))
|| (rv = skStreamWriteSilkHeader(silk_output)))
{
skStreamPrintLastErr(silk_output, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
if (skpcSetup()) {
exit(EXIT_FAILURE);
}
if (skpcProbeCreate(&probe, PROBE_ENUM_NETFLOW_V5)) {
exit(EXIT_FAILURE);
}
skpcProbeSetName(probe, skAppName());
skpcProbeSetFileSource(probe, "/dev/null");
if (parseLogFlags(log_flags)) {
exit(EXIT_FAILURE);
}
if (skpcProbeVerify(probe, 0)) {
exit(EXIT_FAILURE);
}
/* set level to "warning" to avoid the "Started logging" message */
logmask = sklogGetMask();
sklogSetLevel("warning");
sklogOpen();
sklogSetMask(logmask);
return; /* OK */
}
static int
compareFiles(
char **file)
{
skstream_t *stream[2] = {NULL, NULL};
rwRec rec[2];
int i;
int rv;
int status = 2;
uint64_t rec_count = 0;
int eof = -1;
memset(stream, 0, sizeof(stream));
memset(rec, 0, sizeof(rec));
for (i = 0; i < 2; ++i) {
if ((rv = skStreamCreate(&stream[i], SK_IO_READ, SK_CONTENT_SILK_FLOW))
|| (rv = skStreamBind(stream[i], file[i]))
|| (rv = skStreamOpen(stream[i]))
|| (rv = skStreamReadSilkHeader(stream[i], NULL)))
{
/* Give up if we can't read the beginning of the silk header */
if (rv != SKSTREAM_OK) {
if (!quiet) {
skStreamPrintLastErr(stream[i], rv, &skAppPrintErr);
}
goto END;
}
}
}
while ((rv = skStreamReadRecord(stream[0], &rec[0])) == SKSTREAM_OK) {
rv = skStreamReadRecord(stream[1], &rec[1]);
if (rv != SKSTREAM_OK) {
if (rv == SKSTREAM_ERR_EOF) {
/* file 0 longer than file 1 */
status = 1;
eof = 1;
} else {
if (!quiet) {
skStreamPrintLastErr(stream[1], rv, &skAppPrintErr);
}
status = -1;
}
goto END;
}
++rec_count;
if (0 != memcmp(&rec[0], &rec[1], sizeof(rwRec))) {
status = 1;
goto END;
}
}
if (rv != SKSTREAM_ERR_EOF) {
if (!quiet) {
skStreamPrintLastErr(stream[0], rv, &skAppPrintErr);
}
} else {
rv = skStreamReadRecord(stream[1], &rec[1]);
switch (rv) {
case SKSTREAM_OK:
/* file 1 longer than file 0 */
status = 1;
eof = 0;
break;
case SKSTREAM_ERR_EOF:
/* files identical */
status = 0;
break;
default:
if (!quiet) {
skStreamPrintLastErr(stream[1], rv, &skAppPrintErr);
}
break;
}
}
END:
for (i = 0; i < 2; ++i) {
skStreamDestroy(&stream[i]);
}
if (1 == status && !quiet) {
if (eof != -1) {
printf("%s %s differ: EOF %s\n",
file[0], file[1], file[eof]);
} else {
printf(("%s %s differ: record %" PRIu64 "\n"),
file[0], file[1], rec_count);
#ifdef RWCOMPARE_VERBOSE
printRecords(rec);
#endif
}
}
return status;
}
/*
* Helper function that opens the flowcap file at 'path'. The
* handle to the file is put in the location specified by 'stream'.
* The name of the probe is read from the file's header, and the
* probe object is put into 'probe'. Returns 0 on success, or -1
* on the following error conditions:
*
* -- 'path' is NULL.
* -- unable to open file
* -- file is not a valid SiLK file
* -- file does not contain a Probename Header
* -- the probe_name does not map to a valid probe in probeconf
*
* NOTES:
*
* Flowcap V16 files have the probe name in the header.
*
* Flowcap Files V2-V5 have the sensor name and probe name in the
* header. When these are processed in SiLK 1.0, these get mapped
* to the probe name "<sensor>_<probe>".
*
* Flowcap Files V1 have no probe information and are no longer
* supported.
*/
static int
flowcapSourceCreateFromFile(
const char *path,
skstream_t **stream,
skpc_probe_t **probe)
{
sk_file_header_t *hdr;
sk_header_entry_t *sp_hdr;
const char *probe_name;
int rv;
/* Argument checking */
if (path == NULL) {
ERRMSG("NULL path passed to flowcapSourceCreateFromFile.");
return -1;
}
/* Valid file checking */
rv = skStreamOpenSilkFlow(stream, path, SK_IO_READ);
if (rv) {
CRITMSG("Unable to open '%s' for reading.", path);
skStreamPrintLastErr(*stream, rv, &ERRMSG);
goto error;
}
/*
* File should have a Probename header
*
* Flowcap V16 files have the probe name in the header.
*
* Flowcap Files V2-V5 have a separate sensor name and probe name
* in the header. In SiLK 1.0, these get mapped to the single
* probe name "<sensor>_<probe>".
*
* Flowcap Files V1 have no probe information and are not
* supported.
*/
hdr = skStreamGetSilkHeader(*stream);
sp_hdr = skHeaderGetFirstMatch(hdr, SK_HENTRY_PROBENAME_ID);
if (sp_hdr == NULL) {
CRITMSG("No probename header in %s.", path);
goto error;
}
probe_name = skHentryProbenameGetProbeName(sp_hdr);
if (probe_name == NULL || probe_name[0] == '\0') {
CRITMSG("Unable to get probename from flowcap file '%s'.",
path);
goto error;
}
/* Use the probe_name to find the skpc_probe_t object. */
*probe = (skpc_probe_t*)skpcProbeLookupByName(probe_name);
if (*probe == NULL) {
CRITMSG("The sensor configuration file does not define probe '%s'",
probe_name);
goto error;
}
/* Verify that the probe has sensors associated with it */
if (skpcProbeGetSensorCount(*probe) == 0) {
CRITMSG("Probe '%s' is not associated with a sensor", probe_name);
goto error;
}
/* File has been validated. We're done. */
return 0;
error:
skStreamDestroy(stream);
return -1;
}
/*
* status = readerGetRecord(&out_rwrec, &out_probe, flow_processor);
*
* Invoked by input_mode_type->get_record_fn();
*/
static fp_get_record_result_t
readerGetRecord(
rwRec *out_rwrec,
const skpc_probe_t **out_probe,
flow_proc_t *fproc)
{
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
skstream_t *fcfile;
const char *filename;
fp_get_record_result_t retVal = FP_GET_ERROR;
int rv;
pthread_mutex_lock(&mutex);
/* If we don't have a source, get a file from the directory poller
* and start processing it. */
if (fproc->flow_src == NULL) {
switch (readerGetNextValidFile(fproc)) {
case 0:
/* Success */
break;
case -1:
/* Error getting file name (maybe in shutdown?) */
goto END;
case -2: /* Error opening file */
default: /* Unexpected value */
retVal = FP_FATAL_ERROR;
goto END;
}
}
fcfile = (skstream_t*)fproc->flow_src;
/* Assume we can get a record from the probe. */
retVal = FP_RECORD;
*out_probe = fproc->probe;
/* Try to get a record */
rv = skStreamReadRecord(fcfile, out_rwrec);
if (rv) {
/* get failed: either at EOF or got an error. */
if (rv != SKSTREAM_ERR_EOF) {
skStreamPrintLastErr(fcfile, rv, &WARNINGMSG);
}
retVal = FP_FILE_BREAK;
*out_probe = NULL;
/* Print results for the file we just processed. */
filename = skStreamGetPathname(fcfile);
INFOMSG("Processed file %s, %" PRIu64 " records.",
filename, skStreamGetRecordCount(fcfile));
skStreamClose(fcfile);
/* Either archive the file or remove it */
archiveDirectoryInsertOrRemove(filename, NULL);
/* All done with the flow source */
skStreamDestroy(&fcfile);
fproc->flow_src = NULL;
fproc->probe = NULL;
}
END:
pthread_mutex_unlock(&mutex);
return retVal;
}
/*
* skplugin_err_t pmapfile_handler(const char *opt_arg, void *cbdata)
*
* Handler for --pmap-file option. Actually registers the filter and
* fields.
*/
static skplugin_err_t
pmapfile_handler(
const char *opt_arg,
void UNUSED(*cbdata))
{
/* Whether we have seen any unnamed pmaps */
static int have_unnamed_pmap = 0;
skPrefixMapErr_t map_error = SKPREFIXMAP_OK;
skstream_t *stream = NULL;
skPrefixMap_t *prefix_map = NULL;
pmap_data_t *pmap_data = NULL;
int ok;
const char *filename;
const char *sep;
const char *mapname;
char *prefixed_name = NULL;
char *short_prefixed_name;
size_t namelen = 0;
size_t i;
int rv = SKPLUGIN_ERR;
skplugin_callbacks_t regdata;
/* We can only have one pmap whenever we any any pmap without a
* mapname. If we've seen one and enter this function a second
* time, it is an error */
if (have_unnamed_pmap) {
skAppPrintErr(("Invalid %s: You may use only one prefix map"
" when you are\n"
"\tusing a prefix map without specifying a mapname"),
pmap_file_option);
return SKPLUGIN_ERR;
}
/* Parse the argument into a field name and file name */
sep = strchr(opt_arg, ':');
if (NULL == sep) {
/* We do not have a mapname. We'll check for one in the pmap
* once we read it. */
mapname = NULL;
filename = opt_arg;
} else {
/* A mapname was supplied on the command line */
if (sep == opt_arg) {
skAppPrintErr("Invalid %s: Zero length mapnames are not allowed",
pmap_file_option);
return SKPLUGIN_ERR;
}
if (memchr(opt_arg, ',', sep - opt_arg) != NULL) {
skAppPrintErr("Invalid %s: The mapname may not include a comma",
pmap_file_option);
return SKPLUGIN_ERR;
}
mapname = opt_arg;
filename = sep + 1;
namelen = sep - opt_arg;
}
ok = skpinOpenDataInputStream(&stream, SK_CONTENT_SILK, filename);
if (ok == -1) {
/* problem opening file */
skAppPrintErr("Failed to open the prefix map file '%s'",
filename);
return SKPLUGIN_ERR;
}
if (ok == 1) {
/* master needs to process the file, since it may contain the
* map name we use for creating switches */
if ((rv = skStreamCreate(&stream, SK_IO_READ, SK_CONTENT_SILK))
|| (rv = skStreamBind(stream, filename))
|| (rv = skStreamOpen(stream)))
{
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
skStreamDestroy(&stream);
return SKPLUGIN_ERR;
}
/* the master can ignore the file for filtering */
ignore_prefix_map = 1;
}
map_error = skPrefixMapRead(&prefix_map, stream);
if (SKPREFIXMAP_OK != map_error) {
if (SKPREFIXMAP_ERR_IO == map_error) {
skStreamPrintLastErr(stream, skStreamGetLastReturnValue(stream),
&skAppPrintErr);
} else {
skAppPrintErr("Failed to read the prefix map file '%s': %s",
opt_arg, skPrefixMapStrerror(map_error));
}
skStreamDestroy(&stream);
return SKPLUGIN_ERR;
}
skStreamDestroy(&stream);
if (NULL == mapname) {
/* No mapname was supplied on the command line. Check for a
* mapname insided the pmap. */
mapname = skPrefixMapGetMapName(prefix_map);
if (mapname) {
/* The pmap supplied a mapname */
namelen = strlen(mapname);
} else {
/* No mapname. Accept for legacy purposes, unless we have
* read any other pmaps */
have_unnamed_pmap = 1;
if (skVectorGetCount(pmap_vector) != 0) {
skAppPrintErr(("Invalid %s: You may use only one prefix map"
" when you are\n"
"\t using a prefix map without"
" specifying a mapname"),
pmap_file_option);
goto END;
}
}
}
/* Allocate the pmap_data structure */
pmap_data = (pmap_data_t *)calloc(1, sizeof(*pmap_data));
if (pmap_data == NULL) {
ERR_NO_MEM(pmap_data);
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
/* pmap_data now "owns" prefix_map */
pmap_data->pmap = prefix_map;
prefix_map = NULL;
/* Cache the content type */
pmap_data->type = skPrefixMapGetContentType(pmap_data->pmap);
/* Fill the direction structure for each direction */
pmap_data->sdir.dir = DIR_SOURCE;
pmap_data->ddir.dir = DIR_DEST;
pmap_data->adir.dir = DIR_ANY;
pmap_data->sdir.data = pmap_data;
pmap_data->ddir.data = pmap_data;
pmap_data->adir.data = pmap_data;
/* Record the path to the pmap file */
pmap_data->filepath = strdup(filename);
if (NULL == pmap_data->filepath) {
ERR_NO_MEM(pmap_data->filepath);
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
if (mapname == NULL) {
/* Pmap without a mapname. */
/* Add the proper legacy option names to the pmap_data structure */
switch (pmap_data->type) {
case SKPREFIXMAP_CONT_ADDR_V4:
case SKPREFIXMAP_CONT_ADDR_V6:
pmap_data->sdir.filter_option = strdup(pmap_saddress_option);
pmap_data->ddir.filter_option = strdup(pmap_daddress_option);
pmap_data->adir.filter_option = strdup(pmap_aaddress_option);
break;
case SKPREFIXMAP_CONT_PROTO_PORT:
pmap_data->sdir.filter_option = strdup(pmap_sport_proto_option);
pmap_data->ddir.filter_option = strdup(pmap_dport_proto_option);
pmap_data->adir.filter_option = strdup(pmap_aport_proto_option);
break;
}
if ((pmap_data->sdir.filter_option == NULL)
|| (pmap_data->ddir.filter_option == NULL)
|| (pmap_data->adir.filter_option == NULL))
{
ERR_NO_MEM(filter_option);
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
pmap_data->mapname = strdup(pmap_title_val);
pmap_data->sdir.field_name = strdup(pmap_title_sval);
pmap_data->ddir.field_name = strdup(pmap_title_dval);
if ((pmap_data->mapname == NULL)
|| (pmap_data->sdir.field_name == NULL)
|| (pmap_data->ddir.field_name == NULL))
{
ERR_NO_MEM(field_name);
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
} else { /* if (mapname == NULL) */
/* Create the field names*/
pmap_data->mapname = (char*)malloc(namelen + 1);
if (NULL == pmap_data->mapname) {
ERR_NO_MEM(pmap_data->mapname);
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
strncpy(pmap_data->mapname, mapname, namelen);
pmap_data->mapname[namelen] = '\0';
/* Allocate space for the [pmap-]{src-,dst-}<mapname> string */
prefixed_name
= (char*)malloc(namelen + pmap_prefix_len + dir_name_len + 1);
if (NULL == prefixed_name) {
ERR_NO_MEM(prefixed_name);
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
/* Copy in the pmap- prefix */
strncpy(prefixed_name, pmap_prefix, pmap_prefix_len);
/* short name (for fields) starts at the {src-,dst-} */
short_prefixed_name = prefixed_name + pmap_prefix_len;
/* add in the actual field name, and zero terminate it */
strncpy(short_prefixed_name + dir_name_len, mapname, namelen);
short_prefixed_name[namelen + dir_name_len] = '\0';
/* Create the destination-themed names */
strncpy(short_prefixed_name, src_dir_name, dir_name_len);
pmap_data->sdir.filter_option = strdup(prefixed_name);
pmap_data->sdir.field_name = strdup(short_prefixed_name);
if ((pmap_data->sdir.filter_option == NULL)
|| (pmap_data->sdir.field_name == NULL))
{
ERR_NO_MEM(pmap_data->sdir);
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
strncpy(short_prefixed_name, dst_dir_name, dir_name_len);
pmap_data->ddir.filter_option = strdup(prefixed_name);
pmap_data->ddir.field_name = strdup(short_prefixed_name);
if ((pmap_data->ddir.filter_option == NULL)
|| (pmap_data->ddir.field_name == NULL))
{
ERR_NO_MEM(pmap_data->ddir);
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
strncpy(short_prefixed_name, any_dir_name, dir_name_len);
pmap_data->adir.filter_option = strdup(prefixed_name);
if (pmap_data->adir.filter_option == NULL) {
ERR_NO_MEM(pmap_data->adir);
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
/* Free the temporary name buffer */
free(prefixed_name);
prefixed_name = NULL;
} /* if (mapname == NULL) */
/* Verify unique field names */
for (i = 0; i < skVectorGetCount(pmap_vector); i++) {
pmap_data_t *p;
skVectorGetValue(&p, pmap_vector, i);
if ((strcmp(pmap_data->mapname, p->mapname) == 0) ||
(strcmp(pmap_data->sdir.field_name, p->sdir.field_name) == 0) ||
(strcmp(pmap_data->ddir.field_name, p->ddir.field_name) == 0))
{
skAppPrintErr(("Invalid %s: Multiple pmaps use the mapname '%s':\n"
"\t%s\n\t%s"),
pmap_file_option, pmap_data->mapname,
p->filepath, pmap_data->filepath);
rv = SKPLUGIN_ERR;
goto END;
}
}
/* Register fields and filter options */
memset(®data, 0, sizeof(regdata));
regdata.init = pmap_field_init;
regdata.column_width = 0;
regdata.bin_bytes = 4;
regdata.rec_to_text = pmap_text_fn;
regdata.rec_to_bin = pmap_bin_fn;
regdata.bin_to_text = pmap_bin_to_text_fn;
for (i = 0; i < 2; ++i) {
directed_pmap_data_t *dir = ((0 == i)
? &pmap_data->sdir
: &pmap_data->ddir);
skpinRegField(&dir->field, dir->field_name, NULL, ®data, dir);
skpinRegOption2(dir->filter_option,
REQUIRED_ARG,
NULL, &pmap_filter_help,
&pmap_handle_filter_option, dir,
1, SKPLUGIN_FN_FILTER);
}
/* Register the "any" filter separately */
skpinRegOption2(pmap_data->adir.filter_option,
REQUIRED_ARG,
NULL, &pmap_filter_help,
&pmap_handle_filter_option, &pmap_data->adir,
1, SKPLUGIN_FN_FILTER);
if (skVectorAppendValue(pmap_vector, &pmap_data)) {
rv = SKPLUGIN_ERR_FATAL;
goto END;
}
rv = SKPLUGIN_OK;
END:
/* Free the temporary name buffer */
if (prefixed_name) {
free(prefixed_name);
}
if (rv != SKPLUGIN_OK) {
if (prefix_map) {
skPrefixMapDelete(prefix_map);
}
if (pmap_data) {
pmap_data_destroy(pmap_data);
}
}
return (skplugin_err_t)rv;
}
/*
* appSetup(argc, argv);
*
* Perform all the setup for this application include setting up
* required modules, parsing options, etc. This function should be
* passed the same arguments that were passed into main().
*
* Returns to the caller if all setup succeeds. If anything fails,
* this function will cause the application to exit with a FAILURE
* exit status.
*/
static void
appSetup(
int argc,
char **argv)
{
SILK_FEATURES_DEFINE_STRUCT(features);
int arg_index;
int rv;
/* verify same number of options and help strings */
assert((sizeof(appHelp)/sizeof(char *)) ==
(sizeof(appOptions)/sizeof(struct option)));
/* register the application */
skAppRegister(argv[0]);
skAppVerifyFeatures(&features, NULL);
skOptionsSetUsageCallback(&appUsageLong);
/* register the options */
if (skOptionsRegister(appOptions, &appOptionsHandler, NULL))
{
skAppPrintErr("Unable to register options");
exit(EXIT_FAILURE);
}
/* register the teardown hanlder */
if (atexit(appTeardown) < 0) {
skAppPrintErr("Unable to register appTeardown() with atexit()");
appTeardown();
exit(EXIT_FAILURE);
}
/* parse the options */
arg_index = skOptionsParse(argc, argv);
if (arg_index < 0) {
/* options parsing should print error */
skAppUsage(); /* never returns */
}
/* check for extraneous arguments */
if (arg_index != argc) {
skAppPrintErr("Too many arguments or unrecognized switch '%s'",
argv[arg_index]);
skAppUsage(); /* never returns */
}
/* check for one and only one of --input-file and --address */
if (NULL != app_opt.input_file) {
if (NULL != app_opt.address) {
skAppPrintErr("Only one of --%s or --%s may be specified.",
appOptions[OPT_ADDRESS].name,
appOptions[OPT_INPUT_FILE].name);
skAppUsage();
}
} else if (NULL == app_opt.address) {
skAppPrintErr("Either the --%s or --%s option is required.",
appOptions[OPT_ADDRESS].name,
appOptions[OPT_INPUT_FILE].name);
skAppUsage();
}
/* find and load the map file */
if (skCountrySetup(app_opt.map_file, &skAppPrintErr)) {
exit(EXIT_FAILURE);
}
/* use default for print-ips if unset by user */
if (-1 == app_opt.print_ips) {
if (app_opt.input_file) {
app_opt.print_ips = 1;
} else {
app_opt.print_ips = 0;
}
}
/* create the output stream. Only page output when processing
* input file */
if ((rv = skStreamCreate(&out, SK_IO_WRITE, SK_CONTENT_TEXT))
|| (rv = skStreamBind(out, app_opt.output_path)))
{
skStreamPrintLastErr(out, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
if (app_opt.input_file) {
rv = skStreamPageOutput(out, app_opt.pager);
if (rv) {
skStreamPrintLastErr(out, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
}
rv = skStreamOpen(out);
if (rv) {
skStreamPrintLastErr(out, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
return; /* OK */
}
/*
* Process a single input stream (file) of SiLK Flow records: Copy
* the header entries from the input stream to the output stream,
* read the file, fill a Key and Counter for each flow record, and
* add the Key and Counter to the AggBag.
*/
static int
processFile(
skstream_t *stream)
{
sk_aggbag_field_t k_it;
sk_aggbag_field_t c_it;
sk_aggbag_aggregate_t key;
sk_aggbag_aggregate_t counter;
skipaddr_t ip;
rwRec rwrec;
ssize_t rv;
ssize_t err;
/* copy invocation and notes (annotations) from the SiLK Flow
* files to the output stream; these headers will not be written
* to the output if --invocation-strip or --notes-strip was
* specified. */
rv = skHeaderCopyEntries(skStreamGetSilkHeader(output),
skStreamGetSilkHeader(stream),
SK_HENTRY_INVOCATION_ID);
if (rv) {
skStreamPrintLastErr(output, rv, &skAppPrintErr);
}
rv = skHeaderCopyEntries(skStreamGetSilkHeader(output),
skStreamGetSilkHeader(stream),
SK_HENTRY_ANNOTATION_ID);
if (rv) {
skStreamPrintLastErr(output, rv, &skAppPrintErr);
}
err = SKAGGBAG_OK;
while (SKSTREAM_OK == (rv = skStreamReadRecord(stream, &rwrec))) {
skAggBagInitializeKey(ab, &key, &k_it);
do {
switch (skAggBagFieldIterGetType(&k_it)) {
case SKAGGBAG_FIELD_SIPv6:
case SKAGGBAG_FIELD_SIPv4:
rwRecMemGetSIP(&rwrec, &ip);
skAggBagAggregateSetIPAddress(&key, &k_it, &ip);
break;
case SKAGGBAG_FIELD_DIPv6:
case SKAGGBAG_FIELD_DIPv4:
rwRecMemGetDIP(&rwrec, &ip);
skAggBagAggregateSetIPAddress(&key, &k_it, &ip);
break;
case SKAGGBAG_FIELD_NHIPv6:
case SKAGGBAG_FIELD_NHIPv4:
rwRecMemGetNhIP(&rwrec, &ip);
skAggBagAggregateSetIPAddress(&key, &k_it, &ip);
break;
case SKAGGBAG_FIELD_SPORT:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetSPort(&rwrec));
break;
case SKAGGBAG_FIELD_DPORT:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetDPort(&rwrec));
break;
case SKAGGBAG_FIELD_ICMP_TYPE:
skAggBagAggregateSetUnsigned(
&key, &k_it,
(rwRecIsICMP(&rwrec) ? rwRecGetIcmpType(&rwrec) : 0));
break;
case SKAGGBAG_FIELD_ICMP_CODE:
skAggBagAggregateSetUnsigned(
&key, &k_it,
(rwRecIsICMP(&rwrec) ? rwRecGetIcmpCode(&rwrec) : 0));
break;
case SKAGGBAG_FIELD_PROTO:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetProto(&rwrec));
break;
case SKAGGBAG_FIELD_PACKETS:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetPkts(&rwrec));
break;
case SKAGGBAG_FIELD_BYTES:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetBytes(&rwrec));
break;
case SKAGGBAG_FIELD_FLAGS:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetFlags(&rwrec));
break;
case SKAGGBAG_FIELD_SID:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetSensor(&rwrec));
break;
case SKAGGBAG_FIELD_INPUT:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetInput(&rwrec));
break;
case SKAGGBAG_FIELD_OUTPUT:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetOutput(&rwrec));
break;
case SKAGGBAG_FIELD_INIT_FLAGS:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetInitFlags(&rwrec));
break;
case SKAGGBAG_FIELD_REST_FLAGS:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetRestFlags(&rwrec));
break;
case SKAGGBAG_FIELD_TCP_STATE:
skAggBagAggregateSetUnsigned(
&key, &k_it,
(rwRecGetTcpState(&rwrec) & SK_TCPSTATE_ATTRIBUTE_MASK));
break;
case SKAGGBAG_FIELD_APPLICATION:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetApplication(&rwrec));
break;
case SKAGGBAG_FIELD_FTYPE_CLASS:
skAggBagAggregateSetUnsigned(
&key, &k_it,
sksiteFlowtypeGetClassID(rwRecGetFlowType(&rwrec)));
break;
case SKAGGBAG_FIELD_FTYPE_TYPE:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetFlowType(&rwrec));
break;
case SKAGGBAG_FIELD_STARTTIME:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetStartSeconds(&rwrec));
break;
case SKAGGBAG_FIELD_ELAPSED:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetElapsedSeconds(&rwrec));
break;
case SKAGGBAG_FIELD_ENDTIME:
skAggBagAggregateSetUnsigned(
&key, &k_it, rwRecGetEndSeconds(&rwrec));
break;
default:
break;
}
} while (skAggBagFieldIterNext(&k_it) == SK_ITERATOR_OK);
skAggBagInitializeCounter(ab, &counter, &c_it);
do {
switch (skAggBagFieldIterGetType(&c_it)) {
case SKAGGBAG_FIELD_RECORDS:
skAggBagAggregateSetUnsigned(&counter, &c_it, 1);
break;
case SKAGGBAG_FIELD_SUM_BYTES:
skAggBagAggregateSetUnsigned(
&counter, &c_it, rwRecGetBytes(&rwrec));
break;
case SKAGGBAG_FIELD_SUM_PACKETS:
skAggBagAggregateSetUnsigned(
&counter, &c_it, rwRecGetPkts(&rwrec));
break;
case SKAGGBAG_FIELD_SUM_ELAPSED:
skAggBagAggregateSetUnsigned(
&counter, &c_it, rwRecGetElapsedSeconds(&rwrec));
break;
default:
break;
}
} while (skAggBagFieldIterNext(&c_it) == SK_ITERATOR_OK);
err = skAggBagKeyCounterAdd(ab, &key, &counter, NULL);
if (err) {
skAppPrintErr("Unable to add to key: %s", skAggBagStrerror(err));
break;
}
}
if (rv != SKSTREAM_ERR_EOF) {
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
return -1;
}
return 0;
}
/*
* status = processInputFile(filein);
*
* For every line in 'filein', look up the address in Country Code
* map and print out the corresponding country code. There should
* be as many lines of output as there are of input.
*/
static int
processInputFile(
const char *f_name)
{
char final_delim[] = {'\0', '\0'};
char line[2048];
skstream_t *stream = NULL;
skIPWildcardIterator_t iter;
skIPWildcard_t ipwild;
skipaddr_t ip;
int retval = 1;
int rv;
int lc = 0;
char cc[32];
char ipbuf[SK_NUM2DOT_STRLEN];
if (!app_opt.no_final_delimiter) {
final_delim[0] = app_opt.column_separator;
}
/* open input */
if ((rv = skStreamCreate(&stream, SK_IO_READ, SK_CONTENT_TEXT))
|| (rv = skStreamBind(stream, f_name))
|| (rv = skStreamSetCommentStart(stream, "#"))
|| (rv = skStreamOpen(stream)))
{
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
goto END;
}
/* read until end of file */
while ((rv = skStreamGetLine(stream, line, sizeof(line), &lc))
!= SKSTREAM_ERR_EOF)
{
switch (rv) {
case SKSTREAM_OK:
/* good, we got our line */
break;
case SKSTREAM_ERR_LONG_LINE:
/* bad: line was longer than sizeof(line) */
skAppPrintErr("Input line %d too long. ignored",
lc);
continue;
default:
/* unexpected error */
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
goto END;
}
/* parse the line: fill in octet_bitmap */
rv = skStringParseIPWildcard(&ipwild, line);
if (rv && rv != SKUTILS_ERR_EMPTY) {
/* error */
skAppPrintErr("Error on line %d: %s\n",
lc, skStringParseStrerror(rv));
goto END;
}
#if SK_ENABLE_IPV6
if (skIPWildcardIsV6(&ipwild)) {
continue;
}
#endif /* SK_ENABLE_IPV6 */
skIPWildcardIteratorBind(&iter, &ipwild);
while (skIPWildcardIteratorNext(&iter, &ip) == SK_ITERATOR_OK) {
skCountryLookupName(&ip, cc, sizeof(cc));
if (!app_opt.print_ips) {
skStreamPrint(out, "%s\n", cc);
} else {
skipaddrString(ipbuf, &ip, ip_flags);
if (app_opt.no_columns) {
skStreamPrint(out, "%s%c%s%s\n",
ipbuf, app_opt.column_separator,
cc, final_delim);
} else {
skStreamPrint(out, "%15s%c%2s%s\n",
ipbuf, app_opt.column_separator,
cc, final_delim);
}
}
}
}
retval = 0;
END:
skStreamDestroy(&stream);
return retval;
}
/*
* sortRandom();
*
* Don't make any assumptions about the input. Store the input
* records in a large buffer, and sort those in-core records once
* all records are processed or the buffer is full. If the buffer
* fills up, store the sorted records into temporary files. Once
* all records are read, use mergeFiles() above to merge-sort the
* temporary files.
*
* Exits the application if an error occurs.
*/
static void
sortRandom(
void)
{
int temp_file_idx = -1;
skstream_t *input_rwios = NULL; /* input stream */
uint8_t *record_buffer = NULL; /* Region of memory for records */
uint8_t *cur_node = NULL; /* Ptr into record_buffer */
uint8_t *next_node = NULL; /* Ptr into record_buffer */
uint32_t buffer_max_recs; /* max buffer size (in number of recs) */
uint32_t buffer_recs; /* current buffer size (# records) */
uint32_t buffer_chunk_recs; /* how to grow from current to max buf */
uint32_t num_chunks; /* how quickly to grow buffer */
uint32_t record_count = 0; /* Number of records read */
int rv;
/* Determine the maximum number of records that will fit into the
* buffer if it grows the maximum size */
buffer_max_recs = buffer_size / NODE_SIZE;
TRACEMSG((("buffer_size = %" PRIu64
"\nnode_size = %" PRIu32
"\nbuffer_max_recs = %" PRIu32),
buffer_size, NODE_SIZE, buffer_max_recs));
/* We will grow to the maximum size in chunks */
num_chunks = NUM_CHUNKS;
if (num_chunks <= 0) {
num_chunks = 1;
}
/* Attempt to allocate the initial chunk. If we fail, increment
* the number of chunks---which will decrease the amount we
* attempt to allocate at once---and try again. */
for (;;) {
buffer_chunk_recs = buffer_max_recs / num_chunks;
TRACEMSG((("num_chunks = %" PRIu32
"\nbuffer_chunk_recs = %" PRIu32),
num_chunks, buffer_chunk_recs));
record_buffer = (uint8_t*)malloc(NODE_SIZE * buffer_chunk_recs);
if (record_buffer) {
/* malloc was successful */
break;
} else if (buffer_chunk_recs < MIN_IN_CORE_RECORDS) {
/* give up at this point */
skAppPrintErr("Error allocating space for %d records",
MIN_IN_CORE_RECORDS);
appExit(EXIT_FAILURE);
} else {
/* reduce the amount we allocate at once by increasing the
* number of chunks and try again */
TRACEMSG(("malloc() failed"));
++num_chunks;
}
}
buffer_recs = buffer_chunk_recs;
TRACEMSG((("buffer_recs = %" PRIu32), buffer_recs));
/* open first file */
rv = appNextInput(&input_rwios);
if (rv < 0) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
record_count = 0;
cur_node = record_buffer;
while (input_rwios != NULL) {
/* read record */
if ((rv = skStreamReadRecord(input_rwios, (rwRec*)cur_node))
!= SKSTREAM_OK)
{
if (rv != SKSTREAM_ERR_EOF) {
skStreamPrintLastErr(input_rwios, rv, &skAppPrintErr);
}
/* end of file: close current and open next */
skStreamDestroy(&input_rwios);
rv = appNextInput(&input_rwios);
if (rv < 0) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
continue;
}
++record_count;
cur_node += NODE_SIZE;
if (record_count == buffer_recs) {
/* Filled the current buffer */
/* If buffer not at max size, see if we can grow it */
if (buffer_recs < buffer_max_recs) {
uint8_t *old_buf = record_buffer;
/* add a chunk of records. if we are near the max,
* set the size to the max */
buffer_recs += buffer_chunk_recs;
if (buffer_recs + buffer_chunk_recs > buffer_max_recs) {
buffer_recs = buffer_max_recs;
}
TRACEMSG((("Buffer full---attempt to grow to %" PRIu32
" records, %" PRIu32 " bytes"),
buffer_recs, NODE_SIZE * buffer_recs));
/* attempt to grow */
record_buffer = (uint8_t*)realloc(record_buffer,
NODE_SIZE * buffer_recs);
if (record_buffer) {
/* Success, make certain cur_node points into the
* new buffer */
cur_node = (record_buffer + (record_count * NODE_SIZE));
} else {
/* Unable to grow it */
TRACEMSG(("realloc() failed"));
record_buffer = old_buf;
buffer_max_recs = buffer_recs = record_count;
}
}
/* Either buffer at maximum size or attempt to grow it
* failed. */
if (record_count == buffer_max_recs) {
/* Sort */
skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare);
/* Write to temp file */
if (skTempFileWriteBufferStream(
tmpctx, &temp_file_idx,
record_buffer, NODE_SIZE, record_count))
{
skAppPrintSyserror(
"Error writing sorted buffer to temporary file");
free(record_buffer);
appExit(EXIT_FAILURE);
}
/* Reset record buffer to 'empty' */
record_count = 0;
cur_node = record_buffer;
}
}
}
/* Sort (and maybe store) last batch of records */
if (record_count > 0) {
skQSort(record_buffer, record_count, NODE_SIZE, &rwrecCompare);
if (temp_file_idx >= 0) {
/* Write last batch to temp file */
if (skTempFileWriteBufferStream(
tmpctx, &temp_file_idx,
record_buffer, NODE_SIZE, record_count))
{
skAppPrintSyserror(
"Error writing sorted buffer to temporary file");
free(record_buffer);
appExit(EXIT_FAILURE);
}
}
}
/* Generate the output */
if (record_count == 0 && temp_file_idx == -1) {
/* No records were read at all; write the header to the output
* file */
rv = skStreamWriteSilkHeader(out_rwios);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
}
} else if (temp_file_idx == -1) {
/* No temp files written, just output batch of records */
uint32_t c;
TRACEMSG((("Writing %" PRIu32 " records to '%s'"),
record_count, skStreamGetPathname(out_rwios)));
/* get first two records from the sorted buffer */
cur_node = record_buffer;
next_node = record_buffer + NODE_SIZE;
for (c = 1; c < record_count; ++c, next_node += NODE_SIZE) {
if (0 != rwrecCompare(cur_node, next_node)) {
/* records differ. print earlier record */
rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
if (SKSTREAM_ERROR_IS_FATAL(rv)) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
}
cur_node = next_node;
}
/* else records are duplicates: ignore latter record */
}
/* print remaining record */
rv = skStreamWriteRecord(out_rwios, (rwRec*)cur_node);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
if (SKSTREAM_ERROR_IS_FATAL(rv)) {
free(record_buffer);
appExit(EXIT_FAILURE);
}
}
} else {
/* no longer have a need for the record buffer */
free(record_buffer);
record_buffer = NULL;
/* now merge all the temp files */
mergeFiles(temp_file_idx);
}
if (record_buffer) {
free(record_buffer);
}
}
/*
* mergeFiles(temp_file_idx)
*
* Merge the temporary files numbered from 0 to 'temp_file_idx'
* inclusive into the output file 'out_ios', maintaining sorted
* order. Exits the application if an error occurs.
*/
static void
mergeFiles(
int temp_file_idx)
{
char errbuf[2 * PATH_MAX];
skstream_t *fps[MAX_MERGE_FILES];
uint8_t recs[MAX_MERGE_FILES][NODE_SIZE];
uint8_t lowest_rec[NODE_SIZE];
int j;
uint16_t open_count;
uint16_t i;
uint16_t lowest;
uint16_t *top_heap;
int tmp_idx_a;
int tmp_idx_b;
skstream_t *fp_intermediate = NULL;
int tmp_idx_intermediate;
skheap_t *heap;
uint32_t heap_count;
int opened_all_temps = 0;
ssize_t rv;
/* the index of the first temp file to the merge */
tmp_idx_a = 0;
TRACEMSG(("Merging #%d through #%d to '%s'",
tmp_idx_a, temp_file_idx, skStreamGetPathname(out_rwios)));
heap = skHeapCreate2(compHeapNodes, MAX_MERGE_FILES, sizeof(uint16_t),
NULL, recs);
if (NULL == heap) {
skAppPrintOutOfMemory("heap");
appExit(EXIT_FAILURE);
}
/* This loop repeats as long as we haven't read all of the temp
* files generated in the qsort stage. */
do {
assert(SKHEAP_ERR_EMPTY==skHeapPeekTop(heap,(skheapnode_t*)&top_heap));
/* the index of the list temp file to merge */
tmp_idx_b = temp_file_idx;
/* open an intermediate temp file. The merge-sort will have
* to write records here if there are not enough file handles
* available to open all the existing tempoary files. */
fp_intermediate = skTempFileCreateStream(tmpctx, &tmp_idx_intermediate);
if (fp_intermediate == NULL) {
skAppPrintSyserror("Error creating new temporary file");
appExit(EXIT_FAILURE);
}
/* count number of files we open */
open_count = 0;
/* Attempt to open up to MAX_MERGE_FILES, though we an open
* may fail due to lack of resources (EMFILE or ENOMEM) */
for (j = tmp_idx_a; j <= tmp_idx_b; ++j) {
fps[open_count] = skTempFileOpenStream(tmpctx, j);
if (fps[open_count] == NULL) {
if ((open_count > 0)
&& ((errno == EMFILE) || (errno == ENOMEM)))
{
/* We cannot open any more files. Rewind counter
* by one to catch this file on the next merge */
tmp_idx_b = j - 1;
TRACEMSG((("FILE limit hit--"
"merging #%d through #%d into #%d: %s"),
tmp_idx_a, tmp_idx_b, tmp_idx_intermediate,
strerror(errno)));
break;
} else {
skAppPrintSyserror(("Error opening existing"
" temporary file '%s'"),
skTempFileGetName(tmpctx, j));
appExit(EXIT_FAILURE);
}
}
/* read the first record */
rv = skStreamRead(fps[open_count], recs[open_count], NODE_SIZE);
if (NODE_SIZE == rv) {
/* insert the file index into the heap */
skHeapInsert(heap, &open_count);
++open_count;
if (open_count == MAX_MERGE_FILES) {
/* We've reached the limit for this pass. Set
* tmp_idx_b to the file we just opened. */
tmp_idx_b = j;
TRACEMSG((("MAX_MERGE_FILES limit hit--"
"merging #%d through #%d to #%d"),
tmp_idx_a, tmp_idx_b, tmp_idx_intermediate));
break;
}
} else if (0 == rv) {
TRACEMSG(("Ignoring empty temporary file '%s'",
skTempFileGetName(tmpctx, j)));
skStreamDestroy(&fps[open_count]);
} else {
if (rv > 0) {
snprintf(errbuf, sizeof(errbuf),
"Short read %" SK_PRIdZ "/%" PRIu32 " from '%s'",
rv, NODE_SIZE,
skStreamGetPathname(fps[open_count]));
} else {
skStreamLastErrMessage(
fps[open_count], rv, errbuf, sizeof(errbuf));
}
skAppPrintErr(
"Error reading first record from temporary file: %s",
errbuf);
appExit(EXIT_FAILURE);
}
}
/* Here, we check to see if we've opened all temp files. If
* so, set a flag so we write data to final destination and
* break out of the loop after we're done. */
if (tmp_idx_b == temp_file_idx) {
opened_all_temps = 1;
/* no longer need the intermediate temp file */
skStreamDestroy(&fp_intermediate);
} else {
/* we could not open all temp files, so merge all opened
* temp files into the intermediate file. Add the
* intermediate file to the list of files to merge */
temp_file_idx = tmp_idx_intermediate;
}
TRACEMSG((("Merging %" PRIu16 " temporary files"), open_count));
heap_count = skHeapGetNumberEntries(heap);
assert(heap_count == open_count);
/* get the index of the file with the lowest record; which is
* at the top of the heap */
if (skHeapPeekTop(heap, (skheapnode_t*)&top_heap) != SKHEAP_OK) {
skAppPrintErr("Unable to open and read any temporary files.");
appExit(EXIT_FAILURE);
}
lowest = *top_heap;
/* exit this do...while() once all records for all opened
* files have been read */
do {
/* lowest_rec is the record pointed to by the index at the
* top of the heap */
memcpy(lowest_rec, recs[lowest], NODE_SIZE);
/* write the record */
if (fp_intermediate) {
/* write the record to intermediate tmp file */
rv = skStreamWrite(fp_intermediate, lowest_rec, NODE_SIZE);
if (NODE_SIZE != rv) {
skAppPrintSyserror(
"Error writing record to temporary file '%s'",
skTempFileGetName(tmpctx, tmp_idx_intermediate));
appExit(EXIT_FAILURE);
}
} else {
/* we successfully opened all (remaining) temp files,
* write to record to the final destination */
rv = skStreamWriteRecord(out_rwios, (rwRec*)lowest_rec);
if (0 != rv) {
skStreamPrintLastErr(out_rwios, rv, &skAppPrintErr);
if (SKSTREAM_ERROR_IS_FATAL(rv)) {
appExit(EXIT_FAILURE);
}
}
}
/* replace the record we just processed and loop over all
* files until we get a record that is not a duplicate */
do {
if ((rv = skStreamRead(fps[lowest], recs[lowest], NODE_SIZE))
!= NODE_SIZE)
{
/* read failed. there is no more data for this
* file; remove it from the heap; if the heap is
* empty, exit the loop */
skHeapExtractTop(heap, NULL);
--heap_count;
#if TRACEMSG_LEVEL > 0
if (rv == 0) {
TRACEMSG(
("Finished reading file #%u: EOF; %u files remain",
(tmp_idx_a + lowest), heap_count));
} else if (rv > 0) {
TRACEMSG(
("Finished reading file #%u: Short read "
"%" SK_PRIdZ "/%" PRIu32 "; %u files remain",
tmp_idx_a + lowest, rv, NODE_SIZE, heap_count));
} else {
skStreamLastErrMessage(
fps[open_count], rv, errbuf, sizeof(errbuf));
TRACEMSG(
("Finished reading file #%u: %s; %u files remain",
(tmp_idx_a + lowest), errbuf, heap_count));
}
#endif /* TRACEMSG_LEVEL */
if (0 == heap_count) {
break;
}
} else if (rwrecCompare(lowest_rec, recs[lowest])) {
/* read succeeded. new record is not a
* duplicate and we insert it into the heap */
/* FIXME: This comparison reduces work when the
* keys are the same, but it adds another
* comparison when the keys are different; is this
* an overall win or lose? */
skHeapReplaceTop(heap, &lowest, NULL);
} else {
/* read succeeded. record is a duplicate; ignore
* the record and leave the heap unchanged */
continue;
}
/* get the record at the top of the heap and see if it
* is a duplicate; if it is, ignore it. */
skHeapPeekTop(heap, (skheapnode_t*)&top_heap);
lowest = *top_heap;
} while (0 == rwrecCompare(lowest_rec, recs[lowest]));
} while (heap_count > 0);
TRACEMSG((("Finished processing #%d through #%d"),
tmp_idx_a, tmp_idx_b));
/* Close all open temp files */
for (i = 0; i < open_count; ++i) {
skStreamDestroy(&fps[i]);
}
/* Delete all temp files we opened (or attempted to open) this
* time */
for (j = tmp_idx_a; j <= tmp_idx_b; ++j) {
skTempFileRemove(tmpctx, j);
}
/* Close the intermediate temp file. */
if (fp_intermediate) {
rv = skStreamClose(fp_intermediate);
if (rv) {
skStreamLastErrMessage(
fp_intermediate, rv, errbuf, sizeof(errbuf));
skAppPrintErr("Error closing temporary file: %s", errbuf);
skStreamDestroy(&fp_intermediate);
appExit(EXIT_FAILURE);
}
skStreamDestroy(&fp_intermediate);
}
/* Start the next merge with the next input temp file */
tmp_idx_a = tmp_idx_b + 1;
} while (!opened_all_temps);
skHeapFree(heap);
}
/*
* appSetup(argc, argv);
*
* Perform all the setup for this application include setting up
* required modules, parsing options, etc. This function should be
* passed the same arguments that were passed into main().
*
* Returns to the caller if all setup succeeds. If anything fails,
* this function will cause the application to exit with a FAILURE
* exit status.
*/
static void
appSetup(
int argc,
char **argv)
{
SILK_FEATURES_DEFINE_STRUCT(features);
unsigned int optctx_flags;
int rv;
/* verify same number of options and help strings */
assert((sizeof(appHelp)/sizeof(char *)) ==
(sizeof(appOptions)/sizeof(struct option)));
/* register the application */
skAppRegister(argv[0]);
skAppVerifyFeatures(&features, NULL);
skOptionsSetUsageCallback(&appUsageLong);
/* initialize globals */
memset(&ab_options, 0, sizeof(sk_aggbag_options_t));
ab_options.existing_silk_files = 1;
ab_options.argc = argc;
ab_options.argv = argv;
optctx_flags = (SK_OPTIONS_CTX_INPUT_SILK_FLOW | SK_OPTIONS_CTX_ALLOW_STDIN
| SK_OPTIONS_CTX_XARGS | SK_OPTIONS_CTX_COPY_INPUT
| SK_OPTIONS_CTX_PRINT_FILENAMES);
/* register the options */
if (skOptionsCtxCreate(&optctx, optctx_flags)
|| skOptionsCtxOptionsRegister(optctx)
|| skOptionsRegister(appOptions, &appOptionsHandler, NULL)
|| skAggBagOptionsRegister(&ab_options)
|| skIPv6PolicyOptionsRegister(&ipv6_policy)
|| sksiteOptionsRegister(SK_SITE_FLAG_CONFIG_FILE))
{
skAppPrintErr("Unable to register options");
exit(EXIT_FAILURE);
}
/* register the teardown handler */
if (atexit(appTeardown) < 0) {
skAppPrintErr("Unable to register appTeardown() with atexit()");
exit(EXIT_FAILURE);
}
/* parse options */
rv = skOptionsCtxOptionsParse(optctx, argc, argv);
if (rv < 0) {
skAppUsage(); /* never returns */
}
/* try to load site config file; if it fails, we will not be able
* to resolve flowtype and sensor from input file names, but we
* should not consider it a complete failure */
sksiteConfigure(0);
/* make sure the user specified at least one key field and one
* counter field */
if (keys_arg == NULL || keys_arg[0] == '\0') {
skAppPrintErr("The --%s switch is required",
appOptions[OPT_KEYS].name);
skAppUsage(); /* never returns */
}
if (counters_arg == NULL || counters_arg[0] == '\0') {
skAppPrintErr("The --%s switch is required",
appOptions[OPT_COUNTERS].name);
skAppUsage(); /* never returns */
}
/* set up the key_name_map and counter_name_map */
if (createStringmaps()) {
exit(EXIT_FAILURE);
}
/* create the aggregate bag */
if (skAggBagCreate(&ab)) {
exit(EXIT_FAILURE);
}
skAggBagOptionsBind(ab, &ab_options);
/* parse the --keys and --counters switches */
if (parseFields(key_name_map, keys_arg, OPT_KEYS)) {
exit(EXIT_FAILURE);
}
if (parseFields(counter_name_map, counters_arg, OPT_COUNTERS)) {
exit(EXIT_FAILURE);
}
/* create output stream to stdout if no --output-path was given */
if (NULL == output) {
if ((rv = skStreamCreate(&output, SK_IO_WRITE, SK_CONTENT_SILK))
|| (rv = skStreamBind(output, "-")))
{
skStreamPrintLastErr(output, rv, &skAppPrintErr);
skStreamDestroy(&output);
exit(EXIT_FAILURE);
}
}
/* make certain stdout is not being used for multiple outputs */
if (skStreamIsStdout(output) && skOptionsCtxCopyStreamIsStdout(optctx)) {
skAppPrintErr("May not use stdout for multiple output streams");
exit(EXIT_FAILURE);
}
/* open the output stream but do not write anything yet */
rv = skStreamOpen(output);
if (rv) {
skStreamPrintLastErr(output, rv, &skAppPrintErr);
skStreamDestroy(&output);
exit(EXIT_FAILURE);
}
/* open the --copy-input stream */
if (skOptionsCtxOpenStreams(optctx, &skAppPrintErr)) {
exit(EXIT_FAILURE);
}
return; /* OK */
}
/*
* buildIPSetRanges(stream);
*
* Read IP addresses from the stream named by 'stream' and use them
* to build the global ipset. Allow the input to support ranges of
* IPs. Return 0 on success or -1 on failure.
*/
static int
buildIPSetRanges(
skstream_t *stream)
{
#if SK_ENABLE_IPV6
int saw_integer = 0;
#endif
int lc = 0;
char line_buf[512];
char *sep;
skipaddr_t ip;
skipaddr_t ip_min;
skipaddr_t ip_max;
uint32_t prefix;
const int delim_is_space = isspace((int)delimiter);
int rv;
/* read until end of file */
while ((rv = skStreamGetLine(stream, line_buf, sizeof(line_buf), &lc))
!= SKSTREAM_ERR_EOF)
{
switch (rv) {
case SKSTREAM_OK:
/* good, we got our line */
break;
case SKSTREAM_ERR_LONG_LINE:
/* bad: line was longer than sizeof(line_buf) */
skAppPrintErr("Input line %d too long. ignored",
lc);
continue;
default:
/* unexpected error */
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
goto END;
}
/* support whitespace separators */
if (!delim_is_space) {
sep = strchr(line_buf, delimiter);
} else {
/* ignore leading whitespace */
sep = line_buf;
while (isspace((int)*sep)) {
++sep;
}
sep = strchr(sep, delimiter);
if (sep) {
/* allow a lone IP to have trailing whitespace */
char *cp = sep;
while (isspace((int)*cp)) {
++cp;
}
if (*cp == '\0') {
sep = NULL;
}
}
}
if (sep == NULL) {
/* parse as IP with possible CIDR designation */
rv = skStringParseCIDR(&ip, &prefix, line_buf);
if (rv != 0) {
skAppPrintErr("Invalid IP on line %d: %s",
lc, skStringParseStrerror(rv));
goto END;
}
#if SK_ENABLE_IPV6
/* do not allow integers mixed with IPv6 addresses */
if (saw_integer) {
if (skipaddrIsV6(&ip)) {
skAppPrintErr("Error on line %d: %s",
lc, SETBUILD_ERR_MIX_INT_V6);
rv = -1;
goto END;
}
} else if (SETBUILD_BUF_IS_INT(line_buf)) {
saw_integer = 1;
if (skIPSetIsV6(ipset)) {
skAppPrintErr("Error on line %d: %s",
lc, SETBUILD_ERR_MIX_INT_V6);
rv = -1;
goto END;
}
}
#endif /* SK_ENABLE_IPV6 */
rv = skIPSetInsertAddress(ipset, &ip, prefix);
if (rv) {
skAppPrintErr("Error adding IP on line %d to IPset: %s",
lc, skIPSetStrerror(rv));
goto END;
}
continue;
}
/* parse two IP addresses */
*sep = '\0';
++sep;
rv = skStringParseIP(&ip_min, line_buf);
if (rv != 0) {
skAppPrintErr("Invalid minimum IP on line %d: %s",
lc, skStringParseStrerror(rv));
goto END;
}
rv = skStringParseIP(&ip_max, sep);
if (rv != 0) {
skAppPrintErr("Invalid maximum IP on line %d: %s",
lc, skStringParseStrerror(rv));
goto END;
}
if (skipaddrCompare(&ip_min, &ip_max) > 0) {
skAppPrintErr("Invalid IP range on line %d: min > max",
lc);
rv = -1;
goto END;
}
#if SK_ENABLE_IPV6
/* do not allow integers mixed with IPv6 addresses */
if (saw_integer) {
if (skipaddrIsV6(&ip_min) || skipaddrIsV6(&ip_max)) {
skAppPrintErr("Error on line %d: %s",
lc, SETBUILD_ERR_MIX_INT_V6);
rv = -1;
goto END;
}
} else if (SETBUILD_BUF_IS_INT(line_buf) || SETBUILD_BUF_IS_INT(sep)) {
saw_integer = 1;
if (skIPSetIsV6(ipset)) {
skAppPrintErr("Error on line %d: %s",
lc, SETBUILD_ERR_MIX_INT_V6);
rv = -1;
goto END;
}
}
#endif /* SK_ENABLE_IPV6 */
rv = skIPSetInsertRange(ipset, &ip_min, &ip_max);
if (rv) {
skAppPrintErr("Error adding IP range on line %d to IPset: %s",
lc, skIPSetStrerror(rv));
goto END;
}
}
/* success */
rv = 0;
END:
if (rv != 0) {
return -1;
}
return 0;
}
/*
* appSetup(argc, argv);
*
* Perform all the setup for this application include setting up
* required modules, parsing options, etc. This function should be
* passed the same arguments that were passed into main().
*
* Returns to the caller if all setup succeeds. If anything fails,
* this function will cause the application to exit with a FAILURE
* exit status.
*/
static void
appSetup(
int argc,
char **argv)
{
SILK_FEATURES_DEFINE_STRUCT(features);
const char *in_fname;
const char *out_fname;
int arg_index;
int rv;
/* verify same number of options and help strings */
assert((sizeof(appHelp)/sizeof(char *)) ==
(sizeof(appOptions)/sizeof(struct option)));
/* register the application */
skAppRegister(argv[0]);
skAppVerifyFeatures(&features, NULL);
skOptionsSetUsageCallback(&appUsageLong);
/* initialize globals */
memset(&set_options, 0, sizeof(skipset_options_t));
set_options.argc = argc;
set_options.argv = argv;
/* register the options */
if (skOptionsRegister(appOptions, &appOptionsHandler, NULL)
|| skIPSetOptionsRegister(&set_options))
{
skAppPrintErr("Unable to register options");
exit(EXIT_FAILURE);
}
/* register the teardown handler */
if (atexit(appTeardown) < 0) {
skAppPrintErr("Unable to register appTeardown() with atexit()");
appTeardown();
exit(EXIT_FAILURE);
}
/* parse the options */
arg_index = skOptionsParse(argc, argv);
if (arg_index < 0) {
skAppUsage(); /* never returns */
}
/* default is to read from stdin and write to stdout */
in_fname = "-";
out_fname = "-";
/* process files named on the command line */
switch (argc - arg_index) {
case 2:
in_fname = argv[arg_index++];
out_fname = argv[arg_index++];
break;
case 1:
in_fname = argv[arg_index++];
break;
case 0:
/* Do not allow reading from a tty when no input */
if (FILEIsATty(stdin)) {
skAppPrintErr("Must specify '-' as the input to read"
" from a terminal");
exit(EXIT_FAILURE);
}
break;
default:
skAppPrintErr("Too many arguments;"
" a maximum of two files may be specified");
skAppUsage();
}
/* we should have processed all arguments */
assert(arg_index == argc);
/* create the IPset */
if (skIPSetCreate(&ipset, 0)) {
EXIT_NO_MEMORY;
}
skIPSetOptionsBind(ipset, &set_options);
/* create input */
if ((rv = skStreamCreate(&in_stream, SK_IO_READ, SK_CONTENT_TEXT))
|| (rv = skStreamBind(in_stream, in_fname))
|| (rv = skStreamSetCommentStart(in_stream, "#")))
{
skStreamPrintLastErr(in_stream, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
/* create output */
if ((rv = skStreamCreate(&out_stream, SK_IO_WRITE, SK_CONTENT_SILK))
|| (rv = skStreamBind(out_stream, out_fname)))
{
skStreamPrintLastErr(out_stream, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
/* open the streams */
rv = skStreamOpen(in_stream);
if (rv) {
skStreamPrintLastErr(in_stream, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
rv = skStreamOpen(out_stream);
if (rv) {
skStreamPrintLastErr(out_stream, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
}
/*
* status = appOptionsHandler(cData, opt_index, opt_arg);
*
* This function is passed to skOptionsRegister(); it will be called
* by skOptionsParse() for each user-specified switch that the
* application has registered; it should handle the switch as
* required---typically by setting global variables---and return 1
* if the switch processing failed or 0 if it succeeded. Returning
* a non-zero from from the handler causes skOptionsParse() to return
* a negative value.
*
* The clientData in 'cData' is typically ignored; 'opt_index' is
* the index number that was specified as the last value for each
* struct option in appOptions[]; 'opt_arg' is the user's argument
* to the switch for options that have a REQUIRED_ARG or an
* OPTIONAL_ARG.
*/
static int
appOptionsHandler(
clientData UNUSED(cData),
int opt_index,
char *opt_arg)
{
size_t sz;
int rv;
switch ((appOptionsEnum)opt_index) {
case OPT_SILK_OUTPUT:
if (silk_output) {
skAppPrintErr("Invalid %s: Switch used multiple times",
appOptions[opt_index].name);
return 1;
}
if ((rv =skStreamCreate(&silk_output,SK_IO_WRITE,SK_CONTENT_SILK_FLOW))
|| (rv = skStreamBind(silk_output, opt_arg)))
{
skStreamPrintLastErr(silk_output, rv, &skAppPrintErr);
exit(EXIT_FAILURE);
}
break;
case OPT_PRINT_STATISTICS:
print_statistics = 1;
break;
case OPT_LOG_DESTINATION:
if ('\0' != log_destination[0]) {
skAppPrintErr("Invalid %s: Switch used multiple times",
appOptions[opt_index].name);
}
if ('\0' == opt_arg[0]) {
skAppPrintErr("Invalid %s: Path name is required",
appOptions[opt_index].name);
return 1;
}
if (0 == strcmp("stdout", opt_arg)
|| 0 == strcmp("stderr", opt_arg)
|| 0 == strcmp("none", opt_arg))
{
strncpy(log_destination, opt_arg, sizeof(log_destination));
break;
}
if ('/' == opt_arg[0]) {
if (strlen(opt_arg) >= sizeof(log_destination)) {
skAppPrintErr("Invalid %s: Name is too long",
appOptions[opt_index].name);
return 1;
}
strncpy(log_destination, opt_arg, sizeof(log_destination));
break;
}
if (NULL == getcwd(log_destination, sizeof(log_destination))) {
skAppPrintSyserror("Unable to get current directory");
return 1;
}
sz = strlen(log_destination);
if (sz + strlen(opt_arg) + 1 >= sizeof(log_destination)) {
skAppPrintErr("Invalid %s: Name is too long",
appOptions[opt_index].name);
return 1;
}
snprintf(log_destination + sz, sizeof(log_destination) - sz, "/%s",
opt_arg);
break;
case OPT_LOG_FLAGS:
if (log_flags) {
skAppPrintErr("Invaild %s: Switch used multiple times",
appOptions[opt_index].name);
return 1;
}
log_flags = opt_arg;
break;
}
return 0; /* OK */
}
/*
* appSetup(argc, argv);
*
* Perform all the setup for this application include setting up
* required modules, parsing options, etc. This function should be
* passed the same arguments that were passed into main().
*
* Returns to the caller if all setup succeeds. If anything fails,
* this function will cause the application to exit with a FAILURE
* exit status.
*/
static void
appSetup(
int argc,
char **argv)
{
SILK_FEATURES_DEFINE_STRUCT(features);
int rv;
/* verify same number of options and help strings */
assert((sizeof(appHelp)/sizeof(char *)) ==
(sizeof(appOptions)/sizeof(struct option)));
/* register the application */
skAppRegister(argv[0]);
skAppVerifyFeatures(&features, NULL);
skOptionsSetUsageCallback(&appUsageLong);
/* initialize globals */
#error "Initialize any global variables here"
/* for example: set global output to NULL */
out_stream = NULL;
/* register the options */
if (skOptionsRegister(appOptions, &appOptionsHandler, NULL)
|| sksiteOptionsRegister(SK_SITE_FLAG_CONFIG_FILE))
{
skAppPrintErr("Unable to register options");
exit(EXIT_FAILURE);
}
#error "Do any other module setup here"
/* register the teardown handler */
if (atexit(appTeardown) < 0) {
skAppPrintErr("Unable to register appTeardown() with atexit()");
appTeardown();
exit(EXIT_FAILURE);
}
/* parse the options */
arg_index = skOptionsParse(argc, argv);
if (arg_index < 0) {
/* options parsing should print error */
skAppUsage(); /* never returns */
}
/* try to load site config file; if it fails, we will not be able
* to resolve flowtype and sensor from input file names */
sksiteConfigure(0);
#error "Do any options validation here"
#error "If you expect filenames on command line keep this:"
/* arg_index is looking at first file name to process */
if (arg_index == argc) {
if (FILEIsATty(stdin)) {
skAppPrintErr("No input files on command line and"
" stdin is connected to a terminal");
skAppUsage(); /* never returns */
}
}
#error "If you do NOT expect filenames on command line, keep this:"
/* check for extraneous arguments */
if (arg_index != argc) {
skAppPrintErr("Too many arguments or unrecognized switch '%s'",
argv[arg_index]);
skAppUsage(); /* never returns */
}
#error "Once all options are set, open input and output"
/* for example, open a SiLK flow file as an output file */
rv = skStreamOpenSilkFlow(&out_stream, output_path, SK_IO_WRITE);
if (rv) {
skStreamPrintLastErr(out_stream, rv, &skAppPrintErr);
skAppPrintErr("Unable to open output file. Exiting");
skStreamDestroy(&out_stream);
exit(EXIT_FAILURE);
}
return; /* OK */
}
/*
* buildIPSetWildcards(stream);
*
* Read IP addresses from the stream named by 'stream' and use them
* to build the global ipset. Allow the input to contain
* IPWildcards. Return 0 on success or -1 on failure.
*/
static int
buildIPSetWildcards(
skstream_t *stream)
{
#if SK_ENABLE_IPV6
int saw_integer = 0;
#endif
int lc = 0;
char line_buf[512];
skIPWildcard_t ipwild;
skipaddr_t ip;
uint32_t prefix;
char *cp;
int rv;
/* read until end of file */
while ((rv = skStreamGetLine(stream, line_buf, sizeof(line_buf), &lc))
!= SKSTREAM_ERR_EOF)
{
switch (rv) {
case SKSTREAM_OK:
/* good, we got our line */
break;
case SKSTREAM_ERR_LONG_LINE:
/* bad: line was longer than sizeof(line_buf) */
skAppPrintErr("Input line %d too long. ignored",
lc);
continue;
default:
/* unexpected error */
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
goto END;
}
/* first, attempt to parse as a CIDR block */
rv = skStringParseCIDR(&ip, &prefix, line_buf);
if (rv == 0) {
#if SK_ENABLE_IPV6
/* do not allow integers mixed with IPv6 addresses */
if (saw_integer) {
if (skipaddrIsV6(&ip)) {
skAppPrintErr("Error on line %d: %s",
lc, SETBUILD_ERR_MIX_INT_V6);
rv = -1;
goto END;
}
} else if (SETBUILD_BUF_IS_INT(line_buf)) {
saw_integer = 1;
if (skIPSetIsV6(ipset)) {
skAppPrintErr("Error on line %d: %s",
lc, SETBUILD_ERR_MIX_INT_V6);
rv = -1;
goto END;
}
}
#endif /* SK_ENABLE_IPV6 */
rv = skIPSetInsertAddress(ipset, &ip, prefix);
if (rv) {
skAppPrintErr("Error adding IP on line %d to IPset: %s",
lc, skIPSetStrerror(rv));
goto END;
}
continue;
}
/* else parse the line as an IPWildcard */
rv = skStringParseIPWildcard(&ipwild, line_buf);
if (rv != 0) {
/* failed to parse an IPWildcard. See if the user has
* entered two IP addresses, and if so, suggest they use
* the --ip-ranges switch. */
int rv2;
rv2 = skStringParseIP(&ip, line_buf);
if (rv2 > 0) {
/* parsed an IP and there is extra text after the IP
* address; check to see if it is another IP addr */
#if SK_ENABLE_IPV6
if (skipaddrIsV6(&ip)
&& ((cp = strchr(line_buf + rv2, ':')) != NULL))
{
while (isxdigit((int) *(cp - 1))) {
--cp;
}
if (skStringParseIP(&ip, cp) == 0) {
skAppPrintErr(("Invalid IP on line %d: "
SUGGEST_IP_RANGES),
lc);
goto END;
}
}
#endif
if (!skipaddrIsV6(&ip)
&& ((cp = strchr(line_buf + rv2, '.')) != NULL))
{
while (isxdigit((int) *(cp - 1))) {
--cp;
}
if (skStringParseIP(&ip, cp) == 0) {
skAppPrintErr(("Invalid IP on line %d: "
SUGGEST_IP_RANGES),
lc);
goto END;
}
}
}
/* report initial error */
skAppPrintErr("Invalid IP Wildcard on line %d: %s",
lc, skStringParseStrerror(rv));
goto END;
}
#if SK_ENABLE_IPV6
/* do not allow integers mixed with IPv6 addresses */
if (saw_integer && skIPWildcardIsV6(&ipwild)) {
skAppPrintErr("Error on line %d: %s",
lc, SETBUILD_ERR_MIX_INT_V6);
rv = -1;
goto END;
}
#endif /* SK_ENABLE_IPV6 */
rv = skIPSetInsertIPWildcard(ipset, &ipwild);
if (rv) {
skAppPrintErr("Error adding IP Wildcard on line %d to IPset: %s",
lc, skIPSetStrerror(rv));
goto END;
}
}
/* success */
rv = 0;
END:
if (rv != 0) {
return -1;
}
return 0;
}
/*
* status = cutFile(stream);
*
* Read SiLK flow records from the file at 'stream' and maybe print
* them according the values in 'skip_recs' and 'num_recs'.
*
* Return -1 on error. Return 1 if all requested records have been
* printed and processing should stop. Return 0 if processing
* should continue.
*/
static int
cutFile(
skstream_t *stream)
{
static int copy_input_only = 0;
rwRec rwrec;
int rv = SKSTREAM_OK;
size_t num_skipped;
int ret_val = 0;
/* handle case where all requested records have been printed, but
* we need to write all records to the --copy-input stream. */
if (copy_input_only) {
while ((rv = skStreamSkipRecords(stream, CUT_SKIP_COUNT, NULL))
== SKSTREAM_OK)
; /* empty */
if (SKSTREAM_ERR_EOF != rv) {
ret_val = -1;
}
goto END;
}
/* skip any leading records */
if (skip_recs) {
rv = skStreamSkipRecords(stream, skip_recs, &num_skipped);
switch (rv) {
case SKSTREAM_OK:
skip_recs -= num_skipped;
break;
case SKSTREAM_ERR_EOF:
skip_recs -= num_skipped;
goto END;
default:
ret_val = -1;
goto END;
}
}
if (0 == num_recs) {
/* print all records */
while ((rv = skStreamReadRecord(stream, &rwrec)) == SKSTREAM_OK) {
rwAsciiPrintRec(ascii_str, &rwrec);
}
if (SKSTREAM_ERR_EOF != rv) {
ret_val = -1;
}
} else {
while (num_recs
&& ((rv = skStreamReadRecord(stream, &rwrec)) == SKSTREAM_OK))
{
rwAsciiPrintRec(ascii_str, &rwrec);
--num_recs;
}
switch (rv) {
case SKSTREAM_OK:
case SKSTREAM_ERR_EOF:
break;
default:
ret_val = -1;
goto END;
}
if (0 == num_recs) {
if (0 == skOptionsCtxCopyStreamIsActive(optctx)) {
/* we're done */
ret_val = 1;
} else {
/* send all remaining records to copy-input */
copy_input_only = 1;
while ((rv = skStreamSkipRecords(stream, CUT_SKIP_COUNT, NULL))
== SKSTREAM_OK)
; /* empty */
if (SKSTREAM_ERR_EOF != rv) {
ret_val = -1;
}
}
}
}
END:
if (-1 == ret_val) {
skStreamPrintLastErr(stream, rv, &skAppPrintErr);
}
return ret_val;
}
